[llvm] [SPIR-V] Add SPIR-V structurizer (PR #97606)
Nathan Gauër via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 3 09:44:37 PDT 2024
https://github.com/Keenuts created https://github.com/llvm/llvm-project/pull/97606
This commit adds an initial SPIR-V structurizer.
It leverages the previously merged passes, and the convergence region analysis to determine the correct merge and continue blocks for SPIR-V.
The first part replaces switches with a single edge with a simple branch.
Then, we add OpLoopMerge instruction to each loop encountered. Then, we add OpSelectionMerge for each conditional branch which has no OpLoopMerge instruction.
Finally, we fixup the merge instructions:
- we split blocks with multiple headers into 2 blocks.
- we split blocks that are a merge blocks for 2 or more constructs: SPIR-V spec disallow a merge block to be shared by 2 loop/switch/condition construct.
- we split merge & continue blocks: SPIR-V spec disallow a basic block to be both a continue block, and a merge block.
- we remove superfluous headers: when a header doesn't bring more info than the parent on the divergence state, it must be removed.
- We sort blocks depending on the dominator tree order: SPIR-V spec requires blocks to be sorted in a specific way.
As is, this code seems to pass most DXC structurization tests, except the ones relying on unrelated features this backend doesn't yet support like i64 switches, boolean types.
One known case fails, but it is because the MergeExit pass doesn't supports switches yet (there is a FIXME).
This PR is already big-enough as-is, so I think we should get this in first, and then add support for those switches.
>From d468102c0ec46e447d064d1cc073c133f2ae67b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Wed, 15 May 2024 17:23:46 +0200
Subject: [PATCH] [SPIR-V] Add SPIR-V structurizer
This commit adds an initial SPIR-V structurizer.
It leverages the previously merged passes, and the convergence region
analysis to determine the correct merge and continue blocks for SPIR-V.
The first part replaces switches with a single edge with a simple
branch.
Then, we add OpLoopMerge instruction to each loop encountered.
Then, we add OpSelectionMerge for each conditional branch which has
no OpLoopMerge instruction.
Finally, we fixup the merge instructions:
- we split blocks with multiple headers into 2 blocks.
- we split blocks that are a merge blocks for 2 or more constructs:
SPIR-V spec disallow a merge block to be shared by 2
loop/switch/condition construct.
- we split merge & continue blocks:
SPIR-V spec disallow a basic block to be both a continue
block, and a merge block.
- we remove superfluous headers: when a header doesn't bring more info
than the parent on the divergence state, it must be removed.
- We sort blocks depending on the dominator tree order:
SPIR-V spec requires blocks to be sorted in a specific way.
As is, this code seems to pass most DXC structurization tests, except
the ones relying on unrelated features this backend doesn't yet support
like i64 switches, boolean types.
One known case fails, but it is because the MergeExit pass doesn't
supports switches yet (there is a FIXME).
This PR is already big-enough as-is, so I think we should get this in
first, and then add support for those switches.
---
.../SPIRVConvergenceRegionAnalysis.cpp | 4 +-
.../Analysis/SPIRVConvergenceRegionAnalysis.h | 3 +
llvm/lib/Target/SPIRV/CMakeLists.txt | 1 +
llvm/lib/Target/SPIRV/SPIRV.h | 1 +
.../SPIRV/SPIRVMergeRegionExitTargets.cpp | 20 +-
llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp | 649 ++++++++++++++++++
llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 7 +-
.../CodeGen/SPIRV/scfg-add-pre-headers.ll | 80 ---
.../SPIRV/structurizer/condition-linear.ll | 127 ++++
.../CodeGen/SPIRV/structurizer/do-break.ll | 88 +++
.../CodeGen/SPIRV/structurizer/do-continue.ll | 124 ++++
.../CodeGen/SPIRV/structurizer/do-nested.ll | 102 +++
.../CodeGen/SPIRV/structurizer/do-plain.ll | 98 +++
.../CodeGen/SPIRV/structurizer/logical-or.ll | 158 +++++
.../SPIRV/structurizer/merge-exit-break.ll | 39 +-
.../merge-exit-convergence-in-break.ll | 58 +-
.../structurizer/merge-exit-multiple-break.ll | 73 +-
.../merge-exit-simple-white-identity.ll | 10 +-
.../SPIRV/structurizer/return-early.ll | 130 ++++
19 files changed, 1593 insertions(+), 179 deletions(-)
create mode 100644 llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
delete mode 100644 llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-break.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
index 25e285e35f933..d77900001ea4b 100644
--- a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
@@ -230,7 +230,8 @@ class ConvergenceRegionAnalyzer {
auto *Terminator = From->getTerminator();
for (unsigned i = 0; i < Terminator->getNumSuccessors(); ++i) {
auto *To = Terminator->getSuccessor(i);
- if (isBackEdge(From, To))
+ // Ignore back edges and self edges.
+ if (From == To || isBackEdge(From, To))
continue;
auto ChildSet = findPathsToMatch(LI, To, isMatch);
@@ -276,7 +277,6 @@ class ConvergenceRegionAnalyzer {
while (ToProcess.size() != 0) {
auto *L = ToProcess.front();
ToProcess.pop();
- assert(L->isLoopSimplifyForm());
auto CT = getConvergenceToken(L->getHeader());
SmallPtrSet<BasicBlock *, 8> RegionBlocks(L->block_begin(),
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
index f9e30e4effa1d..e435c88c919c9 100644
--- a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
+++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
@@ -130,6 +130,9 @@ class ConvergenceRegionInfo {
}
const ConvergenceRegion *getTopLevelRegion() const { return TopLevelRegion; }
+ ConvergenceRegion *getWritableTopLevelRegion() const {
+ return TopLevelRegion;
+ }
};
} // namespace SPIRV
diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
index 14647e92f5d08..425da84b62db1 100644
--- a/llvm/lib/Target/SPIRV/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_target(SPIRVCodeGen
SPIRVMCInstLower.cpp
SPIRVMetadata.cpp
SPIRVModuleAnalysis.cpp
+ SPIRVStructurizer.cpp
SPIRVPreLegalizer.cpp
SPIRVPostLegalizer.cpp
SPIRVPrepareFunctions.cpp
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index e597a1dc8dc06..d6597db8bc0e6 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -20,6 +20,7 @@ class InstructionSelector;
class RegisterBankInfo;
ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM);
+FunctionPass *createSPIRVStructurizerPass();
FunctionPass *createSPIRVMergeRegionExitTargetsPass();
FunctionPass *createSPIRVStripConvergenceIntrinsicsPass();
FunctionPass *createSPIRVRegularizerPass();
diff --git a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
index 0747dd1bbaf40..9930d067173df 100644
--- a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
@@ -133,7 +133,7 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
// Run the pass on the given convergence region, ignoring the sub-regions.
// Returns true if the CFG changed, false otherwise.
bool runOnConvergenceRegionNoRecurse(LoopInfo &LI,
- const SPIRV::ConvergenceRegion *CR) {
+ SPIRV::ConvergenceRegion *CR) {
// Gather all the exit targets for this region.
SmallPtrSet<BasicBlock *, 4> ExitTargets;
for (BasicBlock *Exit : CR->Exits) {
@@ -198,14 +198,19 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
for (auto Exit : CR->Exits)
replaceBranchTargets(Exit, ExitTargets, NewExitTarget);
+ CR = CR->Parent;
+ while (CR) {
+ CR->Blocks.insert(NewExitTarget);
+ CR = CR->Parent;
+ }
+
return true;
}
/// Run the pass on the given convergence region and sub-regions (DFS).
/// Returns true if a region/sub-region was modified, false otherwise.
/// This returns as soon as one region/sub-region has been modified.
- bool runOnConvergenceRegion(LoopInfo &LI,
- const SPIRV::ConvergenceRegion *CR) {
+ bool runOnConvergenceRegion(LoopInfo &LI, SPIRV::ConvergenceRegion *CR) {
for (auto *Child : CR->Children)
if (runOnConvergenceRegion(LI, Child))
return true;
@@ -235,10 +240,10 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
virtual bool runOnFunction(Function &F) override {
LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
- const auto *TopLevelRegion =
+ auto *TopLevelRegion =
getAnalysis<SPIRVConvergenceRegionAnalysisWrapperPass>()
.getRegionInfo()
- .getTopLevelRegion();
+ .getWritableTopLevelRegion();
// FIXME: very inefficient method: each time a region is modified, we bubble
// back up, and recompute the whole convergence region tree. Once the
@@ -246,9 +251,6 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
// to be efficient instead of simple.
bool modified = false;
while (runOnConvergenceRegion(LI, TopLevelRegion)) {
- TopLevelRegion = getAnalysis<SPIRVConvergenceRegionAnalysisWrapperPass>()
- .getRegionInfo()
- .getTopLevelRegion();
modified = true;
}
@@ -262,6 +264,8 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
AU.addRequired<DominatorTreeWrapperPass>();
AU.addRequired<LoopInfoWrapperPass>();
AU.addRequired<SPIRVConvergenceRegionAnalysisWrapperPass>();
+
+ AU.addPreserved<SPIRVConvergenceRegionAnalysisWrapperPass>();
FunctionPass::getAnalysisUsage(AU);
}
};
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
new file mode 100644
index 0000000000000..ac20f16905022
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -0,0 +1,649 @@
+//===-- SPIRVStructurizer.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass adds the required OpSelection/OpLoop merge instructions to
+// generate valid SPIR-V.
+// This pass trims convergence intrinsics as those were only useful when
+// modifying the CFG during IR passes.
+//
+//===----------------------------------------------------------------------===//
+
+#include <stack>
+
+#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
+#include "SPIRV.h"
+#include "SPIRVSubtarget.h"
+#include "SPIRVTargetMachine.h"
+#include "SPIRVUtils.h"
+#include "llvm/ADT/BreadthFirstIterator.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/IntrinsicLowering.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsSPIRV.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
+
+using namespace llvm;
+using namespace SPIRV;
+
+namespace llvm {
+void initializeSPIRVStructurizerPass(PassRegistry &);
+}
+
+namespace {
+
+// Returns the exact convergence region in the tree defined by `Node` for which
+// `MBB` is the header, nullptr otherwise.
+const ConvergenceRegion *getRegionForHeader(const ConvergenceRegion *Node,
+ MachineBasicBlock *MBB) {
+ if (Node->Entry == MBB->getBasicBlock())
+ return Node;
+
+ for (auto *Child : Node->Children) {
+ const auto *CR = getRegionForHeader(Child, MBB);
+ if (CR != nullptr)
+ return CR;
+ }
+ return nullptr;
+}
+
+// Returns the MachineBasicBlock in `MF` matching `BB`, nullptr otherwise.
+MachineBasicBlock *getMachineBlockFor(MachineFunction &MF, BasicBlock *BB) {
+ for (auto &MBB : MF)
+ if (MBB.getBasicBlock() == BB)
+ return &MBB;
+ return nullptr;
+}
+
+// Gather all the successors of |BB|.
+// This function asserts if the terminator neither a branch, switch or return.
+std::unordered_set<BasicBlock *> gatherSuccessors(BasicBlock *BB) {
+ std::unordered_set<BasicBlock *> output;
+ auto *T = BB->getTerminator();
+
+ if (auto *BI = dyn_cast<BranchInst>(T)) {
+ output.insert(BI->getSuccessor(0));
+ if (BI->isConditional())
+ output.insert(BI->getSuccessor(1));
+ return output;
+ }
+
+ if (auto *SI = dyn_cast<SwitchInst>(T)) {
+ output.insert(SI->getDefaultDest());
+ for (auto &Case : SI->cases())
+ output.insert(Case.getCaseSuccessor());
+ return output;
+ }
+
+ if (auto *RI = dyn_cast<ReturnInst>(T))
+ return output;
+
+ assert(false && "Unhandled terminator type.");
+ return output;
+}
+
+// Returns the single MachineBasicBlock exiting the convergence region `CR`,
+// nullptr if no such exit exists. MF must be the function CR belongs to.
+MachineBasicBlock *getExitFor(MachineFunction &MF,
+ const ConvergenceRegion *CR) {
+ std::unordered_set<BasicBlock *> ExitTargets;
+ for (BasicBlock *Exit : CR->Exits) {
+ for (BasicBlock *Target : gatherSuccessors(Exit)) {
+ if (CR->Blocks.count(Target) == 0)
+ ExitTargets.insert(Target);
+ }
+ }
+
+ assert(ExitTargets.size() <= 1);
+ if (ExitTargets.size() == 0)
+ return nullptr;
+
+ auto *Exit = *ExitTargets.begin();
+ return getMachineBlockFor(MF, Exit);
+}
+
+// Returns true is |I| is a OpLoopMerge or OpSelectionMerge instruction.
+bool isMergeInstruction(const MachineInstr &I) {
+ return I.getOpcode() == SPIRV::OpLoopMerge ||
+ I.getOpcode() == SPIRV::OpSelectionMerge;
+}
+
+// Returns the first OpLoopMerge/OpSelectionMerge instruction found in |MBB|,
+// nullptr otherwise.
+MachineInstr *getMergeInstruction(MachineBasicBlock &MBB) {
+ for (auto &I : MBB) {
+ if (isMergeInstruction(I))
+ return &I;
+ }
+ return nullptr;
+}
+
+// Returns the first OpLoopMerge instruction found in |MBB|, nullptr otherwise.
+MachineInstr *getLoopMergeInstruction(MachineBasicBlock &MBB) {
+ for (auto &I : MBB) {
+ if (I.getOpcode() == SPIRV::OpLoopMerge)
+ return &I;
+ }
+ return nullptr;
+}
+
+// Returns the first OpSelectionMerge instruction found in |MBB|, nullptr
+// otherwise.
+MachineInstr *getSelectionMergeInstruction(MachineBasicBlock &MBB) {
+ for (auto &I : MBB) {
+ if (I.getOpcode() == SPIRV::OpSelectionMerge)
+ return &I;
+ }
+ return nullptr;
+}
+
+// Traverses the CFG DFS-style starting from the entry point.
+// Calls |op| on each basic block encountered during the traversal.
+void visit(MachineFunction &MF, std::function<void(MachineBasicBlock *)> op) {
+ std::stack<MachineBasicBlock *> ToVisit;
+ SmallPtrSet<MachineBasicBlock *, 8> Seen;
+
+ ToVisit.push(&*MF.begin());
+ Seen.insert(ToVisit.top());
+ while (ToVisit.size() != 0) {
+ MachineBasicBlock *MBB = ToVisit.top();
+ ToVisit.pop();
+
+ op(MBB);
+
+ for (auto Succ : MBB->successors()) {
+ if (Seen.contains(Succ))
+ continue;
+ ToVisit.push(Succ);
+ Seen.insert(Succ);
+ }
+ }
+}
+
+// Returns all basic blocks in |MF| with at least one SelectionMerge/LoopMerge
+// instruction.
+SmallPtrSet<MachineBasicBlock *, 8> getHeaderBlocks(MachineFunction &MF) {
+ SmallPtrSet<MachineBasicBlock *, 8> output;
+ for (MachineBasicBlock &MBB : MF) {
+ auto *MI = getMergeInstruction(MBB);
+ if (MI != nullptr)
+ output.insert(&MBB);
+ }
+ return output;
+}
+
+// Returns all basic blocks in |MF| referenced by at least 1
+// OpSelectionMerge/OpLoopMerge instruction.
+SmallPtrSet<MachineBasicBlock *, 8> getMergeBlocks(MachineFunction &MF) {
+ SmallPtrSet<MachineBasicBlock *, 8> output;
+ for (MachineBasicBlock &MBB : MF) {
+ auto *MI = getMergeInstruction(MBB);
+ if (MI != nullptr)
+ output.insert(MI->getOperand(0).getMBB());
+ }
+ return output;
+}
+
+// Returns all basic blocks in |MF| referenced as continue target by at least 1
+// OpLoopMerge.
+SmallPtrSet<MachineBasicBlock *, 8> getContinueBlocks(MachineFunction &MF) {
+ SmallPtrSet<MachineBasicBlock *, 8> output;
+ for (MachineBasicBlock &MBB : MF) {
+ auto *MI = getMergeInstruction(MBB);
+ if (MI != nullptr && MI->getOpcode() == SPIRV::OpLoopMerge)
+ output.insert(MI->getOperand(1).getMBB());
+ }
+ return output;
+}
+
+// Returns the block immediatly post-dominating every block in |range| if any,
+// nullptr otherwise.
+MachineBasicBlock *findNearestCommonDominator(
+ const iterator_range<std::vector<MachineBasicBlock *>::iterator> &range,
+ MachinePostDominatorTree &MPDT) {
+ assert(!range.empty());
+ MachineBasicBlock *Dom = *range.begin();
+ for (MachineBasicBlock *Item : range)
+ Dom = MPDT.findNearestCommonDominator(Dom, Item);
+ return Dom;
+}
+
+// Finds the first merge instruction in |MBB| and store it in |MI|.
+// If it defines a merge target, sets |Merge| to the merge target.
+// If it defines a continue target, sets |Continue| to the continue target.
+// Returns true if such merge instruction was found, false otherwise.
+bool getMergeInstructionTargets(MachineBasicBlock *MBB, MachineInstr **MI,
+ MachineBasicBlock **Merge,
+ MachineBasicBlock **Continue) {
+ *Merge = nullptr;
+ *Continue = nullptr;
+
+ *MI = getMergeInstruction(*MBB);
+ if (*MI == nullptr)
+ return false;
+
+ *Merge = (*MI)->getOperand(0).getMBB();
+ *Continue = (*MI)->getOpcode() == SPIRV::OpLoopMerge
+ ? (*MI)->getOperand(1).getMBB()
+ : nullptr;
+ return true;
+}
+
+} // anonymous namespace
+
+class SPIRVStructurizer : public MachineFunctionPass {
+public:
+ static char ID;
+
+ SPIRVStructurizer() : MachineFunctionPass(ID) {
+ initializeSPIRVStructurizerPass(*PassRegistry::getPassRegistry());
+ };
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ MachineFunctionPass::getAnalysisUsage(AU);
+ AU.addRequired<MachineLoopInfo>();
+ AU.addRequired<DominatorTreeWrapperPass>();
+ AU.addRequired<SPIRVConvergenceRegionAnalysisWrapperPass>();
+ }
+
+ // Creates a MachineBasicBlock ending with an OpReturn just after
+ // |Predecessor|. This function does not add any branch to |Predecessor|, but
+ // adds the new block to its successors.
+ MachineBasicBlock *createReturnBlock(MachineFunction &MF,
+ MachineBasicBlock &Predecessor) {
+ MachineBasicBlock *MBB =
+ MF.CreateMachineBasicBlock(Predecessor.getBasicBlock());
+ MF.push_back(MBB);
+ MBB->moveAfter(&Predecessor);
+ // This code doesn't add a branch instruction to this new return block.
+ // The caller will have to handle that.
+ Predecessor.addSuccessorWithoutProb(MBB);
+
+ MachineIRBuilder MIRBuilder(MF);
+ MIRBuilder.setInsertPt(*MBB, MBB->end());
+ MIRBuilder.buildInstr(SPIRV::OpReturn);
+
+ return MBB;
+ }
+
+ // Replace switches with a single target with an unconditional branch.
+ bool replaceEmptySwitchWithBranch(MachineFunction &MF) {
+ bool modified = false;
+ for (MachineBasicBlock &MBB : MF) {
+ MachineInstr *I = &*MBB.rbegin();
+ GIntrinsic *II = dyn_cast<GIntrinsic>(I);
+ if (!II || II->getIntrinsicID() != Intrinsic::spv_switch ||
+ II->getNumOperands() > 3)
+ continue;
+
+ modified = true;
+ assert(II->getOperand(2).isMBB());
+ MachineBasicBlock *Target = II->getOperand(2).getMBB();
+
+ MachineIRBuilder MIRBuilder(MF);
+ MIRBuilder.setInsertPt(MBB, MBB.end());
+ MIRBuilder.buildBr(*Target);
+ MBB.erase(I);
+ }
+
+ return modified;
+ }
+
+ // Traverse each loop, and adds an OpLoopMerge instruction to its header
+ // that respect the convergence region node it belongs to.
+ // The Continue target is the only back-edge in that loop.
+ // The merge target is the only exiting node of the convergence region.
+ bool addMergeForLoops(MachineFunction &MF) {
+ auto &TII = *MF.getSubtarget<SPIRVSubtarget>().getInstrInfo();
+ auto &TRI = *MF.getSubtarget<SPIRVSubtarget>().getRegisterInfo();
+ auto &RBI = *MF.getSubtarget<SPIRVSubtarget>().getRegBankInfo();
+
+ const auto &MLI = getAnalysis<MachineLoopInfo>();
+ const auto *TopLevelRegion =
+ getAnalysis<SPIRVConvergenceRegionAnalysisWrapperPass>()
+ .getRegionInfo()
+ .getTopLevelRegion();
+
+ bool modified = false;
+ for (auto &MBB : MF) {
+ // Not a loop header. Ignoring for now.
+ if (!MLI.isLoopHeader(&MBB))
+ continue;
+ auto *L = MLI.getLoopFor(&MBB);
+
+ // This loop header is not the entrance of a convergence region. Ignoring
+ // this block.
+ auto *CR = getRegionForHeader(TopLevelRegion, &MBB);
+ if (CR == nullptr)
+ continue;
+
+ auto *Merge = getExitFor(MF, CR);
+ // This is a special case:
+ // We are indeed in a loop, but there are no exits (infinite loop).
+ // This means the actual branch is unconditional, hence won't require any
+ // OpLoopMerge.
+ if (Merge == nullptr) {
+ Merge = createReturnBlock(MF, MBB);
+ }
+
+ auto *Continue = L->getLoopLatch();
+
+ // Conditional branch are built using a fallthrough if false + BR.
+ // So the last instruction is not always the first branch.
+ auto *I = &*MBB.getFirstTerminator();
+ BuildMI(MBB, I, I->getDebugLoc(), TII.get(SPIRV::OpLoopMerge))
+ .addMBB(Merge)
+ .addMBB(Continue)
+ .addImm(SPIRV::SelectionControl::None)
+ .constrainAllUses(TII, TRI, RBI);
+ modified = true;
+ }
+
+ return modified;
+ }
+
+ // Add an OpSelectionMerge to each node with an out-degree of 2 or more.
+ bool addMergeForConditionalBranches(MachineFunction &MF) {
+ MachinePostDominatorTree MPDT(MF);
+
+ auto &TII = *MF.getSubtarget<SPIRVSubtarget>().getInstrInfo();
+ auto &TRI = *MF.getSubtarget<SPIRVSubtarget>().getRegisterInfo();
+ auto &RBI = *MF.getSubtarget<SPIRVSubtarget>().getRegBankInfo();
+
+ auto MergeBlocks = getMergeBlocks(MF);
+ auto ContinueBlocks = getContinueBlocks(MF);
+
+ for (auto &MBB : MF) {
+ if (MBB.succ_size() <= 1)
+ continue;
+
+ // Block already has an OpSelectionMerge instruction. Ignoring.
+ if (getSelectionMergeInstruction(MBB)) {
+ continue;
+ }
+
+ assert(MBB.succ_size() >= 2);
+ size_t NonStructurizedTargets = 0;
+ for (MachineBasicBlock *Successor : MBB.successors()) {
+ if (!MergeBlocks.contains(Successor) &&
+ !ContinueBlocks.contains(Successor))
+ NonStructurizedTargets += 1;
+ }
+
+ if (NonStructurizedTargets <= 1)
+ continue;
+
+ MachineBasicBlock *Merge =
+ findNearestCommonDominator(MBB.successors(), MPDT);
+ if (!Merge) {
+ // TODO: we should check which block is not another construct merge
+ // block, and select this one. For now, tests passes with this strategy,
+ // but once we find a test case, we should fix that.
+ Merge = *MBB.succ_begin();
+ }
+
+ assert(Merge);
+ auto *II = MBB.getFirstTerminator() == MBB.end()
+ ? &*MBB.rbegin()
+ : &*MBB.getFirstTerminator();
+ BuildMI(MBB, II, II->getDebugLoc(), TII.get(SPIRV::OpSelectionMerge))
+ .addMBB(Merge)
+ .addImm(SPIRV::SelectionControl::None)
+ .constrainAllUses(TII, TRI, RBI);
+ }
+
+ return false;
+ }
+
+ // Cut |Block| just after the first OpLoopMerge/OpSelectionMerge instruction.
+ // The newly created block lies just after |Block|, and |Block| branches
+ // unconditionally to this new block. Returns the newly created block.
+ MachineBasicBlock *splitHeaderBlock(MachineFunction &MF,
+ MachineBasicBlock &Block) {
+ auto FirstMerge = Block.begin();
+ while (!isMergeInstruction(*FirstMerge)) {
+ FirstMerge++;
+ }
+
+ MachineBasicBlock *NewBlock = Block.splitAt(*FirstMerge);
+
+ MachineIRBuilder MIRBuilder(MF);
+ MIRBuilder.setInsertPt(Block, Block.end());
+ MIRBuilder.buildBr(*NewBlock);
+
+ return NewBlock;
+ }
+
+ // Split basic blocks containing multiple OpLoopMerge/OpSelectionMerge
+ // instructions so each basic block contains only a single merge instruction.
+ bool splitBlocksWithMultipleHeaders(MachineFunction &MF) {
+ bool modified = false;
+ for (auto &MBB : MF) {
+ MachineInstr *SelectionMerge = getSelectionMergeInstruction(MBB);
+ MachineInstr *LoopMerge = getLoopMergeInstruction(MBB);
+ if (!SelectionMerge || !LoopMerge) {
+ continue;
+ }
+
+ splitHeaderBlock(MF, MBB);
+ modified = true;
+ }
+ return modified;
+ }
+
+ // Splits the basic block |OldMerge| in two.
+ // The newly created block will become the predecessor of |OldMerge|.
+ // |HeaderBlock| becomes the only block using |OldMerge| as merge target.
+ // Each other Merge instruction having |OldMerge| as target will have the
+ // newly created block as target.
+ MachineBasicBlock *splitMergeBlock(MachineDominatorTree &MDT,
+ MachineFunction &MF,
+ MachineBasicBlock &OldMerge,
+ MachineBasicBlock &HeaderBlock) {
+
+ std::vector<MachineBasicBlock *> toUpdate;
+ for (MachineBasicBlock *Predecessor : OldMerge.predecessors())
+ toUpdate.push_back(Predecessor);
+
+ MachineBasicBlock *NewMerge =
+ MF.CreateMachineBasicBlock(OldMerge.getBasicBlock());
+ MF.push_back(NewMerge);
+ NewMerge->moveBefore(&OldMerge);
+ NewMerge->addSuccessorWithoutProb(&OldMerge);
+ MachineIRBuilder MIRBuilder(MF);
+ MIRBuilder.setInsertPt(*NewMerge, NewMerge->end());
+ MIRBuilder.buildBr(OldMerge);
+
+ for (MachineBasicBlock *Predecessor : toUpdate) {
+ if (!MDT.dominates(&HeaderBlock, Predecessor))
+ continue;
+
+ OldMerge.replacePhiUsesWith(Predecessor, NewMerge);
+ Predecessor->removeSuccessor(&OldMerge);
+ Predecessor->addSuccessorWithoutProb(NewMerge);
+ for (auto &I : *Predecessor) {
+ for (auto &O : I.operands()) {
+ if (O.isMBB() && O.getMBB() == &OldMerge)
+ O.setMBB(NewMerge);
+ }
+ }
+ }
+
+ auto *MI = getMergeInstruction(HeaderBlock);
+ assert(MI);
+ MI->getOperand(0).setMBB(NewMerge);
+
+ return NewMerge;
+ }
+
+ // Modifies the CFG to make sure each merge block is the target of a single
+ // header.
+ bool splitMergeBlocks(MachineFunction &MF) {
+ MachineDominatorTree MDT(MF);
+
+ // Determine all the blocks we need to analyse.
+ auto HeaderBlocks = getHeaderBlocks(MF);
+ // Visit the CFG DFS-style to process header blocks.
+ std::vector<MachineBasicBlock *> ToProcess;
+ visit(MF, [&ToProcess, &HeaderBlocks](MachineBasicBlock *MBB) {
+ if (HeaderBlocks.count(MBB) != 0)
+ ToProcess.push_back(MBB);
+ });
+
+ // Maps each merge-block to its associated header block.
+ std::unordered_map<MachineBasicBlock *, MachineBasicBlock *> MergeToHeader;
+ bool modified = false;
+ for (auto *MBB : ToProcess) {
+ auto *MI = getMergeInstruction(*MBB);
+ assert(MI != nullptr);
+
+ auto *Merge = MI->getOperand(0).getMBB();
+ // If the merge block hasn't been seen yet, no conflict.
+ if (MergeToHeader.count(Merge) == 0) {
+ MergeToHeader.emplace(Merge, MBB);
+ continue;
+ }
+
+ // Otherwise, we need to split the merge block, and update the references.
+ modified = true;
+ MachineBasicBlock *ConflictingHeader = MergeToHeader[Merge];
+ MachineBasicBlock *NewMerge = splitMergeBlock(MDT, MF, *Merge, *MBB);
+ // Each selection/loop construct that is not already processed (hence
+ // deeper in the CFG) shall be updated to use the new merge. Conflicts are
+ // resolved layer by layer.
+ for (auto *Header : HeaderBlocks) {
+ if (Header == ConflictingHeader)
+ continue;
+ auto *Instr = getMergeInstruction(*Header);
+ if (Instr->getOperand(0).getMBB() == Merge)
+ Instr->getOperand(0).setMBB(NewMerge);
+ }
+ MergeToHeader.emplace(NewMerge, MBB);
+ }
+ return modified;
+ }
+
+ // Modifies the CFG to make sure the same block is not both a continue target,
+ // and a merge target.
+ bool splitMergeAndContinueBlocks(MachineFunction &MF) {
+ MachineDominatorTree MDT(MF);
+ std::vector<MachineBasicBlock *> toProcess;
+ visit(MF,
+ [&toProcess](MachineBasicBlock *MBB) { toProcess.push_back(MBB); });
+
+ auto ContinueBlocks = getContinueBlocks(MF);
+ bool modified = false;
+ for (auto *MBB : toProcess) {
+ MachineBasicBlock *Merge = nullptr;
+ MachineBasicBlock *Continue = nullptr;
+ MachineInstr *MI = nullptr;
+ if (!getMergeInstructionTargets(MBB, &MI, &Merge, &Continue))
+ continue;
+
+ if (ContinueBlocks.count(Merge) == 0)
+ continue;
+
+ // This blocks' merge is another block's continue.
+ modified = true;
+ MachineBasicBlock *NewMerge = splitMergeBlock(MDT, MF, *Merge, *MBB);
+ MI->getOperand(0).setMBB(NewMerge);
+ }
+ return modified;
+ }
+
+ // Sorts basic blocks by dominance to respect the SPIR-V spec.
+ bool sortBlocks(MachineFunction &MF) {
+ MachineDominatorTree MDT(MF);
+
+ std::unordered_map<MachineBasicBlock *, size_t> Order;
+ size_t Index = 0;
+ visit(MF,
+ [&Order, &Index](MachineBasicBlock *MBB) { Order[MBB] = Index++; });
+
+ auto Comparator = [&Order](MachineBasicBlock &LHS, MachineBasicBlock &RHS) {
+ return Order[&LHS] < Order[&RHS];
+ };
+
+ MF.sort(Comparator);
+ // FIXME: need to check if the order changed. Maybe if the comparator
+ // returns false once, it did?
+ return true;
+ }
+
+ // In some cases, divergence is allowed without any OpSelectionMerge
+ // instruction because paths only one path doesn't end-up to the parent's
+ // selection/loop construct merge. Example:
+ // A
+ // / \
+ // B<--C
+ // \ \
+ // \ D
+ // \ /
+ // E
+ // In this case, E is A's merge node.
+ // Previous steps marked C as a selection construct header because it has an
+ // out-degree of 2. But the thread divergence state cannot merge earlier: due
+ // to this triangle configuration, there is no earlier merge node than E. This
+ // means we are still in the same selection construct, hence don't require a
+ // new OpSelectionMerge.
+ bool removeSuperfluousSelectionHeaders(MachineFunction &MF) {
+ MachineDominatorTree MDT(MF);
+
+ bool modified = false;
+ for (MachineBasicBlock &MBB : MF) {
+ MachineInstr *MI = getMergeInstruction(MBB);
+ if (MI == nullptr)
+ continue;
+ // This doesn't apply to block targeted by a back-edge.
+ if (MI->getOpcode() == SPIRV::OpLoopMerge)
+ continue;
+
+ size_t dominated_count = 0;
+ for (auto *Successor : MBB.successors()) {
+ if (MDT.dominates(&MBB, Successor))
+ dominated_count += 1;
+ }
+
+ if (dominated_count > 1)
+ continue;
+
+ MBB.erase(MI);
+ modified = true;
+ }
+
+ return modified;
+ }
+
+ virtual bool runOnMachineFunction(MachineFunction &MF) override {
+ bool modified = false;
+
+ modified |= replaceEmptySwitchWithBranch(MF);
+ modified |= addMergeForLoops(MF);
+ modified |= addMergeForConditionalBranches(MF);
+ modified |= splitBlocksWithMultipleHeaders(MF);
+ modified |= splitMergeBlocks(MF);
+ modified |= splitMergeAndContinueBlocks(MF);
+ modified |= removeSuperfluousSelectionHeaders(MF);
+ modified |= sortBlocks(MF);
+
+ return modified;
+ }
+};
+
+char SPIRVStructurizer::ID = 0;
+INITIALIZE_PASS(SPIRVStructurizer, "structurizer", "SPIRV structurizer", false,
+ false)
+
+FunctionPass *llvm::createSPIRVStructurizerPass() {
+ return new SPIRVStructurizer();
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index 52fc6f33b4ef1..27897f1e78b4c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -157,6 +157,8 @@ TargetPassConfig *SPIRVTargetMachine::createPassConfig(PassManagerBase &PM) {
}
void SPIRVPassConfig::addIRPasses() {
+ TargetPassConfig::addIRPasses();
+
if (TM.getSubtargetImpl()->isVulkanEnv()) {
// Once legalized, we need to structurize the CFG to follow the spec.
// This is done through the following 8 steps.
@@ -175,10 +177,8 @@ void SPIRVPassConfig::addIRPasses() {
addPass(createSPIRVMergeRegionExitTargetsPass());
}
- TargetPassConfig::addIRPasses();
addPass(createSPIRVRegularizerPass());
addPass(createSPIRVPrepareFunctionsPass(TM));
- addPass(createSPIRVStripConvergenceIntrinsicsPass());
}
void SPIRVPassConfig::addISelPrepare() {
@@ -197,6 +197,9 @@ void SPIRVPassConfig::addPreLegalizeMachineIR() {
// Use the default legalizer.
bool SPIRVPassConfig::addLegalizeMachineIR() {
+ if (TM.getSubtargetImpl()->isVulkanEnv()) {
+ addPass(createSPIRVStructurizerPass());
+ }
addPass(new Legalizer());
addPass(createSPIRVPostLegalizerPass());
return false;
diff --git a/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll b/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
deleted file mode 100644
index 2ea5c767730e1..0000000000000
--- a/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
+++ /dev/null
@@ -1,80 +0,0 @@
-; RUN: llc -mtriple=spirv-unknown-unknown -O0 %s -o - | FileCheck %s
-
-; CHECK-DAG: OpDecorate %[[#SubgroupLocalInvocationId:]] BuiltIn SubgroupLocalInvocationId
-; CHECK-DAG: %[[#bool:]] = OpTypeBool
-; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint]] 0
-; CHECK-DAG: OpName %[[#main:]] "main"
-
-define void @main() #1 {
- %1 = icmp ne i32 0, 0
- %t1 = call token @llvm.experimental.convergence.entry()
- br i1 %1, label %l1, label %l2
-
-; CHECK: %[[#main]] = OpFunction
-; CHECK: %[[#cond:]] = OpINotEqual %[[#bool]] %[[#uint_0]] %[[#uint_0]]
-; CHECK: OpBranchConditional %[[#cond]] %[[#l1_pre:]] %[[#l2_pre:]]
-
-; CHECK-DAG: %[[#l2_pre]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l2_header:]]
-
-; CHECK-DAG: %[[#l1_pre]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l1_header:]]
-
-l1:
- %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
- br i1 %1, label %l1_body, label %l1_end
-; CHECK-DAG: %[[#l1_header]] = OpLabel
-; CHECK-NEXT: OpBranchConditional %[[#cond]] %[[#l1_body:]] %[[#l1_end:]]
-
-l1_body:
- br label %l1_continue
-; CHECK-DAG: %[[#l1_body]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l1_continue:]]
-
-l1_continue:
- br label %l1
-; CHECK-DAG: %[[#l1_continue]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l1_header]]
-
-l1_end:
- %call = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %tl1) ]
- br label %end
-; CHECK-DAG: %[[#l1_end]] = OpLabel
-; CHECK-DAG: %[[#]] = OpLoad %[[#]] %[[#SubgroupLocalInvocationId]]
-; CHECK-NEXT: OpBranch %[[#end:]]
-
-l2:
- %tl2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
- br i1 %1, label %l2_body, label %l2_end
-; CHECK-DAG: %[[#l2_header]] = OpLabel
-; CHECK-NEXT: OpBranchConditional %[[#cond]] %[[#l2_body:]] %[[#l2_end:]]
-
-l2_body:
- br label %l2_continue
-; CHECK-DAG: %[[#l2_body]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l2_continue:]]
-
-l2_continue:
- br label %l2
-; CHECK-DAG: %[[#l2_continue]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l2_header]]
-
-l2_end:
- br label %end
-; CHECK-DAG: %[[#l2_end]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#end:]]
-
-end:
- ret void
-; CHECK-DAG: %[[#end]] = OpLabel
-; CHECK-NEXT: OpReturn
-}
-
-attributes #1 = { "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" convergent }
-
-declare token @llvm.experimental.convergence.entry()
-declare token @llvm.experimental.convergence.control()
-declare token @llvm.experimental.convergence.loop()
-
-declare i32 @__hlsl_wave_get_lane_index() convergent
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
new file mode 100644
index 0000000000000..4123713d1fa9e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
@@ -0,0 +1,127 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan-compute"
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @fn() #4 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @fn1() #4 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 0
+}
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @fn2() #4 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+define internal spir_func void @main() #0 {
+; CHECK: %[[#cond:]] = OpINotEqual %[[#bool_ty:]] %[[#a:]] %[[#b:]]
+; CHECK: OpSelectionMerge %[[#cond_end:]] None
+; CHECK: OpBranchConditional %[[#cond]] %[[#cond_true:]] %[[#cond_false:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ %c = alloca i32, align 4
+ %val = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ %1 = load i32, ptr %a, align 4
+ %tobool = icmp ne i32 %1, 0
+ br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:
+ %2 = load i32, ptr %b, align 4
+ br label %cond.end
+
+; CHECK: %[[#cond_false]] = OpLabel
+; CHECK: OpBranch %[[#cond_end]]
+cond.false:
+ %3 = load i32, ptr %c, align 4
+ br label %cond.end
+
+; CHECK: %[[#cond_end]] = OpLabel
+; CHECK: %[[#tmp:]] = OpPhi %[[#int_ty:]] %[[#load_cond_true:]] %[[#cond_true]] %[[#load_cond_false:]] %[[#cond_false:]]
+; CHECK: %[[#cond:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0:]]
+; CHECK: OpSelectionMerge %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#cond]] %[[#if_then:]] %[[#if_end]]
+cond.end:
+ %cond = phi i32 [ %2, %cond.true ], [ %3, %cond.false ]
+ %tobool1 = icmp ne i32 %cond, 0
+ br i1 %tobool1, label %if.then, label %if.end
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpSelectionMerge %[[#cond_end8:]] None
+; CHECK: OpBranchConditional %[[#tmp:]] %[[#cond4_true:]] %[[#cond_false6:]]
+if.end:
+ %call2 = call spir_func noundef i32 @fn() #4 [ "convergencectrl"(token %0) ]
+ %tobool3 = icmp ne i32 %call2, 0
+ br i1 %tobool3, label %cond.true4, label %cond.false6
+
+; CHECK: %[[#cond_false6]] = OpLabel
+; CHECK: OpBranch %[[#cond_end8]]
+cond.false6:
+ %call7 = call spir_func noundef i32 @fn2() #4 [ "convergencectrl"(token %0) ]
+ br label %cond.end8
+
+; CHECK: %[[#cond_end8]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_end13:]] None
+; CHECK: OpBranchConditional %[[#tmp:]] %[[#if_then11:]] %[[#if_end13]]
+cond.end8:
+ %cond9 = phi i32 [ %call5, %cond.true4 ], [ %call7, %cond.false6 ]
+ %tobool10 = icmp ne i32 %cond9, 0
+ br i1 %tobool10, label %if.then11, label %if.end13
+
+; CHECK: %[[#if_end13]] = OpLabel
+; CHECK: OpReturn
+if.end13:
+ ret void
+
+; CHECK: %[[#if_then11]] = OpLabel
+; CHECK: OpBranch %[[#if_end13]]
+if.then11:
+ %4 = load i32, ptr %val, align 4
+ %inc12 = add nsw i32 %4, 1
+ store i32 %inc12, ptr %val, align 4
+ br label %if.end13
+
+; CHECK: %[[#cond4_true]] = OpLabel
+; CHECK: OpBranch %[[#cond_end8]]
+cond.true4:
+ %call5 = call spir_func noundef i32 @fn1() #4 [ "convergencectrl"(token %0) ]
+ br label %cond.end8
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#if_end]]
+if.then:
+ %5 = load i32, ptr %val, align 4
+ %inc = add nsw i32 %5, 1
+ store i32 %inc, ptr %val, align 4
+ br label %if.end
+
+; CHECK: %[[#cond_true]] = OpLabel
+; CHECK: OpBranch %[[#cond_end]]
+}
+
+declare token @llvm.experimental.convergence.entry() #2
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent norecurse "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #3 = { convergent }
+attributes #4 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-break.ll
new file mode 100644
index 0000000000000..7fd2bf32918ae
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-break.ll
@@ -0,0 +1,88 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define internal spir_func void @main() #1 {
+; CHECK: %[[#entry:]] = OpLabel
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+; CHECK: %[[#do_body:]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_merge:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then:]] %[[#if_end:]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %2, 1
+ store i32 %inc, ptr %i, align 4
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp sgt i32 %3, 5
+ br i1 %cmp, label %if.then, label %if.end
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpBranch %[[#do_end:]]
+if.end:
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %do.end
+
+; CHECK: %[[#do_end]] = OpLabel
+; CHECK: OpBranch %[[#do_body2:]]
+do.end:
+ br label %do.body2
+
+; CHECK: %[[#do_body2]] = OpLabel
+; CHECK: OpBranch %[[#do_body4:]]
+do.body2:
+ %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %7 = load i32, ptr %i, align 4
+ %inc3 = add nsw i32 %7, 1
+ store i32 %inc3, ptr %i, align 4
+ br label %do.body4
+
+; CHECK: %[[#do_body4]] = OpLabel
+; CHECK: OpBranch %[[#do_end8:]]
+do.body4:
+ %8 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
+ %9 = load i32, ptr %val, align 4
+ %inc5 = add nsw i32 %9, 1
+ store i32 %inc5, ptr %val, align 4
+ br label %do.end8
+
+; CHECK: %[[#do_end8]] = OpLabel
+; CHECK: OpBranch %[[#do_end11:]]
+do.end8:
+ %11 = load i32, ptr %i, align 4
+ %dec = add nsw i32 %11, -1
+ store i32 %dec, ptr %i, align 4
+ br label %do.end11
+
+; CHECK: %[[#do_end11]] = OpLabel
+; CHECK: OpReturn
+do.end11:
+ ret void
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#do_end]]
+if.then:
+ br label %do.end
+}
+
+
+declare token @llvm.experimental.convergence.entry() #0
+declare token @llvm.experimental.convergence.loop() #0
+
+attributes #0 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #1 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
new file mode 100644
index 0000000000000..4681da4e032cb
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
@@ -0,0 +1,124 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define spir_func noundef i32 @_Z3foov() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+define internal spir_func void @main() #2 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK: OpBranch %[[#do_body:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+; CHECK: %[[#do_body]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end:]] %[[#do_continue:]] None
+; CHECK: OpBranch %[[#condition_header:]]
+
+; CHECK: %[[#condition_header]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then:]] %[[#if_end:]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %2, 1
+ store i32 %inc, ptr %i, align 4
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp sgt i32 %3, 5
+ br i1 %cmp, label %if.then, label %if.end
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpBranch %[[#do_continue]]
+if.end:
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %do.cond
+
+; CHECK: %[[#do_continue]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body]] %[[#do_end]]
+do.cond:
+ %5 = load i32, ptr %i, align 4
+ %cmp1 = icmp slt i32 %5, 10
+ br i1 %cmp1, label %do.body, label %do.end
+
+; CHECK: %[[#do_end]] = OpLabel
+; CHECK: OpBranch %[[#do_body2:]]
+do.end:
+ br label %do.body2
+
+; CHECK: %[[#do_body2]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end11:]] %[[#do_cond9:]] None
+; CHECK: OpBranch %[[#do_body4:]]
+do.body2:
+ %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %7 = load i32, ptr %i, align 4
+ %inc3 = add nsw i32 %7, 1
+ store i32 %inc3, ptr %i, align 4
+ br label %do.body4
+
+; CHECK: %[[#do_body4]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end8:]] %[[#do_cond6:]] None
+; CHECK: OpBranch %[[#do_cond6]]
+do.body4:
+ %8 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
+ %9 = load i32, ptr %val, align 4
+ %inc5 = add nsw i32 %9, 1
+ store i32 %inc5, ptr %val, align 4
+ br label %do.cond6
+
+; CHECK: %[[#do_cond6]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body4]] %[[#do_end8]]
+do.cond6:
+ %10 = load i32, ptr %i, align 4
+ %cmp7 = icmp slt i32 %10, 10
+ br i1 %cmp7, label %do.body4, label %do.end8
+
+; CHECK: %[[#do_end8]] = OpLabel
+; CHECK: OpBranch %[[#do_cond9]]
+do.end8:
+ %11 = load i32, ptr %i, align 4
+ %dec = add nsw i32 %11, -1
+ store i32 %dec, ptr %i, align 4
+ br label %do.cond9
+
+; CHECK: %[[#do_cond9]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body2]] %[[#do_end11]]
+do.cond9:
+ %12 = load i32, ptr %val, align 4
+ %cmp10 = icmp slt i32 %12, 10
+ br i1 %cmp10, label %do.body2, label %do.end11
+
+; CHECK: %[[#do_end11]] = OpLabel
+; CHECK: OpReturn
+do.end11:
+ ret void
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#if_end:]]
+if.then:
+ br label %do.cond
+}
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
new file mode 100644
index 0000000000000..d3265f39a5c9d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
@@ -0,0 +1,102 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define internal spir_func void @main() #0 {
+; CHECK: %[[#entry:]] = OpLabel
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ %j = alloca i32, align 4
+ %k = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ store i32 0, ptr %j, align 4
+ store i32 0, ptr %k, align 4
+ br label %do.body
+
+; CHECK: %[[#do_1_header:]] = OpLabel
+; CHECK: OpLoopMerge %[[#end:]] %[[#do_1_latch:]] None
+; CHECK: OpBranch %[[#do_2_header:]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %val, align 4
+ %3 = load i32, ptr %i, align 4
+ %add = add nsw i32 %2, %3
+ store i32 %add, ptr %val, align 4
+ br label %do.body1
+
+; CHECK: %[[#do_2_header]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_2_end:]] %[[#do_2_latch:]] None
+; CHECK: OpBranch %[[#do_2_body:]]
+do.body1:
+ %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %1) ]
+ br label %do.body2
+
+; CHECK: %[[#do_2_body]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_3_end:]] %[[#do_3_header:]] None
+; CHECK: OpBranch %[[#do_3_header]]
+do.body2:
+ %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %4) ]
+ %6 = load i32, ptr %k, align 4
+ %inc = add nsw i32 %6, 1
+ store i32 %inc, ptr %k, align 4
+ br label %do.cond
+
+; CHECK: %[[#do_3_header]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_2_body]] %[[#do_3_end]]
+do.cond:
+ %7 = load i32, ptr %k, align 4
+ %cmp = icmp slt i32 %7, 30
+ br i1 %cmp, label %do.body2, label %do.end
+
+; CHECK: %[[#do_3_end]] = OpLabel
+; CHECK: OpBranch %[[#do_2_latch]]
+do.end:
+ %8 = load i32, ptr %j, align 4
+ %inc3 = add nsw i32 %8, 1
+ store i32 %inc3, ptr %j, align 4
+ br label %do.cond4
+
+; CHECK: %[[#do_2_latch]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_2_header]] %[[#do_2_end]]
+do.cond4:
+ %9 = load i32, ptr %j, align 4
+ %cmp5 = icmp slt i32 %9, 20
+ br i1 %cmp5, label %do.body1, label %do.end6
+
+; CHECK: %[[#do_2_end]] = OpLabel
+; CHECK: OpBranch %[[#do_1_latch]]
+do.end6:
+ %10 = load i32, ptr %i, align 4
+ %inc7 = add nsw i32 %10, 1
+ store i32 %inc7, ptr %i, align 4
+ br label %do.cond8
+
+; CHECK: %[[#do_1_latch]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_1_header]] %[[#end]]
+do.cond8:
+ %11 = load i32, ptr %i, align 4
+ %cmp9 = icmp slt i32 %11, 10
+ br i1 %cmp9, label %do.body, label %do.end10
+
+; CHECK: %[[#end]] = OpLabel
+; CHECK: OpReturn
+do.end10:
+ ret void
+}
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
new file mode 100644
index 0000000000000..cfbbee536d53a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
@@ -0,0 +1,98 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+
+define spir_func noundef i32 @_Z3foov() #0 {
+; CHECK: %[[#foo:]] = OpLabel
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+
+define internal spir_func void @main() #2 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK: OpBranch %[[#do_body:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+; CHECK: %[[#do_body]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end:]] %[[#do_cond:]] None
+; CHECK: OpBranch %[[#do_cond]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ store i32 %2, ptr %val, align 4
+ br label %do.cond
+
+; CHECK: %[[#do_cond]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body]] %[[#do_end]]
+do.cond:
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %3, 10
+ br i1 %cmp, label %do.body, label %do.end
+
+; CHECK: %[[#do_end]] = OpLabel
+; CHECK: OpBranch %[[#do_body1:]]
+do.end:
+ br label %do.body1
+
+; CHECK: %[[#do_body1]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end3:]] %[[#do_cond2:]] None
+; CHECK: OpBranch %[[#do_cond2]]
+do.body1:
+ %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ store i32 0, ptr %val, align 4
+ br label %do.cond2
+
+; CHECK: %[[#do_cond2]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body1]] %[[#do_end3]]
+do.cond2:
+ br i1 true, label %do.body1, label %do.end3
+
+; CHECK: %[[#do_end3]] = OpLabel
+; CHECK: OpBranch %[[#do_body4:]]
+do.end3:
+ br label %do.body4
+
+; CHECK: %[[#do_body4]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end7:]] %[[#do_cond5:]] None
+; CHECK: OpBranch %[[#do_cond5]]
+do.body4:
+ %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ br label %do.cond5
+
+; CHECK: %[[#do_cond5]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body4]] %[[#do_end7]]
+do.cond5:
+ %6 = load i32, ptr %val, align 4
+ %cmp6 = icmp slt i32 %6, 20
+ br i1 %cmp6, label %do.body4, label %do.end7
+
+; CHECK: %[[#do_end7]] = OpLabel
+; CHECK: OpReturn
+do.end7:
+ ret void
+}
+
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
new file mode 100644
index 0000000000000..16f04a5015617
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
@@ -0,0 +1,158 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - --asm-verbose=0 | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; CHECK-DAG: OpName %[[#fn:]] "fn"
+; CHECK-DAG: OpName %[[#main:]] "main"
+; CHECK-DAG: OpName %[[#a:]] "a"
+; CHECK-DAG: OpName %[[#b:]] "b"
+
+; CHECK: %[[#fn]] = OpFunction %[[#param:]] DontInline %[[#ftype:]]
+define spir_func noundef i32 @fn() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; CHECK: %[[#main]] = OpFunction %[[#param:]] DontInline %[[#ftype:]]
+
+define internal spir_func void @main() #3 {
+
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK-DAG: %[[#a]] = OpVariable %[[#type:]] Function
+; CHECK-DAG: %[[#b]] = OpVariable %[[#type:]] Function
+; CHECK: %[[#tmp:]] = OpLoad %[[#type:]] %[[#a]] Aligned 4
+; CHECK: OpSelectionMerge %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then:]] %[[#lor_lhs_false:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ %val = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ %1 = load i32, ptr %a, align 4
+ %tobool = icmp ne i32 %1, 0
+ br i1 %tobool, label %if.then, label %lor.lhs.false
+
+; CHECK: %[[#lor_lhs_false]] = OpLabel
+; CHECK-NOT: %[[#tmp:]] = OpLoad %[[#type:]] %[[#a]] Aligned 4
+; CHECK: %[[#tmp:]] = OpLoad %[[#type:]] %[[#b]] Aligned 4
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then]] %[[#new_end:]]
+
+; CHECK: %[[#new_end]] = OpLabel
+; CHECK: OpBranch %[[#if_end]]
+lor.lhs.false:
+ %2 = load i32, ptr %b, align 4
+ %tobool1 = icmp ne i32 %2, 0
+ br i1 %tobool1, label %if.then, label %if.end
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: %[[#tmp:]] = OpFunctionCall %[[#type:]] %[[#fn]]
+; CHECK: OpSelectionMerge %[[#if_end9:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then7:]] %[[#lor_lhs_false4:]]
+if.end:
+ %call2 = call spir_func noundef i32 @fn() convergent [ "convergencectrl"(token %0) ]
+ %tobool3 = icmp ne i32 %call2, 0
+ br i1 %tobool3, label %if.then7, label %lor.lhs.false4
+
+; CHECK: %[[#lor_lhs_false4]] = OpLabel
+; CHECK: %[[#tmp:]] = OpFunctionCall %[[#type:]] %[[#fn]]
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then7]] %[[#new_end9:]]
+
+; CHECK: %[[#new_end9]] = OpLabel
+; CHECK: OpBranch %[[#if_end9]]
+lor.lhs.false4:
+ %call5 = call spir_func noundef i32 @fn() convergent [ "convergencectrl"(token %0) ]
+ %tobool6 = icmp ne i32 %call5, 0
+ br i1 %tobool6, label %if.then7, label %if.end9
+
+; CHECK: %[[#if_end9]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_end16:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then14:]] %[[#lor_lhs_false11:]]
+if.end9:
+ %3 = load i32, ptr %a, align 4
+ %tobool10 = icmp ne i32 %3, 0
+ br i1 %tobool10, label %if.then14, label %lor.lhs.false11
+
+; CHECK: %[[#lor_lhs_false11]] = OpLabel
+; CHECK: %[[#tmp:]] = OpFunctionCall %[[#type:]] %[[#fn]]
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then14]] %[[#new_end16:]]
+
+; CHECK: %[[#new_end16]] = OpLabel
+; CHECK: OpBranch %[[#if_end16]]
+lor.lhs.false11:
+ %call12 = call spir_func noundef i32 @fn() convergent [ "convergencectrl"(token %0) ]
+ %tobool13 = icmp ne i32 %call12, 0
+ br i1 %tobool13, label %if.then14, label %if.end16
+
+; CHECK: %[[#if_end16]] = OpLabel
+; CHECK: %[[#tmp:]] = OpFunctionCall %[[#type:]] %[[#fn]]
+; CHECK: OpSelectionMerge %[[#if_end23:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then21:]] %[[#lor_lhs_false19:]]
+if.end16:
+ %call17 = call spir_func noundef i32 @fn() convergent [ "convergencectrl"(token %0) ]
+ %tobool18 = icmp ne i32 %call17, 0
+ br i1 %tobool18, label %if.then21, label %lor.lhs.false19
+
+; CHECK: %[[#lor_lhs_false19]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then21]] %[[#new_end32:]]
+
+; CHECK: %[[#new_end32]] = OpLabel
+; CHECK: OpBranch %[[#if_end23]]
+lor.lhs.false19:
+ %4 = load i32, ptr %b, align 4
+ %tobool20 = icmp ne i32 %4, 0
+ br i1 %tobool20, label %if.then21, label %if.end23
+
+; CHECK: %[[#if_end23]] = OpLabel
+; CHECK: OpReturn
+if.end23:
+ ret void
+
+; CHECK: %[[#if_then21]] = OpLabel
+; CHECK: OpBranch %[[#if_end23]]
+if.then21:
+ %5 = load i32, ptr %val, align 4
+ %inc22 = add nsw i32 %5, 1
+ store i32 %inc22, ptr %val, align 4
+ br label %if.end23
+
+; CHECK: %[[#if_then14]] = OpLabel
+; CHECK: OpBranch %[[#if_end16]]
+if.then14:
+ %6 = load i32, ptr %val, align 4
+ %inc15 = add nsw i32 %6, 1
+ store i32 %inc15, ptr %val, align 4
+ br label %if.end16
+
+; CHECK: %[[#if_then7]] = OpLabel
+; CHECK-NOT: %[[#tmp:]] = OpFunctionCall %[[#type:]] %[[#fn]]
+; CHECK: OpBranch %[[#if_end9]]
+if.then7:
+ %7 = load i32, ptr %val, align 4
+ %inc8 = add nsw i32 %7, 1
+ store i32 %inc8, ptr %val, align 4
+ br label %if.end9
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#if_end]]
+if.then:
+ %8 = load i32, ptr %val, align 4
+ %inc = add nsw i32 %8, 1
+ store i32 %inc, ptr %val, align 4
+ br label %if.end
+}
+
+declare token @llvm.experimental.convergence.entry() #2
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #3 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
index e7b1b441405f6..481665fc0ce21 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
@@ -1,4 +1,5 @@
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
target triple = "spirv-unknown-vulkan-compute"
@@ -26,13 +27,32 @@ entry:
; CHECK: %[[#while_cond]] = OpLabel
; CHECK: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
; CHECK: %[[#cmp:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_10]]
-; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end:]]
+; CHECK: OpLoopMerge %[[#new_end:]] %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end]]
+
+; CHECK: %[[#new_end]] = OpLabel
+; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_0]] %[[#while_cond]] %[[#int_1]] %[[#while_body]]
+; CHECK: OpSelectionMerge %[[#while_end:]] None
+; CHECK: OpSwitch %[[#route]] %[[#if_then:]] 1 %[[#while_end_loopexit:]]
while.cond:
%1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
%2 = load i32, ptr %idx, align 4
%cmp = icmp ne i32 %2, 10
br i1 %cmp, label %while.body, label %while.end
+; CHECK: %[[#while_end_loopexit]] = OpLabel
+; CHECK: OpBranch %[[#while_end]]
+
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK: OpReturn
+while.end:
+ ret void
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#while_end]]
+if.then:
+ br label %while.end
+
; CHECK: %[[#while_body]] = OpLabel
; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
@@ -46,27 +66,10 @@ while.body:
%cmp1 = icmp eq i32 %4, 0
br i1 %cmp1, label %if.then, label %if.end
-; CHECK: %[[#if_then:]] = OpLabel
-; CHECK: OpBranch %[[#while_end:]]
-if.then:
- br label %while.end
-
; CHECK: %[[#if_end]] = OpLabel
; CHECK: OpBranch %[[#while_cond]]
if.end:
br label %while.cond
-
-; CHECK: %[[#while_end_loopexit:]] = OpLabel
-; CHECK: OpBranch %[[#while_end]]
-
-; CHECK: %[[#while_end]] = OpLabel
-; CHECK: OpReturn
-while.end:
- ret void
-
-; CHECK: %[[#new_end]] = OpLabel
-; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_1]] %[[#while_cond]] %[[#int_0]] %[[#while_body]]
-; CHECK: OpSwitch %[[#route]] %[[#if_then]] 1 %[[#while_end_loopexit]]
}
declare token @llvm.experimental.convergence.entry() #2
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
index 593e3631c02b9..6de6b008afe5e 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
@@ -1,4 +1,5 @@
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
target triple = "spirv-unknown-vulkan-compute"
@@ -33,12 +34,25 @@ while.cond:
%cmp = icmp ne i32 %2, 10
br i1 %cmp, label %while.body, label %while.end
-; CHECK: %[[#while_body]] = OpLabel
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
-; CHECK-NEXT: %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-; CHECK: OpBranchConditional %[[#cmp1]] %[[#if_then:]] %[[#if_end:]]
+; CHECK: %[[#new_end]] = OpLabel
+; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_0]] %[[#while_cond]] %[[#int_1]] %[[#tail:]]
+; CHECK: OpSelectionMerge %[[#while_end:]] None
+; CHECK: OpSwitch %[[#route]] %[[#while_end_loopexit:]] 1 %[[#while_end]]
+
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK: OpReturn
+while.end:
+ ret void
+
+; CHECK: %[[#while_end_loopexit]] = OpLabel
+; CHECK: OpBranch %[[#while_end]]
+
+; CHECK: %[[#while_body]] = OpLabel
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
+; CHECK-NEXT: %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+; CHECK: OpBranchConditional %[[#cmp1]] %[[#if_then:]] %[[#if_end:]]
while.body:
%3 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %3, ptr %idx, align 4
@@ -46,36 +60,24 @@ while.body:
%cmp1 = icmp eq i32 %4, 0
br i1 %cmp1, label %if.then, label %if.end
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpBranch %[[#while_cond]]
+if.end:
+ br label %while.cond
+
; CHECK: %[[#if_then:]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#tail:]]
+; CHECK-NEXT: OpBranch %[[#tail]]
if.then:
br label %tail
-; CHECK: %[[#tail:]] = OpLabel
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK: OpBranch %[[#new_end:]]
+; CHECK: %[[#tail]] = OpLabel
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK: OpBranch %[[#new_end:]]
tail:
%5 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %5, ptr %idx, align 4
br label %while.end
-
-; CHECK: %[[#if_end]] = OpLabel
-; CHECK: OpBranch %[[#while_cond]]
-if.end:
- br label %while.cond
-
-; CHECK: %[[#while_end_loopexit:]] = OpLabel
-; CHECK: OpBranch %[[#while_end:]]
-
-; CHECK: %[[#while_end]] = OpLabel
-; CHECK: OpReturn
-while.end:
- ret void
-
-; CHECK: %[[#new_end]] = OpLabel
-; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_0]] %[[#while_cond]] %[[#int_1]] %[[#tail]]
-; CHECK: OpSwitch %[[#route]] %[[#while_end_loopexit]] 1 %[[#while_end]]
}
declare token @llvm.experimental.convergence.entry() #2
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
index 9806dd7955468..b560c45ec04c8 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
@@ -1,4 +1,5 @@
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
target triple = "spirv-unknown-vulkan-compute"
@@ -34,12 +35,34 @@ while.cond:
%cmp = icmp ne i32 %2, 10
br i1 %cmp, label %while.body, label %while.end
+; CHECK: %[[#new_end]] = OpLabel
+; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_1]] %[[#while_cond]] %[[#int_0]] %[[#while_body]] %[[#int_2]] %[[#if_end:]]
+; CHECK: OpSwitch %[[#route]] %[[#if_then:]] 1 %[[#while_end_loopexit:]] 2 %[[#if_then2:]]
+
+; CHECK: %[[#if_then2]] = OpLabel
+; CHECK: OpBranch %[[#while_end:]]
+if.then:
+ br label %while.end
+
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK: OpReturn
+while.end:
+ ret void
+
+; CHECK: %[[#while_end_loopexit]] = OpLabel
+; CHECK: OpBranch %[[#while_end]]
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#while_end]]
+if.then2:
+ br label %while.end
+
; CHECK: %[[#while_body]] = OpLabel
; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
; CHECK-NEXT: %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-; CHECK: OpBranchConditional %[[#cmp1]] %[[#new_end]] %[[#if_end:]]
+; CHECK: OpBranchConditional %[[#cmp1]] %[[#new_end]] %[[#if_end]]
while.body:
%3 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %3, ptr %idx, align 4
@@ -47,17 +70,12 @@ while.body:
%cmp1 = icmp eq i32 %4, 0
br i1 %cmp1, label %if.then, label %if.end
-; CHECK: %[[#if_then:]] = OpLabel
-; CHECK: OpBranch %[[#while_end:]]
-if.then:
- br label %while.end
-
-; CHECK: %[[#if_end]] = OpLabel
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
-; CHECK-NEXT: %[[#cmp2:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-; CHECK: OpBranchConditional %[[#cmp2]] %[[#new_end]] %[[#if_end2:]]
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
+; CHECK-NEXT: %[[#cmp2:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+; CHECK: OpBranchConditional %[[#cmp2]] %[[#new_end]] %[[#if_end2:]]
if.end:
%5 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %5, ptr %idx, align 4
@@ -65,37 +83,18 @@ if.end:
%cmp2 = icmp eq i32 %6, 0
br i1 %cmp2, label %if.then2, label %if.end2
-; CHECK: %[[#if_then2:]] = OpLabel
-; CHECK: OpBranch %[[#while_end:]]
-if.then2:
- br label %while.end
-
-; CHECK: %[[#if_end2]] = OpLabel
-; CHECK: OpBranch %[[#while_cond:]]
+; CHECK: %[[#if_end2]] = OpLabel
+; CHECK: OpBranch %[[#while_cond:]]
if.end2:
br label %while.cond
-
-; CHECK: %[[#while_end_loopexit:]] = OpLabel
-; CHECK: OpBranch %[[#while_end]]
-
-; CHECK: %[[#while_end]] = OpLabel
-; CHECK: OpReturn
-while.end:
- ret void
-
-; CHECK: %[[#new_end]] = OpLabel
-; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_2]] %[[#while_cond]] %[[#int_0]] %[[#while_body]] %[[#int_1]] %[[#if_end]]
-; CHECK: OpSwitch %[[#route]] %[[#if_then]] 1 %[[#if_then2]] 2 %[[#while_end_loopexit]]
}
-declare token @llvm.experimental.convergence.entry() #2
-declare token @llvm.experimental.convergence.loop() #2
-declare i32 @__hlsl_wave_get_lane_index() #3
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+declare i32 @__hlsl_wave_get_lane_index() convergent
attributes #0 = { convergent noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #1 = { convergent norecurse "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
-attributes #3 = { convergent }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
!llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-white-identity.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-white-identity.ll
index a8bf4fb0db989..de818e619b6aa 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-white-identity.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-white-identity.ll
@@ -1,4 +1,5 @@
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
target triple = "spirv-unknown-vulkan-compute"
@@ -21,6 +22,11 @@ while.cond:
%cmp = icmp ne i32 %2, 0
br i1 %cmp, label %while.body, label %while.end
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK-NEXT: OpReturn
+while.end:
+ ret void
+
; CHECK: %[[#while_body]] = OpLabel
; CHECK: OpBranch %[[#while_cond]]
while.body:
@@ -28,10 +34,6 @@ while.body:
store i32 %3, ptr %idx, align 4
br label %while.cond
- ; CHECK: %[[#while_end]] = OpLabel
-; CHECK-NEXT: OpReturn
-while.end:
- ret void
}
declare token @llvm.experimental.convergence.entry() #2
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
new file mode 100644
index 0000000000000..9da4522d85d70
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
@@ -0,0 +1,130 @@
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define internal spir_func void @main() #0 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK: OpBranch %[[#while_cond:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ %cond = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 1, ptr %cond, align 4
+ br label %while.cond
+
+; CHECK: %[[#while_cond]] = OpLabel
+; CHECK: OpSelectionMerge %[[#while_end:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#while_body:]] %[[#while_end]]
+while.cond:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %cond, align 4
+ %tobool = icmp ne i32 %2, 0
+ br i1 %tobool, label %while.body, label %while.end
+
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK: OpReturn
+while.end:
+ ret void
+
+; CHECK: %[[#while_body]] = OpLabel
+; CHECK: OpSelectionMerge %[[#switch_end:]] None
+; CHECK: OpSwitch %[[#cond:]] %[[#case_default:]] 1 %[[#case_1:]] 2 %[[#case_2:]] 5 %[[#case_5:]]
+while.body:
+ %3 = load i32, ptr %b, align 4
+ switch i32 %3, label %sw.default [
+ i32 1, label %sw.bb
+ i32 2, label %sw.bb1
+ i32 5, label %sw.bb2
+ ]
+
+; CHECK: %[[#case_5]] = OpLabel
+; CHECK: OpBranch %[[#switch_end]]
+sw.bb2:
+ store i32 5, ptr %a, align 4
+ br label %while.end
+
+; CHECK: %[[#switch_end]] = OpLabel
+; CHECK: OpBranch %[[#while_end]]
+
+; CHECK: %[[#case_2]] = OpLabel
+; CHECK: OpBranch %[[#switch_end]]
+sw.bb1:
+ store i32 3, ptr %a, align 4
+ br label %while.end
+
+; CHECK: %[[#case_1]] = OpLabel
+; CHECK: OpBranch %[[#switch_end]]
+sw.bb:
+ store i32 1, ptr %a, align 4
+ br label %while.end
+
+
+
+; CHECK: %[[#case_default]] = OpLabel
+; CHECK: OpBranch %[[#for_cond:]]
+sw.default:
+ store i32 0, ptr %i, align 4
+ br label %for.cond
+
+; CHECK: %[[#for_cond]] = OpLabel
+; CHECK: OpSelectionMerge %[[#for_end:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#for_body:]] %[[#for_exit:]]
+for.cond:
+ %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %1) ]
+ %5 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %5, 10
+ br i1 %cmp, label %for.body, label %for.end
+
+; CHECK: %[[#for_exit]] = OpLabel
+; CHECK: OpBranch %[[#for_end]]
+
+; CHECK: %[[#for_end]] = OpLabel
+; CHECK: OpBranch %[[#switch_end]]
+for.end:
+ br label %while.end
+
+; CHECK: %[[#for_body]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_merge:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then:]] %[[#if_else:]]
+for.body:
+ %6 = load i32, ptr %cond, align 4
+ %tobool3 = icmp ne i32 %6, 0
+ br i1 %tobool3, label %if.then, label %if.else
+
+; CHECK: %[[#if_else]] = OpLabel
+; CHECK: OpBranch %[[#if_merge]]
+if.else:
+ br label %while.end
+
+; CHECK: %[[#if_merge]] = OpLabel
+; CHECK: OpBranch %[[#for_end]]
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#if_merge]]
+if.then:
+ br label %while.end
+
+; CHECK-NOT: %[[#for_inc:]] = OpLabel
+; This block is not emitted since it's unreachable.
+for.inc:
+ %7 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %7, 1
+ store i32 %inc, ptr %i, align 4
+ br label %for.cond
+}
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
More information about the llvm-commits
mailing list