[clang] [llvm] [SPIR-V] Add SPIR-V structurizer (PR #107408)
Nathan Gauër via cfe-commits
cfe-commits at lists.llvm.org
Mon Sep 9 08:40:50 PDT 2024
https://github.com/Keenuts updated https://github.com/llvm/llvm-project/pull/107408
>From 0364950b1870123355372cc6a7d4dc0789199b1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Wed, 4 Sep 2024 13:03:39 +0200
Subject: [PATCH 1/7] [SPIR-V] Add SPIR-V structurizer
This commit adds an initial SPIR-V structurizer.
It leverages the previously merged passes, and the convergence region
analysis to determine the correct merge and continue blocks for SPIR-V.
The first part does a branch cleanup (simplifying switches, and
legalizing them), then merge instructions are added to cycles,
convergent and later divergent blocks.
Then comes the important part: splitting critical edges, and making sure
the divergent construct boundaries don't cross.
- we split blocks with multiple headers into 2 blocks.
- we split blocks that are a merge blocks for 2 or more constructs:
SPIR-V spec disallow a merge block to be shared by 2 loop/switch/condition
construct.
- we split merge & continue blocks: SPIR-V spec disallow a basic block to be
both a continue block, and a merge block.
- we remove superfluous headers: when a header doesn't bring more info than
the parent on the divergence state, it must be removed.
This PR leverages the merged SPIR-V simulator for testing, as long as
spirv-val. For now, most DXC structurization tests are passing. The
unsupported ones are either caused by unsupported features like switches
on boolean types, or switches in region exits, because the MergeExit
pass doesn't support those yet (there is a FIXME).
This PR is quite large, and the addition not trivial, so I tried to keep
it simple. E.G: as soon as the CFG changes, I recompute the dominator trees
and other structures instead of updating them.
---
.../CodeGenHLSL/convergence/cf.for.plain.hlsl | 44 +
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 2 +
.../SPIRVConvergenceRegionAnalysis.cpp | 5 +-
.../Analysis/SPIRVConvergenceRegionAnalysis.h | 3 +
llvm/lib/Target/SPIRV/CMakeLists.txt | 1 +
llvm/lib/Target/SPIRV/SPIRV.h | 1 +
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 13 +
.../SPIRV/SPIRVMergeRegionExitTargets.cpp | 20 +-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 172 +-
llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp | 1410 +++++++++++++++++
llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 10 +-
llvm/test/CMakeLists.txt | 1 +
.../CodeGen/SPIRV/scfg-add-pre-headers.ll | 87 -
.../CodeGen/SPIRV/structurizer/cf.cond-op.ll | 168 ++
.../CodeGen/SPIRV/structurizer/cf.do.break.ll | 169 ++
.../SPIRV/structurizer/cf.do.continue.ll | 167 ++
.../SPIRV/structurizer/cf.do.nested.ll | 138 ++
.../SPIRV/structurizer/cf.for.break.ll | 178 +++
.../SPIRV/structurizer/cf.for.continue.hlsl | 47 +
.../SPIRV/structurizer/cf.for.nested.hlsl | 25 +
.../SPIRV/structurizer/cf.for.plain.ll | 105 ++
.../cf.for.short-circuited-cond.hlsl | 42 +
.../SPIRV/structurizer/cf.if.const-cond.hlsl | 33 +
.../CodeGen/SPIRV/structurizer/cf.if.for.hlsl | 46 +
.../SPIRV/structurizer/cf.if.nested.hlsl | 29 +
.../SPIRV/structurizer/cf.if.plain.hlsl | 39 +
.../SPIRV/structurizer/cf.logical-and.hlsl | 27 +
.../SPIRV/structurizer/cf.logical-or.hlsl | 24 +
.../SPIRV/structurizer/cf.return.early.hlsl | 58 +
.../structurizer/cf.return.early.simple.hlsl | 20 +
.../SPIRV/structurizer/cf.return.void.hlsl | 14 +
.../SPIRV/structurizer/cf.switch.ifstmt.hlsl | 122 ++
.../structurizer/cf.switch.ifstmt.simple.hlsl | 25 +
.../cf.switch.ifstmt.simple2.hlsl | 45 +
.../structurizer/cf.switch.opswitch.hlsl | 360 +++++
.../cf.switch.opswitch.literal.hlsl | 36 +
.../cf.switch.opswitch.simple.hlsl | 36 +
.../SPIRV/structurizer/cf.while.break.hlsl | 87 +
.../SPIRV/structurizer/cf.while.continue.hlsl | 89 ++
.../SPIRV/structurizer/cf.while.nested.hlsl | 79 +
.../SPIRV/structurizer/cf.while.plain.hlsl | 101 ++
.../cf.while.short-circuited-cond.hlsl | 20 +
.../SPIRV/structurizer/condition-linear.ll | 128 ++
.../CodeGen/SPIRV/structurizer/do-break.ll | 89 ++
.../CodeGen/SPIRV/structurizer/do-continue.ll | 124 ++
.../CodeGen/SPIRV/structurizer/do-nested.ll | 102 ++
.../CodeGen/SPIRV/structurizer/do-plain.ll | 98 ++
.../CodeGen/SPIRV/structurizer/logical-or.ll | 93 ++
.../SPIRV/structurizer/merge-exit-break.ll | 36 +-
.../merge-exit-convergence-in-break.ll | 56 +-
.../structurizer/merge-exit-multiple-break.ll | 64 +-
.../merge-exit-simple-while-identity.ll | 7 +-
.../SPIRV/structurizer/return-early.ll | 131 ++
53 files changed, 4774 insertions(+), 252 deletions(-)
create mode 100644 clang/test/CodeGenHLSL/convergence/cf.for.plain.hlsl
create mode 100644 llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
delete mode 100644 llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.simple.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.return.void.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.simple.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.hlsl
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-break.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
create mode 100644 llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
diff --git a/clang/test/CodeGenHLSL/convergence/cf.for.plain.hlsl b/clang/test/CodeGenHLSL/convergence/cf.for.plain.hlsl
new file mode 100644
index 00000000000000..2f08854f84d955
--- /dev/null
+++ b/clang/test/CodeGenHLSL/convergence/cf.for.plain.hlsl
@@ -0,0 +1,44 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN: spirv-pc-vulkan-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+int process() {
+// CHECK: entry:
+// CHECK: %[[#entry_token:]] = call token @llvm.experimental.convergence.entry()
+ int val = 0;
+
+// CHECK: for.cond:
+// CHECK-NEXT: %[[#]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %[[#entry_token]]) ]
+// CHECK: br i1 {{.*}}, label %for.body, label %for.end
+ for (int i = 0; i < 10; ++i) {
+
+// CHECK: for.body:
+// CHECK: br label %for.inc
+ val = i;
+
+// CHECK: for.inc:
+// CHECK: br label %for.cond
+ }
+
+// CHECK: for.end:
+// CHECK: br label %for.cond1
+
+ // Infinite loop
+ for ( ; ; ) {
+// CHECK: for.cond1:
+// CHECK-NEXT: %[[#]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %[[#entry_token]]) ]
+// CHECK: br label %for.cond1
+ val = 0;
+ }
+
+// CHECK-NEXT: }
+// This loop in unreachable. Not generated.
+ // Null body
+ for (int j = 0; j < 10; ++j)
+ ;
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index cbf6e04f2844d6..21b3d2ec8b9649 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -31,6 +31,8 @@ let TargetPrefix = "spv" in {
def int_spv_bitcast : Intrinsic<[llvm_any_ty], [llvm_any_ty]>;
def int_spv_ptrcast : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
def int_spv_switch : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>;
+ def int_spv_loop_merge : Intrinsic<[], [llvm_vararg_ty]>;
+ def int_spv_selection_merge : Intrinsic<[], [llvm_vararg_ty]>;
def int_spv_cmpxchg : Intrinsic<[llvm_i32_ty], [llvm_any_ty, llvm_vararg_ty]>;
def int_spv_unreachable : Intrinsic<[], []>;
def int_spv_alloca : Intrinsic<[llvm_any_ty], []>;
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
index 25e285e35f9336..cc6daf7ef34426 100644
--- a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
@@ -203,7 +203,8 @@ class ConvergenceRegionAnalyzer {
private:
bool isBackEdge(const BasicBlock *From, const BasicBlock *To) const {
- assert(From != To && "From == To. This is awkward.");
+ if (From == To)
+ return true;
// We only handle loop in the simplified form. This means:
// - a single back-edge, a single latch.
@@ -230,6 +231,7 @@ class ConvergenceRegionAnalyzer {
auto *Terminator = From->getTerminator();
for (unsigned i = 0; i < Terminator->getNumSuccessors(); ++i) {
auto *To = Terminator->getSuccessor(i);
+ // Ignore back edges.
if (isBackEdge(From, To))
continue;
@@ -276,7 +278,6 @@ class ConvergenceRegionAnalyzer {
while (ToProcess.size() != 0) {
auto *L = ToProcess.front();
ToProcess.pop();
- assert(L->isLoopSimplifyForm());
auto CT = getConvergenceToken(L->getHeader());
SmallPtrSet<BasicBlock *, 8> RegionBlocks(L->block_begin(),
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
index f9e30e4effa1d9..e435c88c919c9c 100644
--- a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
+++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
@@ -130,6 +130,9 @@ class ConvergenceRegionInfo {
}
const ConvergenceRegion *getTopLevelRegion() const { return TopLevelRegion; }
+ ConvergenceRegion *getWritableTopLevelRegion() const {
+ return TopLevelRegion;
+ }
};
} // namespace SPIRV
diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
index 5f8aea5fc8d84d..198483e03a46d7 100644
--- a/llvm/lib/Target/SPIRV/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_target(SPIRVCodeGen
SPIRVMCInstLower.cpp
SPIRVMetadata.cpp
SPIRVModuleAnalysis.cpp
+ SPIRVStructurizer.cpp
SPIRVPreLegalizer.cpp
SPIRVPostLegalizer.cpp
SPIRVPrepareFunctions.cpp
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index 6c35a467f53bef..384133e7b4bd18 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -20,6 +20,7 @@ class InstructionSelector;
class RegisterBankInfo;
ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM);
+FunctionPass *createSPIRVStructurizerPass();
FunctionPass *createSPIRVMergeRegionExitTargetsPass();
FunctionPass *createSPIRVStripConvergenceIntrinsicsPass();
FunctionPass *createSPIRVRegularizerPass();
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index fed82b904af4f7..44f2da11edb051 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2296,6 +2296,19 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
}
return MIB.constrainAllUses(TII, TRI, RBI);
}
+ case Intrinsic::spv_loop_merge:
+ case Intrinsic::spv_selection_merge: {
+ const auto Opcode = IID == Intrinsic::spv_selection_merge
+ ? SPIRV::OpSelectionMerge
+ : SPIRV::OpLoopMerge;
+ auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode));
+ for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
+ assert(I.getOperand(i).isMBB());
+ MIB.addMBB(I.getOperand(i).getMBB());
+ }
+ MIB.addImm(SPIRV::SelectionControl::None);
+ return MIB.constrainAllUses(TII, TRI, RBI);
+ }
case Intrinsic::spv_cmpxchg:
return selectAtomicCmpXchg(ResVReg, ResType, I);
case Intrinsic::spv_unreachable:
diff --git a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
index 0747dd1bbaf40a..9930d067173df7 100644
--- a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
@@ -133,7 +133,7 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
// Run the pass on the given convergence region, ignoring the sub-regions.
// Returns true if the CFG changed, false otherwise.
bool runOnConvergenceRegionNoRecurse(LoopInfo &LI,
- const SPIRV::ConvergenceRegion *CR) {
+ SPIRV::ConvergenceRegion *CR) {
// Gather all the exit targets for this region.
SmallPtrSet<BasicBlock *, 4> ExitTargets;
for (BasicBlock *Exit : CR->Exits) {
@@ -198,14 +198,19 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
for (auto Exit : CR->Exits)
replaceBranchTargets(Exit, ExitTargets, NewExitTarget);
+ CR = CR->Parent;
+ while (CR) {
+ CR->Blocks.insert(NewExitTarget);
+ CR = CR->Parent;
+ }
+
return true;
}
/// Run the pass on the given convergence region and sub-regions (DFS).
/// Returns true if a region/sub-region was modified, false otherwise.
/// This returns as soon as one region/sub-region has been modified.
- bool runOnConvergenceRegion(LoopInfo &LI,
- const SPIRV::ConvergenceRegion *CR) {
+ bool runOnConvergenceRegion(LoopInfo &LI, SPIRV::ConvergenceRegion *CR) {
for (auto *Child : CR->Children)
if (runOnConvergenceRegion(LI, Child))
return true;
@@ -235,10 +240,10 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
virtual bool runOnFunction(Function &F) override {
LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
- const auto *TopLevelRegion =
+ auto *TopLevelRegion =
getAnalysis<SPIRVConvergenceRegionAnalysisWrapperPass>()
.getRegionInfo()
- .getTopLevelRegion();
+ .getWritableTopLevelRegion();
// FIXME: very inefficient method: each time a region is modified, we bubble
// back up, and recompute the whole convergence region tree. Once the
@@ -246,9 +251,6 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
// to be efficient instead of simple.
bool modified = false;
while (runOnConvergenceRegion(LI, TopLevelRegion)) {
- TopLevelRegion = getAnalysis<SPIRVConvergenceRegionAnalysisWrapperPass>()
- .getRegionInfo()
- .getTopLevelRegion();
modified = true;
}
@@ -262,6 +264,8 @@ class SPIRVMergeRegionExitTargets : public FunctionPass {
AU.addRequired<DominatorTreeWrapperPass>();
AU.addRequired<LoopInfoWrapperPass>();
AU.addRequired<SPIRVConvergenceRegionAnalysisWrapperPass>();
+
+ AU.addPreserved<SPIRVConvergenceRegionAnalysisWrapperPass>();
FunctionPass::getAnalysisUsage(AU);
}
};
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index df1b75bc1cb9eb..1784f00be600dd 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -744,79 +744,139 @@ static void insertSpirvDecorations(MachineFunction &MF, MachineIRBuilder MIB) {
MI->eraseFromParent();
}
-// Find basic blocks of the switch and replace registers in spv_switch() by its
-// MBB equivalent.
-static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR,
- MachineIRBuilder MIB) {
- DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
- SmallVector<std::pair<MachineInstr *, SmallVector<MachineInstr *, 8>>>
- Switches;
+// LLVM allows the switches to use registers as cases, while SPIR-V required
+// those to be immediate values. This function replaces such operands with the
+// equivalent immediate constant.
+static void processSwitchesConstants(MachineFunction &MF,
+ SPIRVGlobalRegistry *GR,
+ MachineIRBuilder MIB) {
+ MachineRegisterInfo &MRI = MF.getRegInfo();
for (MachineBasicBlock &MBB : MF) {
- MachineRegisterInfo &MRI = MF.getRegInfo();
- BB2MBB[MBB.getBasicBlock()] = &MBB;
for (MachineInstr &MI : MBB) {
if (!isSpvIntrinsic(MI, Intrinsic::spv_switch))
continue;
- // Calls to spv_switch intrinsics representing IR switches.
- SmallVector<MachineInstr *, 8> NewOps;
- for (unsigned i = 2; i < MI.getNumOperands(); ++i) {
+
+ SmallVector<MachineOperand, 8> NewOperands;
+ NewOperands.push_back(MI.getOperand(0)); // Opcode
+ NewOperands.push_back(MI.getOperand(1)); // Condition
+ NewOperands.push_back(MI.getOperand(2)); // Default
+ for (unsigned i = 3; i < MI.getNumOperands(); i += 2) {
Register Reg = MI.getOperand(i).getReg();
- if (i % 2 == 1) {
- MachineInstr *ConstInstr = getDefInstrMaybeConstant(Reg, &MRI);
- NewOps.push_back(ConstInstr);
- } else {
- MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
- assert(BuildMBB &&
- BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
- BuildMBB->getOperand(1).isBlockAddress() &&
- BuildMBB->getOperand(1).getBlockAddress());
- NewOps.push_back(BuildMBB);
- }
+ MachineInstr *ConstInstr = getDefInstrMaybeConstant(Reg, &MRI);
+ NewOperands.push_back(
+ MachineOperand::CreateCImm(ConstInstr->getOperand(1).getCImm()));
+
+ NewOperands.push_back(MI.getOperand(i + 1));
}
- Switches.push_back(std::make_pair(&MI, NewOps));
+
+ assert(MI.getNumOperands() == NewOperands.size());
+ while (MI.getNumOperands() > 0)
+ MI.removeOperand(0);
+ for (auto &MO : NewOperands)
+ MI.addOperand(MO);
}
}
+}
+// Some instructions are used during CodeGen but should never be emitted.
+// Cleaning up those.
+static void cleanupHelperInstructions(MachineFunction &MF) {
SmallPtrSet<MachineInstr *, 8> ToEraseMI;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ if (isSpvIntrinsic(MI, Intrinsic::spv_track_constant) ||
+ MI.getOpcode() == TargetOpcode::G_BRINDIRECT)
+ ToEraseMI.insert(&MI);
+ }
+ }
+
+ for (MachineInstr *MI : ToEraseMI)
+ MI->eraseFromParent();
+}
+
+// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
+// operands/registers by the actual MBB it references.
+static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
+ MachineIRBuilder MIB) {
+ // Gather the reverse-mapping BB -> MBB.
+ DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
+ for (MachineBasicBlock &MBB : MF)
+ BB2MBB[MBB.getBasicBlock()] = &MBB;
+
+ // Gather instructions requiring patching. For now, only those can use
+ // G_BLOCK_ADDR.
+ SmallVector<MachineInstr *, 8> InstructionsToPatch;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ if (isSpvIntrinsic(MI, Intrinsic::spv_switch) ||
+ isSpvIntrinsic(MI, Intrinsic::spv_loop_merge) ||
+ isSpvIntrinsic(MI, Intrinsic::spv_selection_merge))
+ InstructionsToPatch.push_back(&MI);
+ }
+ }
+
+ // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by
+ // the actual MBB it references. Once those references updated, we can cleanup
+ // remaining G_BLOCK_ADDR references.
SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
- for (auto &SwIt : Switches) {
- MachineInstr &MI = *SwIt.first;
- MachineBasicBlock *MBB = MI.getParent();
- SmallVector<MachineInstr *, 8> &Ins = SwIt.second;
+ SmallPtrSet<MachineInstr *, 8> ToEraseMI;
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+ for (MachineInstr *MI : InstructionsToPatch) {
SmallVector<MachineOperand, 8> NewOps;
- for (unsigned i = 0; i < Ins.size(); ++i) {
- if (Ins[i]->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
- BasicBlock *CaseBB =
- Ins[i]->getOperand(1).getBlockAddress()->getBasicBlock();
- auto It = BB2MBB.find(CaseBB);
- if (It == BB2MBB.end())
- report_fatal_error("cannot find a machine basic block by a basic "
- "block in a switch statement");
- MachineBasicBlock *Succ = It->second;
- ClearAddressTaken.insert(Succ);
- NewOps.push_back(MachineOperand::CreateMBB(Succ));
- if (!llvm::is_contained(MBB->successors(), Succ))
- MBB->addSuccessor(Succ);
- ToEraseMI.insert(Ins[i]);
- } else {
- NewOps.push_back(
- MachineOperand::CreateCImm(Ins[i]->getOperand(1).getCImm()));
+ for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
+ // The operand is not a register, keep as-is.
+ if (!MI->getOperand(i).isReg()) {
+ NewOps.push_back(MI->getOperand(i));
+ continue;
+ }
+
+ Register Reg = MI->getOperand(i).getReg();
+ MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
+ // The register is not the result of G_BLOCK_ADDR, keep as-is.
+ if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) {
+ NewOps.push_back(MI->getOperand(i));
+ continue;
}
+
+ assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
+ BuildMBB->getOperand(1).isBlockAddress() &&
+ BuildMBB->getOperand(1).getBlockAddress());
+ BasicBlock *BB =
+ BuildMBB->getOperand(1).getBlockAddress()->getBasicBlock();
+ auto It = BB2MBB.find(BB);
+ if (It == BB2MBB.end())
+ report_fatal_error("cannot find a machine basic block by a basic block "
+ "in a switch statement");
+ MachineBasicBlock *ReferencedBlock = It->second;
+ NewOps.push_back(MachineOperand::CreateMBB(ReferencedBlock));
+
+ ClearAddressTaken.insert(ReferencedBlock);
+ ToEraseMI.insert(BuildMBB);
}
- for (unsigned i = MI.getNumOperands() - 1; i > 1; --i)
- MI.removeOperand(i);
+
+ // Replace the operands.
+ assert(MI->getNumOperands() == NewOps.size());
+ while (MI->getNumOperands() > 0)
+ MI->removeOperand(0);
for (auto &MO : NewOps)
- MI.addOperand(MO);
- if (MachineInstr *Next = MI.getNextNode()) {
+ MI->addOperand(MO);
+
+ if (MachineInstr *Next = MI->getNextNode()) {
if (isSpvIntrinsic(*Next, Intrinsic::spv_track_constant)) {
ToEraseMI.insert(Next);
- Next = MI.getNextNode();
+ Next = MI->getNextNode();
}
if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)
ToEraseMI.insert(Next);
}
}
+ // BlockAddress operands were used to keep information between passes,
+ // let's undo the "address taken" status to reflect that Succ doesn't
+ // actually correspond to an IR-level basic block.
+ for (MachineBasicBlock *Succ : ClearAddressTaken)
+ Succ->setAddressTakenIRBlock(nullptr);
+
// If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
// this leaves their BasicBlock counterparts in a "address taken" status. This
// would make AsmPrinter to generate a series of unneeded labels of a "Address
@@ -835,12 +895,6 @@ static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR,
}
BlockAddrI->eraseFromParent();
}
-
- // BlockAddress operands were used to keep information between passes,
- // let's undo the "address taken" status to reflect that Succ doesn't
- // actually correspond to an IR-level basic block.
- for (MachineBasicBlock *Succ : ClearAddressTaken)
- Succ->setAddressTakenIRBlock(nullptr);
}
static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
@@ -891,7 +945,11 @@ bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) {
foldConstantsIntoIntrinsics(MF, TrackedConstRegs);
insertBitcasts(MF, GR, MIB);
generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);
- processSwitches(MF, GR, MIB);
+
+ processSwitchesConstants(MF, GR, MIB);
+ processBlockAddr(MF, GR, MIB);
+ cleanupHelperInstructions(MF);
+
processInstrsWithTypeFolding(MF, GR, MIB);
removeImplicitFallthroughs(MF, MIB);
insertSpirvDecorations(MF, MIB);
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
new file mode 100644
index 00000000000000..f663b7f427e235
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -0,0 +1,1410 @@
+//===-- SPIRVStructurizer.cpp ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
+#include "SPIRV.h"
+#include "SPIRVSubtarget.h"
+#include "SPIRVTargetMachine.h"
+#include "SPIRVUtils.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/CodeGen/IntrinsicLowering.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsSPIRV.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
+#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
+#include <queue>
+#include <stack>
+
+using namespace llvm;
+using namespace SPIRV;
+
+namespace llvm {
+
+void initializeSPIRVStructurizerPass(PassRegistry &);
+
+namespace {
+
+using BlockSet = std::unordered_set<BasicBlock *>;
+using Edge = std::pair<BasicBlock *, BasicBlock *>;
+
+// This class implements a partial ordering visitor, which visits a cyclic graph
+// in natural topological-like ordering. Topological ordering is not defined for
+// directed graphs with cycles, so this assumes cycles are a single node, and
+// ignores back-edges. The cycle is visited from the entry in the same
+// topological-like ordering.
+//
+// This means once we visit a node, we know all the possible ancestors have been
+// visited.
+//
+// clang-format off
+//
+// Given this graph:
+//
+// ,-> B -\
+// A -+ +---> D ----> E -> F -> G -> H
+// `-> C -/ ^ |
+// +-----------------+
+//
+// Visit order is:
+// A, [B, C in any order], D, E, F, G, H
+//
+// clang-format on
+//
+// Changing the function CFG between the construction of the visitor and
+// visiting is undefined. The visitor can be reused, but if the CFG is updated,
+// the visitor must be rebuilt.
+class PartialOrderingVisitor {
+ DomTreeBuilder::BBDomTree DT;
+ LoopInfo LI;
+ BlockSet Visited;
+ std::unordered_map<BasicBlock *, size_t> B2R;
+ std::vector<std::pair<BasicBlock *, size_t>> Order;
+
+ // Get all basic-blocks reachable from Start.
+ BlockSet getReachableFrom(BasicBlock *Start) {
+ std::queue<BasicBlock *> ToVisit;
+ ToVisit.push(Start);
+
+ BlockSet Output;
+ while (ToVisit.size() != 0) {
+ BasicBlock *BB = ToVisit.front();
+ ToVisit.pop();
+
+ if (Output.count(BB) != 0)
+ continue;
+ Output.insert(BB);
+
+ for (BasicBlock *Successor : successors(BB)) {
+ if (DT.dominates(Successor, BB))
+ continue;
+ ToVisit.push(Successor);
+ }
+ }
+
+ return Output;
+ }
+
+ size_t visit(BasicBlock *BB, size_t Rank) {
+ if (Visited.count(BB) != 0)
+ return Rank;
+
+ Loop *L = LI.getLoopFor(BB);
+ const bool isLoopHeader = LI.isLoopHeader(BB);
+
+ if (B2R.count(BB) == 0) {
+ B2R.emplace(BB, Rank);
+ } else {
+ B2R[BB] = std::max(B2R[BB], Rank);
+ }
+
+ for (BasicBlock *Predecessor : predecessors(BB)) {
+ if (isLoopHeader && L->contains(Predecessor)) {
+ continue;
+ }
+
+ if (B2R.count(Predecessor) == 0) {
+ return Rank;
+ }
+ }
+
+ Visited.insert(BB);
+
+ SmallVector<BasicBlock *, 2> OtherSuccessors;
+ BasicBlock *LoopSuccessor = nullptr;
+
+ for (BasicBlock *Successor : successors(BB)) {
+ // Ignoring back-edges.
+ if (DT.dominates(Successor, BB))
+ continue;
+
+ if (isLoopHeader && L->contains(Successor)) {
+ assert(LoopSuccessor == nullptr);
+ LoopSuccessor = Successor;
+ } else
+ OtherSuccessors.push_back(Successor);
+ }
+
+ if (LoopSuccessor)
+ Rank = visit(LoopSuccessor, Rank + 1);
+
+ size_t OutputRank = Rank;
+ for (BasicBlock *Item : OtherSuccessors)
+ OutputRank = std::max(OutputRank, visit(Item, Rank + 1));
+ return OutputRank;
+ };
+
+public:
+ // Build the visitor to operate on the function F.
+ PartialOrderingVisitor(Function &F) {
+ DT.recalculate(F);
+ LI = LoopInfo(DT);
+
+ visit(&*F.begin(), 0);
+
+ for (auto &[BB, Rank] : B2R)
+ Order.emplace_back(BB, Rank);
+
+ std::sort(Order.begin(), Order.end(), [](const auto &LHS, const auto &RHS) {
+ return LHS.second < RHS.second;
+ });
+
+ for (size_t i = 0; i < Order.size(); i++)
+ B2R[Order[i].first] = i;
+ }
+
+ // Visit the function starting from the basic block |Start|, and calling |Op|
+ // on each visited BB. This traversal ignores back-edges, meaning this won't
+ // visit a node to which |Start| is not an ancestor.
+ void partialOrderVisit(BasicBlock &Start,
+ std::function<bool(BasicBlock *)> Op) {
+ BlockSet Reachable = getReachableFrom(&Start);
+ assert(B2R.count(&Start) != 0);
+ size_t Rank = Order[B2R[&Start]].second;
+
+ auto It = Order.begin();
+ while (It != Order.end() && It->second < Rank)
+ ++It;
+
+ if (It == Order.end())
+ return;
+
+ size_t EndRank = Order.rbegin()->second + 1;
+ for (; It != Order.end() && It->second <= EndRank; ++It) {
+ if (Reachable.count(It->first) == 0) {
+ continue;
+ }
+
+ if (!Op(It->first)) {
+ EndRank = It->second;
+ }
+ }
+ }
+};
+
+// Helper function to do a partial order visit from the block |Start|, calling
+// |Op| on each visited node.
+void partialOrderVisit(BasicBlock &Start,
+ std::function<bool(BasicBlock *)> Op) {
+ PartialOrderingVisitor V(*Start.getParent());
+ V.partialOrderVisit(Start, Op);
+}
+
+// Returns the exact convergence region in the tree defined by `Node` for which
+// `BB` is the header, nullptr otherwise.
+const ConvergenceRegion *getRegionForHeader(const ConvergenceRegion *Node,
+ BasicBlock *BB) {
+ if (Node->Entry == BB)
+ return Node;
+
+ for (auto *Child : Node->Children) {
+ const auto *CR = getRegionForHeader(Child, BB);
+ if (CR != nullptr)
+ return CR;
+ }
+ return nullptr;
+}
+
+// Returns the single BasicBlock exiting the convergence region `CR`,
+// nullptr if no such exit exists. F must be the function CR belongs to.
+BasicBlock *getExitFor(const ConvergenceRegion *CR) {
+ std::unordered_set<BasicBlock *> ExitTargets;
+ for (BasicBlock *Exit : CR->Exits) {
+ for (BasicBlock *Successor : successors(Exit)) {
+ if (CR->Blocks.count(Successor) == 0)
+ ExitTargets.insert(Successor);
+ }
+ }
+
+ assert(ExitTargets.size() <= 1);
+ if (ExitTargets.size() == 0)
+ return nullptr;
+
+ return *ExitTargets.begin();
+}
+
+// Returns the merge block designated by I if I is a merge instruction, nullptr
+// otherwise.
+BasicBlock *getDesignatedMergeBlock(Instruction *I) {
+ IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
+ if (II == nullptr)
+ return nullptr;
+
+ if (II->getIntrinsicID() != Intrinsic::spv_loop_merge &&
+ II->getIntrinsicID() != Intrinsic::spv_selection_merge)
+ return nullptr;
+
+ BlockAddress *BA = cast<BlockAddress>(II->getOperand(0));
+ return BA->getBasicBlock();
+}
+
+// Returns the continue block designated by I if I is an OpLoopMerge, nullptr
+// otherwise.
+BasicBlock *getDesignatedContinueBlock(Instruction *I) {
+ IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
+ if (II == nullptr)
+ return nullptr;
+
+ if (II->getIntrinsicID() != Intrinsic::spv_loop_merge)
+ return nullptr;
+
+ BlockAddress *BA = cast<BlockAddress>(II->getOperand(1));
+ return BA->getBasicBlock();
+}
+
+// Returns true if Header has one merge instruction which designated Merge as
+// merge block.
+bool isDefinedAsSelectionMergeBy(BasicBlock &Header, BasicBlock &Merge) {
+ for (auto &I : Header) {
+ BasicBlock *MB = getDesignatedMergeBlock(&I);
+ if (MB == &Merge)
+ return true;
+ }
+ return false;
+}
+
+// Returns true if the BB has one OpLoopMerge instruction.
+bool hasLoopMergeInstruction(BasicBlock &BB) {
+ for (auto &I : BB)
+ if (getDesignatedContinueBlock(&I))
+ return true;
+ return false;
+}
+
+// Returns truye is I is an OpSelectionMerge or OpLoopMerge instruction, false
+// otherwise.
+bool isMergeInstruction(Instruction *I) {
+ return getDesignatedMergeBlock(I) != nullptr;
+}
+
+// Returns all blocks in F having at least one OpLoopMerge or OpSelectionMerge
+// instruction.
+SmallPtrSet<BasicBlock *, 2> getHeaderBlocks(Function &F) {
+ SmallPtrSet<BasicBlock *, 2> Output;
+ for (BasicBlock &BB : F) {
+ for (Instruction &I : BB) {
+ if (getDesignatedMergeBlock(&I) != nullptr)
+ Output.insert(&BB);
+ }
+ }
+ return Output;
+}
+
+// Returns all basic blocks in |F| referenced by at least 1
+// OpSelectionMerge/OpLoopMerge instruction.
+SmallPtrSet<BasicBlock *, 2> getMergeBlocks(Function &F) {
+ SmallPtrSet<BasicBlock *, 2> Output;
+ for (BasicBlock &BB : F) {
+ for (Instruction &I : BB) {
+ BasicBlock *MB = getDesignatedMergeBlock(&I);
+ if (MB != nullptr)
+ Output.insert(MB);
+ }
+ }
+ return Output;
+}
+
+// Return all the merge instructions contained in BB.
+// Note: the SPIR-V spec doesn't allow a single BB to contain more than 1 merge
+// instruction, but this can happen while we structurize the CFG.
+std::vector<Instruction *> getMergeInstructions(BasicBlock &BB) {
+ std::vector<Instruction *> Output;
+ for (Instruction &I : BB)
+ if (isMergeInstruction(&I))
+ Output.push_back(&I);
+ return Output;
+}
+
+// Returns all basic blocks in |F| referenced as continue target by at least 1
+// OpLoopMerge instruction.
+SmallPtrSet<BasicBlock *, 2> getContinueBlocks(Function &F) {
+ SmallPtrSet<BasicBlock *, 2> Output;
+ for (BasicBlock &BB : F) {
+ for (Instruction &I : BB) {
+ BasicBlock *MB = getDesignatedContinueBlock(&I);
+ if (MB != nullptr)
+ Output.insert(MB);
+ }
+ }
+ return Output;
+}
+
+// Do a preorder traversal of the CFG starting from the BB |Start|.
+// point. Calls |op| on each basic block encountered during the traversal.
+void visit(BasicBlock &Start, std::function<bool(BasicBlock *)> op) {
+ std::stack<BasicBlock *> ToVisit;
+ SmallPtrSet<BasicBlock *, 8> Seen;
+
+ ToVisit.push(&Start);
+ Seen.insert(ToVisit.top());
+ while (ToVisit.size() != 0) {
+ BasicBlock *BB = ToVisit.top();
+ ToVisit.pop();
+
+ if (!op(BB))
+ continue;
+
+ for (auto Succ : successors(BB)) {
+ if (Seen.contains(Succ))
+ continue;
+ ToVisit.push(Succ);
+ Seen.insert(Succ);
+ }
+ }
+}
+
+// Replaces the conditional and unconditional branch targets of |BB| by
+// |NewTarget| if the target was |OldTarget|. This function also makes sure the
+// associated merge instruction gets updated accordingly.
+void replaceIfBranchTargets(BasicBlock *BB, BasicBlock *OldTarget,
+ BasicBlock *NewTarget) {
+ auto *BI = cast<BranchInst>(BB->getTerminator());
+
+ // 1. Replace all matching successors.
+ for (size_t i = 0; i < BI->getNumSuccessors(); i++) {
+ if (BI->getSuccessor(i) == OldTarget)
+ BI->setSuccessor(i, NewTarget);
+ }
+
+ // Branch was unconditional, no fixup required.
+ if (BI->isUnconditional())
+ return;
+
+ // Branch had 2 successors, maybe now both are the same?
+ if (BI->getSuccessor(0) != BI->getSuccessor(1))
+ return;
+
+ // Note: we may end up here because the original IR had such branches.
+ // This means Target is not necessarily equal to NewTarget.
+ IRBuilder<> Builder(BB);
+ Builder.SetInsertPoint(BI);
+ Builder.CreateBr(BI->getSuccessor(0));
+ BI->eraseFromParent();
+
+ // The branch was the only instruction, nothing else to do.
+ if (BB->size() == 1)
+ return;
+
+ // Otherwise, we need to check: was there an OpSelectionMerge before this
+ // branch? If we removed the OpBranchConditional, we must also remove the
+ // OpSelectionMerge. This is not valid for OpLoopMerge:
+ IntrinsicInst *II =
+ dyn_cast<IntrinsicInst>(BB->getTerminator()->getPrevNode());
+ if (!II || II->getIntrinsicID() != Intrinsic::spv_selection_merge)
+ return;
+
+ Constant *C = cast<Constant>(II->getOperand(0));
+ II->eraseFromParent();
+ if (!C->isConstantUsed())
+ C->destroyConstant();
+}
+
+// Replaces the branching instruction destination of |BB| by |NewTarget| if it
+// was |OldTarget|. This function also fixes the associated merge instruction.
+// Note: this function does not simplify branching instructions, it only updates
+// targets. See also: simplifyBranches.
+void replaceBranchTargets(BasicBlock *BB, BasicBlock *OldTarget,
+ BasicBlock *NewTarget) {
+ auto *T = BB->getTerminator();
+ if (isa<ReturnInst>(T))
+ return;
+
+ if (isa<BranchInst>(T))
+ return replaceIfBranchTargets(BB, OldTarget, NewTarget);
+
+ if (auto *SI = dyn_cast<SwitchInst>(T)) {
+ for (size_t i = 0; i < SI->getNumSuccessors(); i++) {
+ if (SI->getSuccessor(i) == OldTarget)
+ SI->setSuccessor(i, NewTarget);
+ }
+ return;
+ }
+
+ assert(false && "Unhandled terminator type.");
+}
+
+// Replaces basic bloc operands |OldSrc| or OpPhi instructions in |BB| by
+// |NewSrc|. This function does not simplifies the OpPhi instruction once
+// transformed.
+void replacePhiTargets(BasicBlock *BB, BasicBlock *OldSrc, BasicBlock *NewSrc) {
+ for (PHINode &Phi : BB->phis()) {
+ int index = Phi.getBasicBlockIndex(OldSrc);
+ if (index == -1)
+ continue;
+ Phi.setIncomingBlock(index, NewSrc);
+ }
+}
+
+} // anonymous namespace
+
+// Given a reducible CFG, produces a structurized CFG in the SPIR-V sense,
+// adding merge instructions when required.
+class SPIRVStructurizer : public FunctionPass {
+
+ struct DivergentConstruct;
+ // Represents a list of condition/loops/switch constructs.
+ // See SPIR-V 2.11.2. Structured Control-flow Constructs for the list of
+ // constructs.
+ using ConstructList = std::vector<std::unique_ptr<DivergentConstruct>>;
+
+ // Represents a divergent construct in the SPIR-V sense.
+ // Such construct is represented by a header (entry), a merge block (exit),
+ // and possible a continue block (back-edge). Each construct can contain other
+ // constructs, but their boundaries do not cross.
+ struct DivergentConstruct {
+ BasicBlock *Header = nullptr;
+ BasicBlock *Merge = nullptr;
+ BasicBlock *Continue = nullptr;
+
+ DivergentConstruct *Parent = nullptr;
+ ConstructList Children;
+ };
+
+ // An helper class to clean the construct boundaries.
+ // It is used to gather the list of blocks that should belong to each
+ // divergent construct, and possibly modify CFG edges when exits would cross
+ // the boundary of multiple constructs.
+ struct Splitter {
+ Function &F;
+ LoopInfo &LI;
+ DomTreeBuilder::BBDomTree DT;
+ DomTreeBuilder::BBPostDomTree PDT;
+
+ Splitter(Function &F, LoopInfo &LI) : F(F), LI(LI) { invalidate(); }
+
+ void invalidate() {
+ PDT.recalculate(F);
+ DT.recalculate(F);
+ }
+
+ // Returns the list of blocks that belongs to a SPIR-V continue construct.
+ std::vector<BasicBlock *> getContinueConstructBlocks(BasicBlock *Header,
+ BasicBlock *Continue) {
+ std::vector<BasicBlock *> Output;
+ Loop *L = LI.getLoopFor(Continue);
+ BasicBlock *BackEdgeBlock = L->getLoopLatch();
+ assert(BackEdgeBlock);
+
+ partialOrderVisit(*Continue, [&](BasicBlock *BB) {
+ if (BB == Header)
+ return false;
+ Output.push_back(BB);
+ return true;
+ });
+ return Output;
+ }
+
+ // Returns the list of blocks that belongs to a SPIR-V loop construct.
+ std::vector<BasicBlock *> getLoopConstructBlocks(BasicBlock *Header,
+ BasicBlock *Merge,
+ BasicBlock *Continue) {
+ assert(DT.dominates(Header, Merge));
+ std::vector<BasicBlock *> Output;
+ partialOrderVisit(*Header, [&](BasicBlock *BB) {
+ if (BB == Merge)
+ return false;
+ if (DT.dominates(Merge, BB) || !DT.dominates(Header, BB))
+ return false;
+ Output.push_back(BB);
+ return true;
+ });
+ return Output;
+ }
+
+ // Returns the list of blocks that belongs to a SPIR-V selection construct.
+ std::vector<BasicBlock *>
+ getSelectionConstructBlocks(DivergentConstruct *Node) {
+ assert(DT.dominates(Node->Header, Node->Merge));
+ BlockSet OutsideBlocks;
+ OutsideBlocks.insert(Node->Merge);
+
+ for (DivergentConstruct *It = Node->Parent; It != nullptr;
+ It = It->Parent) {
+ OutsideBlocks.insert(It->Merge);
+ if (It->Continue)
+ OutsideBlocks.insert(It->Continue);
+ }
+
+ std::vector<BasicBlock *> Output;
+ partialOrderVisit(*Node->Header, [&](BasicBlock *BB) {
+ if (OutsideBlocks.count(BB) != 0)
+ return false;
+ if (DT.dominates(Node->Merge, BB) || !DT.dominates(Node->Header, BB))
+ return false;
+ Output.push_back(BB);
+ return true;
+ });
+ return Output;
+ }
+
+ // Returns the list of blocks that belongs to a SPIR-V switch construct.
+ std::vector<BasicBlock *> getSwitchConstructBlocks(BasicBlock *Header,
+ BasicBlock *Merge) {
+ assert(DT.dominates(Header, Merge));
+
+ std::vector<BasicBlock *> Output;
+ partialOrderVisit(*Header, [&](BasicBlock *BB) {
+ // the blocks structurally dominated by a switch header,
+ if (!DT.dominates(Header, BB))
+ return false;
+ // excluding blocks structurally dominated by the switch header’s merge
+ // block.
+ if (DT.dominates(Merge, BB) || BB == Merge)
+ return false;
+ Output.push_back(BB);
+ return true;
+ });
+ return Output;
+ }
+
+ // Returns the list of blocks that belongs to a SPIR-V case construct.
+ std::vector<BasicBlock *> getCaseConstructBlocks(BasicBlock *Target,
+ BasicBlock *Merge) {
+ assert(DT.dominates(Target, Merge));
+
+ std::vector<BasicBlock *> Output;
+ partialOrderVisit(*Target, [&](BasicBlock *BB) {
+ // the blocks structurally dominated by an OpSwitch Target or Default
+ // block
+ if (!DT.dominates(Target, BB))
+ return false;
+ // excluding the blocks structurally dominated by the OpSwitch
+ // construct’s corresponding merge block.
+ if (DT.dominates(Merge, BB) || BB == Merge)
+ return false;
+ Output.push_back(BB);
+ return true;
+ });
+ return Output;
+ }
+
+ // Splits the given edges by recreating proxy nodes so that the destination
+ // OpPhi instruction can still be viable.
+ //
+ // clang-format off
+ //
+ // In SPIR-V, construct must have a single exit/merge.
+ // Given A, B nodes in the construct, a C a node outside, with the following edges.
+ // A -> C
+ // B -> C
+ //
+ // In such cases, we must create a new exit node D, that belongs to the construct to make is viable:
+ // A -> D -> C
+ // B -> D -> C
+ //
+ // But if C had a phi node, adding such proxy-block breaks it. In such case, we must add 1 new block per
+ // exit, and patchup the phi node:
+ // A -> D -> D1 -> C
+ // B -> D -> D2 -> C
+ //
+ // A, B, D belongs to the construct. D is the exit. D1 and D2 are empty, just used as
+ // source operands for C's phi node.
+ //
+ // clang-format on
+ std::vector<Edge>
+ createAliasBlocksForComplexEdges(std::vector<Edge> Edges) {
+ std::unordered_map<BasicBlock *, BasicBlock *> Seen;
+ std::vector<Edge> Output;
+
+ for (auto &[Src, Dst] : Edges) {
+ if (Seen.count(Src) == 0) {
+ Seen.emplace(Src, Dst);
+ Output.emplace_back(Src, Dst);
+ continue;
+ }
+
+ // The exact same edge was already seen. Ignoring.
+ if (Seen[Src] == Dst)
+ continue;
+
+ // The same Src block branches to 2 distinct blocks. This will be an
+ // issue for the generated OpPhi. Creating alias block.
+ BasicBlock *NewSrc =
+ BasicBlock::Create(F.getContext(), "new.exit.src", &F);
+ replaceBranchTargets(Src, Dst, NewSrc);
+ replacePhiTargets(Dst, Src, NewSrc);
+
+ IRBuilder<> Builder(NewSrc);
+ Builder.CreateBr(Dst);
+
+ Seen.emplace(NewSrc, Dst);
+ Output.emplace_back(NewSrc, Dst);
+ }
+
+ return Output;
+ }
+
+ // Given a construct defined by |Header|, and a list of exiting edges
+ // |Edges|, creates a new single exit node, fixing up those edges.
+ BasicBlock *createSingleExitNode(BasicBlock *Header,
+ std::vector<Edge> &Edges) {
+ auto NewExit = BasicBlock::Create(F.getContext(), "new.exit", &F);
+ IRBuilder<> ExitBuilder(NewExit);
+
+ BlockSet SeenDst;
+ std::vector<BasicBlock *> Dsts;
+ std::unordered_map<BasicBlock *, ConstantInt *> DstToIndex;
+
+ // Given 2 edges: Src1 -> Dst, Src2 -> Dst:
+ // If Dst has an PHI node, and Src1 and Src2 are both operands, both Src1
+ // and Src2 cannot be hidden by NewExit. Create 2 new nodes: Alias1,
+ // Alias2 to which NewExit will branch before going to Dst. Then, patchup
+ // Dst PHI node to look for Alias1 and Alias2.
+ std::vector<Edge> FixedEdges = createAliasBlocksForComplexEdges(Edges);
+
+ for (auto &[Src, Dst] : FixedEdges) {
+ if (DstToIndex.count(Dst) != 0)
+ continue;
+ DstToIndex.emplace(Dst, ExitBuilder.getInt32(DstToIndex.size()));
+ Dsts.push_back(Dst);
+ }
+
+ if (Dsts.size() == 1) {
+ for (auto &[Src, Dst] : FixedEdges) {
+ replaceBranchTargets(Src, Dst, NewExit);
+ replacePhiTargets(Dst, Src, NewExit);
+ }
+ ExitBuilder.CreateBr(Dsts[0]);
+ return NewExit;
+ }
+
+ PHINode *PhiNode =
+ ExitBuilder.CreatePHI(ExitBuilder.getInt32Ty(), FixedEdges.size());
+
+ for (auto &[Src, Dst] : FixedEdges) {
+ PhiNode->addIncoming(DstToIndex[Dst], Src);
+ replaceBranchTargets(Src, Dst, NewExit);
+ replacePhiTargets(Dst, Src, NewExit);
+ }
+
+ // If we can avoid an OpSwitch, generate an OpBranch. Reason is some
+ // OpBranch are allowed to exist without a new OpSelectionMerge if one of
+ // the branch is the parent's merge node, while OpSwitches are not.
+ if (Dsts.size() == 2) {
+ Value *Condition = ExitBuilder.CreateCmp(CmpInst::ICMP_EQ,
+ DstToIndex[Dsts[0]], PhiNode);
+ ExitBuilder.CreateCondBr(Condition, Dsts[0], Dsts[1]);
+ return NewExit;
+ }
+
+ SwitchInst *Sw =
+ ExitBuilder.CreateSwitch(PhiNode, Dsts[0], Dsts.size() - 1);
+ for (auto It = Dsts.begin() + 1; It != Dsts.end(); ++It) {
+ Sw->addCase(DstToIndex[*It], *It);
+ }
+ return NewExit;
+ }
+ };
+
+ /// Create a value in BB set to the value associated with the branch the block
+ /// terminator will take.
+ Value *createExitVariable(
+ BasicBlock *BB,
+ const DenseMap<BasicBlock *, ConstantInt *> &TargetToValue) {
+ auto *T = BB->getTerminator();
+ if (isa<ReturnInst>(T))
+ return nullptr;
+
+ IRBuilder<> Builder(BB);
+ Builder.SetInsertPoint(T);
+
+ if (auto *BI = dyn_cast<BranchInst>(T)) {
+
+ BasicBlock *LHSTarget = BI->getSuccessor(0);
+ BasicBlock *RHSTarget =
+ BI->isConditional() ? BI->getSuccessor(1) : nullptr;
+
+ Value *LHS = TargetToValue.count(LHSTarget) != 0
+ ? TargetToValue.at(LHSTarget)
+ : nullptr;
+ Value *RHS = TargetToValue.count(RHSTarget) != 0
+ ? TargetToValue.at(RHSTarget)
+ : nullptr;
+
+ if (LHS == nullptr || RHS == nullptr)
+ return LHS == nullptr ? RHS : LHS;
+ return Builder.CreateSelect(BI->getCondition(), LHS, RHS);
+ }
+
+ // TODO: add support for switch cases.
+ llvm_unreachable("Unhandled terminator type.");
+ }
+
+ // Creates a new basic block in F with a single OpUnreachable instruction.
+ BasicBlock *CreateUnreachable(Function &F) {
+ BasicBlock *BB = BasicBlock::Create(F.getContext(), "new.exit", &F);
+ IRBuilder<> Builder(BB);
+ Builder.CreateUnreachable();
+ return BB;
+ }
+
+ // Add OpLoopMerge instruction on cycles.
+ bool addMergeForLoops(Function &F) {
+ LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ auto *TopLevelRegion =
+ getAnalysis<SPIRVConvergenceRegionAnalysisWrapperPass>()
+ .getRegionInfo()
+ .getTopLevelRegion();
+
+ bool Modified = false;
+ for (auto &BB : F) {
+ // Not a loop header. Ignoring for now.
+ if (!LI.isLoopHeader(&BB))
+ continue;
+ auto *L = LI.getLoopFor(&BB);
+
+ // This loop header is not the entrance of a convergence region. Ignoring
+ // this block.
+ auto *CR = getRegionForHeader(TopLevelRegion, &BB);
+ if (CR == nullptr)
+ continue;
+
+ IRBuilder<> Builder(&BB);
+
+ auto *Merge = getExitFor(CR);
+ // We are indeed in a loop, but there are no exits (infinite loop).
+ // TODO: I see no value in having real infinite loops in vulkan shaders.
+ // For now, I need to create a Merge block, and a structurally reachable
+ // block for it, but maybe we'd want to raise an error, as locking up the
+ // system is probably not wanted.
+ if (Merge == nullptr) {
+ BranchInst *Br = cast<BranchInst>(BB.getTerminator());
+ assert(cast<BranchInst>(BB.getTerminator())->isUnconditional());
+
+ Merge = CreateUnreachable(F);
+ Builder.SetInsertPoint(Br);
+ Builder.CreateCondBr(Builder.getFalse(), Merge, Br->getSuccessor(0));
+ Br->eraseFromParent();
+ }
+
+ auto *Continue = L->getLoopLatch();
+
+ Builder.SetInsertPoint(BB.getTerminator());
+ auto MergeAddress = BlockAddress::get(Merge->getParent(), Merge);
+ auto ContinueAddress = BlockAddress::get(Continue->getParent(), Continue);
+ SmallVector<Value *, 2> Args = {MergeAddress, ContinueAddress};
+
+ Builder.CreateIntrinsic(Intrinsic::spv_loop_merge, {}, {Args});
+ Modified = true;
+ }
+
+ return Modified;
+ }
+
+ // Adds an OpSelectionMerge to the immediate dominator or each node with an
+ // in-degree of 2 or more which is not already the merge target of an
+ // OpLoopMerge/OpSelectionMerge.
+ bool addMergeForNodesWithMultiplePredecessors(Function &F) {
+ DomTreeBuilder::BBDomTree DT;
+ DT.recalculate(F);
+
+ bool Modified = false;
+ for (auto &BB : F) {
+ if (pred_size(&BB) <= 1)
+ continue;
+
+ if (hasLoopMergeInstruction(BB) && pred_size(&BB) <= 2)
+ continue;
+
+ assert(DT.getNode(&BB)->getIDom());
+ BasicBlock *Header = DT.getNode(&BB)->getIDom()->getBlock();
+
+ if (isDefinedAsSelectionMergeBy(*Header, BB))
+ continue;
+
+ IRBuilder<> Builder(Header);
+ Builder.SetInsertPoint(Header->getTerminator());
+
+ auto MergeAddress = BlockAddress::get(BB.getParent(), &BB);
+ SmallVector<Value *, 1> Args = {MergeAddress};
+ Builder.CreateIntrinsic(Intrinsic::spv_selection_merge, {}, {Args});
+
+ Modified = true;
+ }
+
+ return Modified;
+ }
+
+ // When a block has multiple OpSelectionMerge/OpLoopMerge, sorts those
+ // instructions to but the "largest" first. A merge instruction is defined as
+ // larger than another when its target merge block post-dominates the other
+ // target's merge block.
+ // (This ordering should match the nesting ordering of the source HLSL).
+ bool sortSelectionMerge(Function &F, BasicBlock &Block) {
+ std::vector<Instruction *> MergeInstructions;
+ for (Instruction &I : Block)
+ if (isMergeInstruction(&I))
+ MergeInstructions.push_back(&I);
+
+ if (MergeInstructions.size() <= 1)
+ return false;
+
+ Instruction *InsertionPoint = *MergeInstructions.begin();
+
+ DomTreeBuilder::BBPostDomTree PDT;
+ PDT.recalculate(F);
+ std::sort(MergeInstructions.begin(), MergeInstructions.end(),
+ [&PDT](Instruction *Left, Instruction *Right) {
+ return PDT.dominates(getDesignatedMergeBlock(Right),
+ getDesignatedMergeBlock(Left));
+ });
+
+ for (Instruction *I : MergeInstructions) {
+ I->moveBefore(InsertionPoint);
+ InsertionPoint = I;
+ }
+
+ return true;
+ }
+
+ // Sorts selection merge headers in |F|.
+ // A is sorted before B if the merge block designated by B is an ancestor of
+ // the one designated by A.
+ bool sortSelectionMergeHeaders(Function &F) {
+ bool Modified = false;
+ for (BasicBlock &BB : F) {
+ Modified |= sortSelectionMerge(F, BB);
+ }
+ return Modified;
+ }
+
+ // Split basic blocks containing multiple OpLoopMerge/OpSelectionMerge
+ // instructions so each basic block contains only a single merge instruction.
+ bool splitBlocksWithMultipleHeaders(Function &F) {
+ std::stack<BasicBlock *> Work;
+ for (auto &BB : F) {
+ std::vector<Instruction *> MergeInstructions = getMergeInstructions(BB);
+ if (MergeInstructions.size() <= 1)
+ continue;
+ Work.push(&BB);
+ }
+
+ const bool Modified = Work.size() > 0;
+ while (Work.size() > 0) {
+ BasicBlock *Header = Work.top();
+ Work.pop();
+
+ std::vector<Instruction *> MergeInstructions =
+ getMergeInstructions(*Header);
+ for (unsigned i = 1; i < MergeInstructions.size(); i++) {
+ BasicBlock *NewBlock =
+ Header->splitBasicBlock(MergeInstructions[i], "new.header");
+
+ if (getDesignatedContinueBlock(MergeInstructions[0]) == nullptr) {
+ BasicBlock *Unreachable = CreateUnreachable(F);
+
+ BranchInst *BI = cast<BranchInst>(Header->getTerminator());
+ IRBuilder<> Builder(Header);
+ Builder.SetInsertPoint(BI);
+ Builder.CreateCondBr(Builder.getTrue(), NewBlock, Unreachable);
+ BI->eraseFromParent();
+ }
+
+ Header = NewBlock;
+ }
+ }
+
+ return Modified;
+ }
+
+ // Adds an OpSelectionMerge to each block with an out-degree >= 2 which
+ // doesn't already have an OpSelectionMerge.
+ bool addMergeForDivergentBlocks(Function &F) {
+ DomTreeBuilder::BBPostDomTree PDT;
+ PDT.recalculate(F);
+ bool Modified = false;
+
+ auto MergeBlocks = getMergeBlocks(F);
+ auto ContinueBlocks = getContinueBlocks(F);
+
+ for (auto &BB : F) {
+ if (getMergeInstructions(BB).size() != 0)
+ continue;
+
+ std::vector<BasicBlock *> Candidates;
+ for (BasicBlock *Successor : successors(&BB)) {
+ if (MergeBlocks.contains(Successor))
+ continue;
+ if (ContinueBlocks.contains(Successor))
+ continue;
+ Candidates.push_back(Successor);
+ }
+
+ if (Candidates.size() <= 1)
+ continue;
+
+ Modified = true;
+ BasicBlock *Merge = Candidates[0];
+
+ auto MergeAddress = BlockAddress::get(Merge->getParent(), Merge);
+ SmallVector<Value *, 1> Args = {MergeAddress};
+ IRBuilder<> Builder(&BB);
+ Builder.SetInsertPoint(BB.getTerminator());
+ Builder.CreateIntrinsic(Intrinsic::spv_selection_merge, {}, {Args});
+ }
+
+ return Modified;
+ }
+
+ // Gather all the exit nodes for the construct header by |Header| and
+ // containing the blocks |Construct|.
+ std::vector<Edge> getExitsFrom(const BlockSet &Construct,
+ BasicBlock &Header) {
+ std::vector<Edge> Output;
+ visit(Header, [&](BasicBlock *Item) {
+ if (Construct.count(Item) == 0)
+ return false;
+
+ for (BasicBlock *Successor : successors(Item)) {
+ if (Construct.count(Successor) == 0)
+ Output.emplace_back(Item, Successor);
+ }
+ return true;
+ });
+
+ return Output;
+ }
+
+ // Build a divergent construct tree searching from |BB|.
+ // If |Parent| is not null, this tree is attached to the parent's tree.
+ void constructDivergentConstruct(BlockSet &Visited, Splitter &S,
+ BasicBlock *BB, DivergentConstruct *Parent) {
+ if (Visited.count(BB) != 0)
+ return;
+ Visited.insert(BB);
+
+ auto MIS = getMergeInstructions(*BB);
+ if (MIS.size() == 0) {
+ for (BasicBlock *Successor : successors(BB))
+ constructDivergentConstruct(Visited, S, Successor, Parent);
+ return;
+ }
+
+ assert(MIS.size() == 1);
+ Instruction *MI = MIS[0];
+
+ BasicBlock *Merge = getDesignatedMergeBlock(MI);
+ BasicBlock *Continue = getDesignatedContinueBlock(MI);
+
+ auto Output = std::make_unique<DivergentConstruct>();
+ Output->Header = BB;
+ Output->Merge = Merge;
+ Output->Continue = Continue;
+ Output->Parent = Parent;
+
+ constructDivergentConstruct(Visited, S, Merge, Parent);
+ if (Continue)
+ constructDivergentConstruct(Visited, S, Continue, Output.get());
+
+ for (BasicBlock *Successor : successors(BB))
+ constructDivergentConstruct(Visited, S, Successor, Output.get());
+
+ if (Parent)
+ Parent->Children.emplace_back(std::move(Output));
+ }
+
+ // Returns the blocks belonging to the divergent construct |Node|.
+ BlockSet getConstructBlocks(Splitter &S, DivergentConstruct *Node) {
+ assert(Node->Header && Node->Merge);
+
+ if (Node->Continue) {
+ auto LoopBlocks =
+ S.getLoopConstructBlocks(Node->Header, Node->Merge, Node->Continue);
+ return BlockSet(LoopBlocks.begin(), LoopBlocks.end());
+ }
+
+ auto SelectionBlocks = S.getSelectionConstructBlocks(Node);
+ return BlockSet(SelectionBlocks.begin(), SelectionBlocks.end());
+ }
+
+ // Fixup the construct |Node| to respect a set of rules defined by the SPIR-V
+ // spec.
+ void fixupConstruct(Splitter &S, DivergentConstruct *Node) {
+ for (auto &Child : Node->Children)
+ fixupConstruct(S, Child.get());
+
+ // This construct is the root construct. Does not represent any real
+ // construct, just a way to access the first level of the forest.
+ if (Node->Parent == nullptr)
+ return;
+
+ // This node's parent is the root. Meaning this is a top-level construct.
+ // There can be multiple exists, but all are guaranteed to exit at most 1
+ // construct since we are at first level.
+ if (Node->Parent->Header == nullptr)
+ return;
+
+ // Health check for the structure.
+ assert(Node->Header && Node->Merge);
+ assert(Node->Parent->Header && Node->Parent->Merge);
+
+ BlockSet ConstructBlocks = getConstructBlocks(S, Node);
+ BlockSet ParentBlocks = getConstructBlocks(S, Node->Parent);
+
+ auto Edges = getExitsFrom(ConstructBlocks, *Node->Header);
+
+ // No edges exiting the construct.
+ if (Edges.size() < 1)
+ return;
+
+ bool HasBadEdge = Node->Merge == Node->Parent->Merge ||
+ Node->Merge == Node->Parent->Continue;
+ // BasicBlock *Target = Edges[0].second;
+ for (auto &[Src, Dst] : Edges) {
+ // - Breaking from a selection construct: S is a selection construct, S is
+ // the innermost structured
+ // control-flow construct containing A, and B is the merge block for S
+ // - Breaking from the innermost loop: S is the innermost loop construct
+ // containing A,
+ // and B is the merge block for S
+ if (Node->Merge == Dst)
+ continue;
+
+ // Entering the innermost loop’s continue construct: S is the innermost
+ // loop construct containing A, and B is the continue target for S
+ if (Node->Continue == Dst)
+ continue;
+
+ // TODO: what about cases branching to another case in the switch? Seems
+ // to work, but need to double check.
+ HasBadEdge = true;
+ }
+
+ if (!HasBadEdge)
+ return;
+
+ // Create a single exit node gathering all exit edges.
+ BasicBlock *NewExit = S.createSingleExitNode(Node->Header, Edges);
+
+ // Fixup this construct's merge node to point to the new exit.
+ // Note: this algorithm fixes inner-most divergence construct first. So
+ // recursive structures sharing a single merge node are fixed from the
+ // inside toward the outside.
+ auto MergeInstructions = getMergeInstructions(*Node->Header);
+ assert(MergeInstructions.size() == 1);
+ Instruction *I = MergeInstructions[0];
+ BlockAddress *BA = cast<BlockAddress>(I->getOperand(0));
+ if (BA->getBasicBlock() == Node->Merge) {
+ auto MergeAddress = BlockAddress::get(NewExit->getParent(), NewExit);
+ I->setOperand(0, MergeAddress);
+ }
+
+ // Clean up of the possible dangling BockAddr operands to prevent MIR
+ // comments about "address of removed block taken".
+ if (!BA->isConstantUsed())
+ BA->destroyConstant();
+
+ Node->Merge = NewExit;
+ // Regenerate the dom trees.
+ S.invalidate();
+ }
+
+ bool splitCriticalEdges(Function &F) {
+ LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ Splitter S(F, LI);
+
+ DivergentConstruct Root;
+ BlockSet Visited;
+ constructDivergentConstruct(Visited, S, &*F.begin(), &Root);
+ fixupConstruct(S, &Root);
+
+ return true;
+ }
+
+ // Simplify branches when possible:
+ // - if the 2 sides of a conditional branch are the same, transforms it to an
+ // unconditional branch.
+ // - if a switch has only 2 distinct successors, converts it to a conditional
+ // branch.
+ bool simplifyBranches(Function &F) {
+ bool Modified = false;
+
+ for (BasicBlock &BB : F) {
+ SwitchInst *SI = dyn_cast<SwitchInst>(BB.getTerminator());
+ if (!SI)
+ continue;
+ if (SI->getNumCases() > 1)
+ continue;
+
+ Modified = true;
+ IRBuilder<> Builder(&BB);
+ Builder.SetInsertPoint(SI);
+
+ if (SI->getNumCases() == 0) {
+ Builder.CreateBr(SI->getDefaultDest());
+ } else {
+ Value *Condition =
+ Builder.CreateCmp(CmpInst::ICMP_EQ, SI->getCondition(),
+ SI->case_begin()->getCaseValue());
+ Builder.CreateCondBr(Condition, SI->case_begin()->getCaseSuccessor(),
+ SI->getDefaultDest());
+ }
+ SI->eraseFromParent();
+ }
+
+ return Modified;
+ }
+
+ // Makes sure every case target in |F| are unique. If 2 case branch to the
+ // same basic block, one of the target is updated so it jumps to a new basic
+ // block ending with a single unconditional branch to the original target.
+ bool splitSwitchCases(Function &F) {
+ bool Modified = false;
+
+ for (BasicBlock &BB : F) {
+ SwitchInst *SI = dyn_cast<SwitchInst>(BB.getTerminator());
+ if (!SI)
+ continue;
+
+ BlockSet Seen;
+ Seen.insert(SI->getDefaultDest());
+
+ auto It = SI->case_begin();
+ while (It != SI->case_end()) {
+ BasicBlock *Target = It->getCaseSuccessor();
+ if (Seen.count(Target) == 0) {
+ Seen.insert(Target);
+ ++It;
+ continue;
+ }
+
+ Modified = true;
+ BasicBlock *NewTarget =
+ BasicBlock::Create(F.getContext(), "new.sw.case", &F);
+ IRBuilder<> Builder(NewTarget);
+ Builder.CreateBr(Target);
+ SI->addCase(It->getCaseValue(), NewTarget);
+ It = SI->removeCase(It);
+ }
+ }
+
+ return Modified;
+ }
+
+ bool IsRequiredForPhiNode(BasicBlock *BB) {
+ for (BasicBlock *Successor : successors(BB)) {
+ for (PHINode &Phi : Successor->phis()) {
+ if (Phi.getBasicBlockIndex(BB) != -1)
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ bool removeUselessBlocks(Function &F) {
+ std::vector<BasicBlock *> ToRemove;
+
+ auto MergeBlocks = getMergeBlocks(F);
+ auto ContinueBlocks = getContinueBlocks(F);
+
+ for (BasicBlock &BB : F) {
+ if (BB.size() != 1)
+ continue;
+
+ if (isa<ReturnInst>(BB.getTerminator()))
+ continue;
+
+ if (MergeBlocks.count(&BB) != 0 || ContinueBlocks.count(&BB) != 0)
+ continue;
+
+ if (IsRequiredForPhiNode(&BB))
+ continue;
+
+ if (BB.getUniqueSuccessor() == nullptr)
+ continue;
+
+ BasicBlock *Successor = BB.getUniqueSuccessor();
+ std::vector<BasicBlock *> Predecessors(predecessors(&BB).begin(),
+ predecessors(&BB).end());
+ for (BasicBlock *Predecessor : Predecessors)
+ replaceBranchTargets(Predecessor, &BB, Successor);
+ ToRemove.push_back(&BB);
+ }
+
+ for (BasicBlock *BB : ToRemove)
+ BB->eraseFromParent();
+
+ return ToRemove.size() != 0;
+ }
+
+ bool addHeaderToRemainingDivergentDAG(Function &F) {
+ bool Modified = false;
+
+ auto MergeBlocks = getMergeBlocks(F);
+ auto ContinueBlocks = getContinueBlocks(F);
+ auto HeaderBlocks = getHeaderBlocks(F);
+
+ DomTreeBuilder::BBDomTree DT;
+ DomTreeBuilder::BBPostDomTree PDT;
+ PDT.recalculate(F);
+ DT.recalculate(F);
+
+ for (BasicBlock &BB : F) {
+ if (HeaderBlocks.count(&BB) != 0)
+ continue;
+ if (succ_size(&BB) < 2)
+ continue;
+
+ size_t CandidateEdges = 0;
+ for (BasicBlock *Successor : successors(&BB)) {
+ if (MergeBlocks.count(Successor) != 0 ||
+ ContinueBlocks.count(Successor) != 0)
+ continue;
+ if (HeaderBlocks.count(Successor) != 0)
+ continue;
+ CandidateEdges += 1;
+ }
+
+ if (CandidateEdges <= 1)
+ continue;
+
+ BasicBlock *Header = &BB;
+ BasicBlock *Merge = PDT.getNode(&BB)->getIDom()->getBlock();
+
+ bool HasBadBlock = false;
+ visit(*Header, [&](const BasicBlock *Node) {
+ if (DT.dominates(Header, Node))
+ return false;
+ if (PDT.dominates(Merge, Node))
+ return false;
+ if (Node == Header || Node == Merge)
+ return true;
+
+ HasBadBlock |= MergeBlocks.count(Node) != 0 ||
+ ContinueBlocks.count(Node) != 0 ||
+ HeaderBlocks.count(Node) != 0;
+ return !HasBadBlock;
+ });
+
+ if (HasBadBlock)
+ continue;
+
+ Modified = true;
+ Instruction *SplitInstruction = Merge->getTerminator();
+ if (isMergeInstruction(SplitInstruction->getPrevNode()))
+ SplitInstruction = SplitInstruction->getPrevNode();
+ BasicBlock *NewMerge =
+ Merge->splitBasicBlockBefore(SplitInstruction, "new.merge");
+
+ IRBuilder<> Builder(Header);
+ Builder.SetInsertPoint(Header->getTerminator());
+
+ auto MergeAddress = BlockAddress::get(NewMerge->getParent(), NewMerge);
+ SmallVector<Value *, 1> Args = {MergeAddress};
+ Builder.CreateIntrinsic(Intrinsic::spv_selection_merge, {}, {Args});
+ }
+
+ return Modified;
+ }
+
+public:
+ static char ID;
+
+ SPIRVStructurizer() : FunctionPass(ID) {
+ initializeSPIRVStructurizerPass(*PassRegistry::getPassRegistry());
+ };
+
+ virtual bool runOnFunction(Function &F) override {
+ bool Modified = false;
+
+ // In LLVM, Switches are allowed to have several cases branching to the same
+ // basic block. In SPIR-V, each target must be a distrinct block. This
+ // function makes sure each target is unique.
+ Modified |= splitSwitchCases(F);
+
+ // LLVM allows conditional branches to have both side jumping to the same
+ // block. It also allows switched to have a single default, or just one
+ // case. Cleaning this up now.
+ Modified |= simplifyBranches(F);
+
+ // At this state, we should have a reducible CFG with cycles.
+ // STEP 1: Adding OpLoopMerge instructions to loop headers.
+ Modified |= addMergeForLoops(F);
+
+ // STEP 2: adding OpSelectionMerge to each node with an in-degree >= 2.
+ Modified |= addMergeForNodesWithMultiplePredecessors(F);
+
+ // STEP 3:
+ // Sort selection merge, the largest construct goes first.
+ // This simpligies the next step.
+ Modified |= sortSelectionMergeHeaders(F);
+
+ // STEP 4: As this stage, we can have a single basic block with multiple
+ // OpLoopMerge/OpSelectionMerge instructions. Splitting this block so each
+ // BB has a single merge instruction.
+ Modified |= splitBlocksWithMultipleHeaders(F);
+
+ // STEP 5: In the previous steps, we added merge blocks the loops and
+ // natural merge blocks (in-degree >= 2). What remains are conditions with
+ // an exiting branch (return, unreachable). In such case, we must start from
+ // the header, and add headers to divergent construct with no headers.
+ Modified |= addMergeForDivergentBlocks(F);
+
+ // STEP 6: At this stage, we have several divergent construct defines by a
+ // header and a merge block. But their boundaries have no constraints: a
+ // construct exit could be outside of the parents' construct exit. Such
+ // edges are called critical edges. What we need is to split those edges
+ // into several parts. Each part exiting the parent's construct by its merge
+ // block.
+ Modified |= splitCriticalEdges(F);
+
+ // STEP 7: The previous steps possibly created a lot of "proxy" blocks.
+ // Blocks with a single unconditional branch, used to create a valid
+ // divergent construct tree. Some nodes are still requires (e.g: nodes
+ // allowing a valid exit through the parent's merge block). But some are
+ // left-overs of past transformations, and could cause actual validation
+ // issues. E.g: the SPIR-V spec allows a construct to break to the parents
+ // loop construct without an OpSelectionMerge, but this requires a straight
+ // jump. If a proxy block lies between the conditional branch and the
+ // parent's merge, the CFG is not valid.
+ Modified |= removeUselessBlocks(F);
+
+ // STEP 8: Final fix-up steps: our tree boundaries are correct, but some
+ // blocks are branching with no header. Those are often simple conditional
+ // branches with 1 or 2 returning edges. Adding a header for those.
+ Modified |= addHeaderToRemainingDivergentDAG(F);
+
+ return Modified;
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<DominatorTreeWrapperPass>();
+ AU.addRequired<LoopInfoWrapperPass>();
+ AU.addRequired<SPIRVConvergenceRegionAnalysisWrapperPass>();
+
+ AU.addPreserved<SPIRVConvergenceRegionAnalysisWrapperPass>();
+ FunctionPass::getAnalysisUsage(AU);
+ }
+};
+} // namespace llvm
+
+char SPIRVStructurizer::ID = 0;
+
+INITIALIZE_PASS_BEGIN(SPIRVStructurizer, "structurizer", "structurize SPIRV",
+ false, false)
+INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(SPIRVConvergenceRegionAnalysisWrapperPass)
+
+INITIALIZE_PASS_END(SPIRVStructurizer, "structurize", "structurize SPIRV",
+ false, false)
+
+FunctionPass *llvm::createSPIRVStructurizerPass() {
+ return new SPIRVStructurizer();
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index 7058b15d53aa0b..e5384b2eb2c2c1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -159,11 +159,9 @@ TargetPassConfig *SPIRVTargetMachine::createPassConfig(PassManagerBase &PM) {
}
void SPIRVPassConfig::addIRPasses() {
- if (TM.getSubtargetImpl()->isVulkanEnv()) {
- // Once legalized, we need to structurize the CFG to follow the spec.
- // This is done through the following 8 steps.
- // TODO(#75801): add the remaining steps.
+ TargetPassConfig::addIRPasses();
+ if (TM.getSubtargetImpl()->isVulkanEnv()) {
// 1. Simplify loop for subsequent transformations. After this steps, loops
// have the following properties:
// - loops have a single entry edge (pre-header to loop header).
@@ -175,9 +173,11 @@ void SPIRVPassConfig::addIRPasses() {
// regions are single-entry, single-exit. This will help determine the
// correct merge block.
addPass(createSPIRVMergeRegionExitTargetsPass());
+
+ // 3. Structurize.
+ addPass(createSPIRVStructurizerPass());
}
- TargetPassConfig::addIRPasses();
addPass(createSPIRVRegularizerPass());
addPass(createSPIRVPrepareFunctionsPass(TM));
addPass(createSPIRVStripConvergenceIntrinsicsPass());
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 0f449fa2d45be9..7c52dd51cea818 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -24,6 +24,7 @@ llvm_canonicalize_cmake_booleans(
LLVM_ENABLE_REVERSE_ITERATION
LLVM_INCLUDE_DXIL_TESTS
LLVM_TOOL_LLVM_DRIVER_BUILD
+ LLVM_INCLUDE_SPIRV_SIMULATOR_TESTS
LLVM_INCLUDE_SPIRV_TOOLS_TESTS
LLVM_APPEND_VC_REV
LLVM_HAS_LOGF128
diff --git a/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll b/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
deleted file mode 100644
index be482c675245ef..00000000000000
--- a/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
+++ /dev/null
@@ -1,87 +0,0 @@
-; RUN: llc -mtriple=spirv-unknown-unknown -O0 %s -o - | FileCheck %s
-
-; CHECK-DAG: OpDecorate %[[#SubgroupLocalInvocationId:]] BuiltIn SubgroupLocalInvocationId
-; CHECK-DAG: %[[#bool:]] = OpTypeBool
-; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint]] 0
-; CHECK-DAG: OpName %[[#main:]] "main"
-
-define void @main() #1 {
- %1 = icmp ne i32 0, 0
- %t1 = call token @llvm.experimental.convergence.entry()
- br i1 %1, label %l1, label %l2
-
-; CHECK: %[[#main]] = OpFunction
-; CHECK: %[[#cond:]] = OpINotEqual %[[#bool]] %[[#uint_0]] %[[#uint_0]]
-; CHECK: OpBranchConditional %[[#cond]] %[[#l1_pre:]] %[[#l2_pre:]]
-
-; CHECK-DAG: %[[#l2_pre]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l2_header:]]
-
-; CHECK-DAG: %[[#l2_header]] = OpLabel
-; CHECK-NEXT: OpBranchConditional %[[#cond]] %[[#l2_body:]] %[[#l2_end:]]
-
-; CHECK-DAG: %[[#l2_end]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#end:]]
-
-; CHECK-DAG: %[[#end]] = OpLabel
-; CHECK-NEXT: OpReturn
-
-; CHECK-DAG: %[[#l2_body]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l2_continue:]]
-
-; CHECK-DAG: %[[#l2_continue]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l2_header]]
-
-l1:
- %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
- br i1 %1, label %l1_body, label %l1_end
-
-; CHECK-DAG: %[[#l1_pre]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l1_header:]]
-
-; CHECK-DAG: %[[#l1_header]] = OpLabel
-; CHECK-NEXT: OpBranchConditional %[[#cond]] %[[#l1_body:]] %[[#l1_end:]]
-
-; CHECK-DAG: %[[#l1_end]] = OpLabel
-; CHECK-DAG: %[[#]] = OpLoad %[[#]] %[[#SubgroupLocalInvocationId]]
-; CHECK-NEXT: OpBranch %[[#end]]
-
-l1_body:
- br label %l1_continue
-; CHECK-DAG: %[[#l1_body]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l1_continue:]]
-
-l1_continue:
- br label %l1
-; CHECK-DAG: %[[#l1_continue]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#l1_header]]
-
-l1_end:
- %call = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %tl1) ]
- br label %end
-
-l2:
- %tl2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
- br i1 %1, label %l2_body, label %l2_end
-
-l2_body:
- br label %l2_continue
-
-l2_continue:
- br label %l2
-
-l2_end:
- br label %end
-
-end:
- ret void
-}
-
-attributes #1 = { "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" convergent }
-
-declare token @llvm.experimental.convergence.entry()
-declare token @llvm.experimental.convergence.control()
-declare token @llvm.experimental.convergence.loop()
-
-declare i32 @__hlsl_wave_get_lane_index() convergent
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
new file mode 100644
index 00000000000000..7e0720aeaa9476
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
@@ -0,0 +1,168 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; CHECK-DAG: OpName %[[#process:]] "_Z7processv"
+; CHECK-DAG: OpName %[[#fn:]] "_Z2fnv"
+; CHECK-DAG: OpName %[[#fn1:]] "_Z3fn1v"
+; CHECK-DAG: OpName %[[#fn2:]] "_Z3fn2v"
+; CHECK-DAG: OpName %[[#val:]] "val"
+; CHECK-DAG: OpName %[[#a:]] "a"
+; CHECK-DAG: OpName %[[#b:]] "b"
+; CHECK-DAG: OpName %[[#c:]] "c"
+
+; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool
+; CHECK-DAG: %[[#int_pfty:]] = OpTypePointer Function %[[#int_ty]]
+
+; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#int_ty]] 0
+
+declare token @llvm.experimental.convergence.entry() #1
+
+; CHECK: %[[#fn]] = OpFunction %[[#int_ty]]
+define spir_func noundef i32 @_Z2fnv() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; CHECK: %[[#fn1]] = OpFunction %[[#int_ty]]
+define spir_func noundef i32 @_Z3fn1v() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 0
+}
+
+; CHECK: %[[#fn2]] = OpFunction %[[#int_ty]]
+define spir_func noundef i32 @_Z3fn2v() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; CHECK: %[[#process]] = OpFunction %[[#int_ty]]
+define spir_func noundef i32 @_Z7processv() #0 {
+entry:
+ ; CHECK: %[[#entry:]] = OpLabel
+ %0 = call token @llvm.experimental.convergence.entry()
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ %c = alloca i32, align 4
+ %val = alloca i32, align 4
+ store i32 0, ptr %a, align 4
+ store i32 1, ptr %b, align 4
+ store i32 2, ptr %c, align 4
+ store i32 0, ptr %val, align 4
+ ; CHECK-DAG: %[[#a]] = OpVariable %[[#int_pfty]] Function
+ ; CHECK-DAG: %[[#b]] = OpVariable %[[#int_pfty]] Function
+ ; CHECK-DAG: %[[#c]] = OpVariable %[[#int_pfty]] Function
+ ; CHECK-DAG: %[[#val]] = OpVariable %[[#int_pfty]] Function
+ %1 = load i32, ptr %a, align 4
+ %tobool = icmp ne i32 %1, 0
+ br i1 %tobool, label %cond.true, label %cond.false
+ ; CHECK: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#a]]
+ ; CHECK: %[[#cond:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+ ; CHECK: OpSelectionMerge %[[#cond_end:]]
+ ; CHECK: OpBranchConditional %[[#cond]] %[[#cond_true:]] %[[#cond_false:]]
+
+cond.false: ; preds = %entry
+ %2 = load i32, ptr %c, align 4
+ br label %cond.end
+ ; CHECK: %[[#cond_false]] = OpLabel
+ ; CHECK: %[[#load_c:]] = OpLoad %[[#]] %[[#c]]
+ ; CHECK: OpBranch %[[#cond_end]]
+
+cond.end: ; preds = %cond.false, %cond.true
+ %cond = phi i32 [ %6, %cond.true ], [ %2, %cond.false ]
+ %tobool1 = icmp ne i32 %cond, 0
+ br i1 %tobool1, label %if.then, label %if.end
+ ; CHECK: %[[#cond_end]] = OpLabel
+ ; CHECK: %[[#tmp:]] = OpPhi %[[#int_ty]] %[[#load_b:]] %[[#cond_true]] %[[#load_c]] %[[#cond_false]]
+ ; CHECK: OpSelectionMerge %[[#if_end:]]
+ ; CHECK: OpBranchConditional %[[#]] %[[#if_then:]] %[[#if_end]]
+
+if.end: ; preds = %if.then, %cond.end
+ %call2 = call spir_func noundef i32 @_Z2fnv() #4 [ "convergencectrl"(token %0) ]
+ %tobool3 = icmp ne i32 %call2, 0
+ br i1 %tobool3, label %cond.true4, label %cond.false6
+ ; CHECK: %[[#if_end]] = OpLabel
+ ; CHECK: OpSelectionMerge %[[#cond_end8:]]
+ ; CHECK: OpBranchConditional %[[#]] %[[#cond_true4:]] %[[#cond_false6:]]
+
+cond.false6: ; preds = %if.end
+ %call7 = call spir_func noundef i32 @_Z3fn2v() #4 [ "convergencectrl"(token %0) ]
+ br label %cond.end8
+ ; CHECK: %[[#cond_false6]] = OpLabel
+ ; CHECK: OpBranch %[[#cond_end8]]
+
+cond.end8: ; preds = %cond.false6, %cond.true4
+ %cond9 = phi i32 [ %call5, %cond.true4 ], [ %call7, %cond.false6 ]
+ %tobool10 = icmp ne i32 %cond9, 0
+ br i1 %tobool10, label %if.then11, label %if.end13
+ ; CHECK: %[[#cond_end8]] = OpLabel
+ ; CHECK: OpSelectionMerge %[[#if_end13:]]
+ ; CHECK: OpBranchConditional %[[#]] %[[#if_then11:]] %[[#if_end13]]
+
+if.end13: ; preds = %if.then11, %cond.end8
+ %3 = load i32, ptr %val, align 4
+ ret i32 %3
+ ; CHECK: %[[#if_end13]] = OpLabel
+ ; CHECK: OpReturnValue
+
+if.then11: ; preds = %cond.end8
+ %4 = load i32, ptr %val, align 4
+ %inc12 = add nsw i32 %4, 1
+ store i32 %inc12, ptr %val, align 4
+ br label %if.end13
+ ; CHECK: %[[#if_then11]] = OpLabel
+ ; CHECK: OpBranch %[[#if_end13]]
+
+cond.true4: ; preds = %if.end
+ %call5 = call spir_func noundef i32 @_Z3fn1v() #4 [ "convergencectrl"(token %0) ]
+ br label %cond.end8
+ ; CHECK: %[[#cond_true4]] = OpLabel
+ ; CHECK: OpBranch %[[#cond_end8]]
+
+if.then: ; preds = %cond.end
+ %5 = load i32, ptr %val, align 4
+ %inc = add nsw i32 %5, 1
+ store i32 %inc, ptr %val, align 4
+ br label %if.end
+ ; CHECK: %[[#if_then]] = OpLabel
+ ; CHECK: OpBranch %[[#if_end]]
+
+cond.true: ; preds = %entry
+ %6 = load i32, ptr %b, align 4
+ br label %cond.end
+ ; CHECK: %[[#cond_true]] = OpLabel
+ ; CHECK: OpBranch %[[#cond_end]]
+}
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal spir_func void @main() #2 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %call1 = call spir_func noundef i32 @_Z7processv() #4 [ "convergencectrl"(token %0) ]
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define void @main.1() #3 {
+entry:
+ call void @main()
+ ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #4 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll
new file mode 100644
index 00000000000000..3fc440dc445e14
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.break.ll
@@ -0,0 +1,169 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=2
+
+; int foo() { return true; }
+;
+; int process() {
+; int val = 0;
+; int i = 0;
+;
+; do {
+; ++i;
+; if (i > 5) {
+; break;
+; break; // No SPIR-V should be emitted for this statement.
+; val = i; // No SPIR-V should be emitted for this statement.
+; while(true); // No SPIR-V should be emitted for this statement.
+; }
+;
+; val = i;
+; {
+; {
+; break;
+; }
+; }
+; val = val * 2; // No SPIR-V should be emitted for this statement.
+;
+; } while (i < 10);
+;
+; ////////////////////////////////////////////////////////////////////////////////
+; // Nested do-while loops with break statements //
+; // Each break statement should branch to the corresponding loop's break block //
+; ////////////////////////////////////////////////////////////////////////////////
+;
+; do {
+; ++i;
+;
+; do {
+; ++val;
+; break;
+; } while (i < 10);
+;
+; --i;
+;
+; {
+; break;
+; }
+;
+; } while(val < 10);
+;
+; return val;
+; }
+;
+; [numthreads(1, 1, 1)]
+; void main() {
+; process();
+; }
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @_Z3foov() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+do.body: ; preds = %do.cond, %entry
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %2, 1
+ store i32 %inc, ptr %i, align 4
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp sgt i32 %3, 5
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %do.body
+ br label %do.end
+
+if.end: ; preds = %do.body
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %do.end
+
+do.cond: ; No predecessors!
+ %5 = load i32, ptr %i, align 4
+ %cmp1 = icmp slt i32 %5, 10
+ br i1 %cmp1, label %do.body, label %do.end
+
+do.end: ; preds = %do.cond, %if.end, %if.then
+ br label %do.body2
+
+do.body2: ; preds = %do.cond9, %do.end
+ %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %7 = load i32, ptr %i, align 4
+ %inc3 = add nsw i32 %7, 1
+ store i32 %inc3, ptr %i, align 4
+ br label %do.body4
+
+do.body4: ; preds = %do.cond6, %do.body2
+ %8 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
+ %9 = load i32, ptr %val, align 4
+ %inc5 = add nsw i32 %9, 1
+ store i32 %inc5, ptr %val, align 4
+ br label %do.end8
+
+do.cond6: ; No predecessors!
+ %10 = load i32, ptr %i, align 4
+ %cmp7 = icmp slt i32 %10, 10
+ br i1 %cmp7, label %do.body4, label %do.end8
+
+do.end8: ; preds = %do.cond6, %do.body4
+ %11 = load i32, ptr %i, align 4
+ %dec = add nsw i32 %11, -1
+ store i32 %dec, ptr %i, align 4
+ br label %do.end11
+
+do.cond9: ; No predecessors!
+ %12 = load i32, ptr %val, align 4
+ %cmp10 = icmp slt i32 %12, 10
+ br i1 %cmp10, label %do.body2, label %do.end11
+
+do.end11: ; preds = %do.cond9, %do.end8
+ %13 = load i32, ptr %val, align 4
+ ret i32 %13
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal spir_func void @main() #2 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %call1 = call spir_func noundef i32 @_Z7processv() #4 [ "convergencectrl"(token %0) ]
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define void @main.1() #3 {
+entry:
+ call void @main()
+ ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #4 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll
new file mode 100644
index 00000000000000..7fa35132d31f7c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.continue.ll
@@ -0,0 +1,167 @@
+; RUN: %if spirv-tools %{ llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=10
+;
+; Source HLSL:
+;
+; int foo() { return true; }
+;
+; int process() {
+; int val = 0;
+; int i = 0;
+;
+; do {
+; ++i;
+; if (i > 5) {
+; {
+; {
+; continue;
+; }
+; }
+; val = i; // No SPIR-V should be emitted for this statement.
+; while(true); // No SPIR-V should be emitted for this statement.
+; }
+; val = i;
+; continue;
+; val = val * 2; // No SPIR-V should be emitted for this statement.
+; continue; // No SPIR-V should be emitted for this statement.
+;
+; } while (i < 10);
+;
+; //////////////////////////////////////////////////////////////////////////////////////
+; // Nested do-while loops with continue statements //
+; // Each continue statement should branch to the corresponding loop's continue block //
+; //////////////////////////////////////////////////////////////////////////////////////
+;
+; do {
+; ++i;
+; do {
+; ++val;
+; continue;
+; } while (i < 10);
+;
+; --i;
+; continue;
+; continue; // No SPIR-V should be emitted for this statement.
+;
+; } while(val < 10);
+;
+; return val;
+; }
+;
+; [numthreads(1, 1, 1)]
+; void main() {
+; process();
+; }
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @_Z3foov() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+do.body: ; preds = %do.cond, %entry
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %2, 1
+ store i32 %inc, ptr %i, align 4
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp sgt i32 %3, 5
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %do.body
+ br label %do.cond
+
+if.end: ; preds = %do.body
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %do.cond
+
+do.cond: ; preds = %if.end, %if.then
+ %5 = load i32, ptr %i, align 4
+ %cmp1 = icmp slt i32 %5, 10
+ br i1 %cmp1, label %do.body, label %do.end
+
+do.end: ; preds = %do.cond
+ br label %do.body2
+
+do.body2: ; preds = %do.cond9, %do.end
+ %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %7 = load i32, ptr %i, align 4
+ %inc3 = add nsw i32 %7, 1
+ store i32 %inc3, ptr %i, align 4
+ br label %do.body4
+
+do.body4: ; preds = %do.cond6, %do.body2
+ %8 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
+ %9 = load i32, ptr %val, align 4
+ %inc5 = add nsw i32 %9, 1
+ store i32 %inc5, ptr %val, align 4
+ br label %do.cond6
+
+do.cond6: ; preds = %do.body4
+ %10 = load i32, ptr %i, align 4
+ %cmp7 = icmp slt i32 %10, 10
+ br i1 %cmp7, label %do.body4, label %do.end8
+
+do.end8: ; preds = %do.cond6
+ %11 = load i32, ptr %i, align 4
+ %dec = add nsw i32 %11, -1
+ store i32 %dec, ptr %i, align 4
+ br label %do.cond9
+
+do.cond9: ; preds = %do.end8
+ %12 = load i32, ptr %val, align 4
+ %cmp10 = icmp slt i32 %12, 10
+ br i1 %cmp10, label %do.body2, label %do.end11
+
+do.end11: ; preds = %do.cond9
+ %13 = load i32, ptr %val, align 4
+ ret i32 %13
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal spir_func void @main() #2 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %call1 = call spir_func noundef i32 @_Z7processv() #4 [ "convergencectrl"(token %0) ]
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define void @main.1() #3 {
+entry:
+ call void @main()
+ ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #4 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll
new file mode 100644
index 00000000000000..a28e1c7b942de0
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.do.nested.ll
@@ -0,0 +1,138 @@
+; RUN: %if spirv-tools %{ llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=142
+;
+; Source HLSL:
+;
+; int process() {
+; int val=0, i=0, j=0, k=0;
+;
+; do {
+; val = val + i;
+;
+; do {
+; do {
+; ++k;
+; } while (k < 30);
+;
+; ++j;
+; } while (j < 20);
+;
+; ++i;
+;
+; } while (i < 10);
+;
+; return val + i + j + k;
+; }
+;
+; [numthreads(1, 1, 1)]
+; void main() {
+; process();
+; }
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ %j = alloca i32, align 4
+ %k = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ store i32 0, ptr %j, align 4
+ store i32 0, ptr %k, align 4
+ br label %do.body
+
+do.body: ; preds = %do.cond8, %entry
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %val, align 4
+ %3 = load i32, ptr %i, align 4
+ %add = add nsw i32 %2, %3
+ store i32 %add, ptr %val, align 4
+ br label %do.body1
+
+do.body1: ; preds = %do.cond4, %do.body
+ %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %1) ]
+ br label %do.body2
+
+do.body2: ; preds = %do.cond, %do.body1
+ %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %4) ]
+ %6 = load i32, ptr %k, align 4
+ %inc = add nsw i32 %6, 1
+ store i32 %inc, ptr %k, align 4
+ br label %do.cond
+
+do.cond: ; preds = %do.body2
+ %7 = load i32, ptr %k, align 4
+ %cmp = icmp slt i32 %7, 30
+ br i1 %cmp, label %do.body2, label %do.end
+
+do.end: ; preds = %do.cond
+ %8 = load i32, ptr %j, align 4
+ %inc3 = add nsw i32 %8, 1
+ store i32 %inc3, ptr %j, align 4
+ br label %do.cond4
+
+do.cond4: ; preds = %do.end
+ %9 = load i32, ptr %j, align 4
+ %cmp5 = icmp slt i32 %9, 20
+ br i1 %cmp5, label %do.body1, label %do.end6
+
+do.end6: ; preds = %do.cond4
+ %10 = load i32, ptr %i, align 4
+ %inc7 = add nsw i32 %10, 1
+ store i32 %inc7, ptr %i, align 4
+ br label %do.cond8
+
+do.cond8: ; preds = %do.end6
+ %11 = load i32, ptr %i, align 4
+ %cmp9 = icmp slt i32 %11, 10
+ br i1 %cmp9, label %do.body, label %do.end10
+
+do.end10: ; preds = %do.cond8
+ %12 = load i32, ptr %val, align 4
+ %13 = load i32, ptr %i, align 4
+ %add11 = add nsw i32 %12, %13
+ %14 = load i32, ptr %j, align 4
+ %add12 = add nsw i32 %add11, %14
+ %15 = load i32, ptr %k, align 4
+ %add13 = add nsw i32 %add12, %15
+ ret i32 %add13
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal spir_func void @main() #2 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %call1 = call spir_func noundef i32 @_Z7processv() #4 [ "convergencectrl"(token %0) ]
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define void @main.1() #3 {
+entry:
+ call void @main()
+ ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #4 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
+
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll
new file mode 100644
index 00000000000000..f2e60f916c795b
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.break.ll
@@ -0,0 +1,178 @@
+; RUN: %if spirv-tools %{ llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | spirv-sim --function=_Z7processv --wave=1 --expects=4
+;
+; Source HLSL:
+;
+; int process() {
+; int val = 0;
+;
+; for (int i = 0; i < 10; ++i) {
+; if (i < 5) {
+; break;
+; }
+; val = i;
+; {
+; break;
+; }
+; break; // No SPIR-V should be emitted for this statement.
+; val++; // No SPIR-V should be emitted for this statement.
+; while(true); // No SPIR-V should be emitted for this statement.
+; }
+;
+; ////////////////////////////////////////////////////////////////////////////////
+; // Nested for loops with break statements //
+; // Each break statement should branch to the corresponding loop's break block //
+; ////////////////////////////////////////////////////////////////////////////////
+; for (int j = 0; j < 10; ++j) {
+; val = j+5;
+; for ( ; val < 20; ++val) {
+; int k = val + j;
+; {
+; {
+; break;
+; }
+; }
+; k++; // No SPIR-V should be emitted for this statement.
+; }
+;
+; // FIXME: val--; generates add nsw i32 ..., -1
+; // This is compiled into a OpIAdd %val, MAX_INT.
+; val -= 1;
+; break;
+; break; // No SPIR-V should be emitted for this statement.
+; val = val*10; // No SPIR-V should be emitted for this statement.
+; }
+;
+; return val;
+; }
+;
+; [numthreads(1, 1, 1)]
+; void main() {
+; process();
+; }
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @_Z7processv() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ %j = alloca i32, align 4
+ %k = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %for.cond
+
+for.cond: ; preds = %for.inc, %entry
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %2, 10
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %3 = load i32, ptr %i, align 4
+ %cmp1 = icmp slt i32 %3, 5
+ br i1 %cmp1, label %if.then, label %if.end
+
+if.then: ; preds = %for.body
+ br label %for.end
+
+if.end: ; preds = %for.body
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %for.end
+
+for.inc: ; No predecessors!
+ %5 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %5, 1
+ store i32 %inc, ptr %i, align 4
+ br label %for.cond
+
+for.end: ; preds = %if.end, %if.then, %for.cond
+ store i32 0, ptr %j, align 4
+ br label %for.cond2
+
+for.cond2: ; preds = %for.inc12, %for.end
+ %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %7 = load i32, ptr %j, align 4
+ %cmp3 = icmp slt i32 %7, 10
+ br i1 %cmp3, label %for.body4, label %for.end14
+
+for.body4: ; preds = %for.cond2
+ %8 = load i32, ptr %j, align 4
+ %add = add nsw i32 %8, 5
+ store i32 %add, ptr %val, align 4
+ br label %for.cond5
+
+for.cond5: ; preds = %for.inc9, %for.body4
+ %9 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
+ %10 = load i32, ptr %val, align 4
+ %cmp6 = icmp slt i32 %10, 20
+ br i1 %cmp6, label %for.body7, label %for.end11
+
+for.body7: ; preds = %for.cond5
+ %11 = load i32, ptr %val, align 4
+ %12 = load i32, ptr %j, align 4
+ %add8 = add nsw i32 %11, %12
+ store i32 %add8, ptr %k, align 4
+ br label %for.end11
+
+for.inc9: ; No predecessors!
+ %13 = load i32, ptr %val, align 4
+ %inc10 = add nsw i32 %13, 1
+ store i32 %inc10, ptr %val, align 4
+ br label %for.cond5
+
+for.end11: ; preds = %for.body7, %for.cond5
+ %14 = load i32, ptr %val, align 4
+ %sub = sub nsw i32 %14, 1
+ store i32 %sub, ptr %val, align 4
+ br label %for.end14
+
+for.inc12: ; No predecessors!
+ %15 = load i32, ptr %j, align 4
+ %inc13 = add nsw i32 %15, 1
+ store i32 %inc13, ptr %j, align 4
+ br label %for.cond2
+
+for.end14: ; preds = %for.end11, %for.cond2
+ %16 = load i32, ptr %val, align 4
+ ret i32 %16
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal spir_func void @main() #2 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %call1 = call spir_func noundef i32 @_Z7processv() #4 [ "convergencectrl"(token %0) ]
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define void @main.1() #3 {
+entry:
+ call void @main()
+ ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #4 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
+
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.hlsl
new file mode 100644
index 00000000000000..b6b5f64ea21d7e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.hlsl
@@ -0,0 +1,47 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=19
+
+int process() {
+ int val = 0;
+
+ for (int i = 0; i < 10; ++i) {
+ if (i < 5) {
+ continue;
+ }
+ val = i;
+
+ {
+ continue;
+ }
+ val++; // No SPIR-V should be emitted for this statement.
+ continue; // No SPIR-V should be emitted for this statement.
+ while(true); // No SPIR-V should be emitted for this statement.
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////
+ // Nested for loops with continue statements //
+ // Each continue statement should branch to the corresponding loop's continue block //
+ //////////////////////////////////////////////////////////////////////////////////////
+
+ for (int j = 0; j < 10; ++j) {
+ val = j+5;
+
+ for ( ; val < 20; ++val) {
+ int k = val + j;
+ continue;
+ k++; // No SPIR-V should be emitted for this statement.
+ }
+
+ val -= 1;
+ continue;
+ continue; // No SPIR-V should be emitted for this statement.
+ val = val*10; // No SPIR-V should be emitted for this statement.
+ }
+
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.hlsl
new file mode 100644
index 00000000000000..ac8a1343c40be1
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.hlsl
@@ -0,0 +1,25 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=2563170
+
+int process() {
+ int val = 0;
+
+ for (int i = 0; i < 10; ++i) {
+ val = val + i;
+
+ for (int j = 0; j < 2; ++j) {
+ for (int k = 0; k < 2; ++k) {
+ val = val + k;
+ }
+
+ val = val * 2;
+
+ }
+ }
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll
new file mode 100644
index 00000000000000..2670cf36ea83a9
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll
@@ -0,0 +1,105 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; CHECK-DAG: OpName %[[#process:]] "_Z7processv"
+; CHECK-DAG: OpName %[[#val:]] "val"
+; CHECK-DAG: OpName %[[#i:]] "i"
+
+; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#int_pfty:]] = OpTypePointer Function %[[#int_ty]]
+; CHECK-DAG: %[[#false:]] = OpConstantFalse
+
+
+; CHECK: %[[#process]] = OpFunction %[[#]] DontInline %[[#]]
+define spir_func noundef i32 @_Z7processv() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+
+ ; CHECK-DAG: %[[#val]] = OpVariable %[[#int_pfty]] Function
+ ; CHECK-DAG: %[[#i]] = OpVariable %[[#int_pfty]] Function
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %for.cond
+ ; CHECK: OpBranch %[[#for_cond:]]
+
+for.cond: ; preds = %for.inc, %entry
+ ; CHECK: %[[#for_cond]] = OpLabel
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %2, 10
+
+ ; CHECK: OpLoopMerge %[[#for_end:]] %[[#for_inc:]]
+ ; CHECK: OpBranchConditional %[[#]] %[[#for_body:]] %[[#for_end]]
+ br i1 %cmp, label %for.body, label %for.end
+
+ ; CHECK: %[[#for_end]] = OpLabel
+ ; CHECK: OpBranch %[[#for_cond1:]]
+for.end: ; preds = %for.cond
+ br label %for.cond1
+
+ ; CHECK: %[[#for_cond1]] = OpLabel
+ ; CHECK: OpLoopMerge %[[#unreachable:]] %[[#for_cond1]]
+ ; CHECK: OpBranchConditional %[[#false]] %[[#unreachable]] %[[#for_cond1]]
+for.cond1: ; preds = %for.cond1, %for.end
+ %3 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ store i32 0, ptr %val, align 4
+ br label %for.cond1
+
+ ; CHECK: %[[#unreachable]] = OpLabel
+ ; CHECK-NEXT: OpUnreachable
+
+ ; CHECK: %[[#for_body]] = OpLabel
+ ; CHECK: OpBranch %[[#for_inc]]
+for.body: ; preds = %for.cond
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %for.inc
+
+ ; CHECK: %[[#for_inc]] = OpLabel
+ ; CHECK: OpBranch %[[#for_cond]]
+for.inc: ; preds = %for.body
+ %5 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %5, 1
+ store i32 %inc, ptr %i, align 4
+ br label %for.cond
+
+
+}
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #1
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.loop() #1
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal spir_func void @main() #2 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %call1 = call spir_func noundef i32 @_Z7processv() #4 [ "convergencectrl"(token %0) ]
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define void @main.1() #3 {
+entry:
+ call void @main()
+ ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #4 = { convergent }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.hlsl
new file mode 100644
index 00000000000000..2279787cfb6b6d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.hlsl
@@ -0,0 +1,42 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=9
+
+int process() {
+ int a = 0;
+ int b = 1;
+ int val = 0;
+
+ for (int i = 0; a && b; ++i) {
+ val += 1;
+ }
+
+ for (int i = 0; a || b; ++i) {
+ val += 1;
+ b = 0;
+ }
+
+ b = 1;
+ for (int i = 0; a && ((a || b) && b); ++i) {
+ val += 4;
+ b = 0;
+ }
+
+ b = 1;
+ for (int i = 0; a ? a : b; ++i) {
+ val += 8;
+ b = 0;
+ }
+
+ int x = 0;
+ int y = 0;
+ for (int i = 0; x + (x && y); ++i) {
+ val += 16;
+ }
+
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.hlsl
new file mode 100644
index 00000000000000..9bae981e0cb418
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.const-cond.hlsl
@@ -0,0 +1,33 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=3
+
+int process() {
+ int a = 0;
+ int b = 0;
+
+ if (3 + 5) {
+ a = 1;
+ } else {
+ a = 0;
+ }
+
+ if (4 + 3 > 7 || 4 + 3 < 8) {
+ b = 2;
+ }
+
+ if (4 + 3 > 7 && true) {
+ b = 0;
+ }
+
+ if (true)
+ ;
+
+ if (false) {}
+
+ return a + b;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.hlsl
new file mode 100644
index 00000000000000..dd66c24111ebfd
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.hlsl
@@ -0,0 +1,46 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=6
+
+int process() {
+ int color = 0;
+
+ int val = 0;
+
+ if (color < 0) {
+ val = 1;
+ }
+
+ // for-stmt following if-stmt
+ for (int i = 0; i < 10; ++i) {
+ if (color < 0) { // if-stmt nested in for-stmt
+ val = val + 1;
+ for (int j = 0; j < 15; ++j) { // for-stmt deeply nested in if-then
+ val = val * 2;
+ } // end for (int j
+ val = val + 3;
+ }
+
+ if (color < 1) { // if-stmt following if-stmt
+ val = val * 4;
+ } else {
+ for (int k = 0; k < 20; ++k) { // for-stmt deeply nested in if-else
+ val = val - 5;
+ if (val < 0) { // deeply nested if-stmt
+ val = val + 100;
+ }
+ } // end for (int k
+ } // end elsek
+ } // end for (int i
+
+ // if-stmt following for-stmt
+ if (color < 2) {
+ val = val + 6;
+ }
+
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.hlsl
new file mode 100644
index 00000000000000..ccb579d9d5c8ce
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.hlsl
@@ -0,0 +1,29 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=3
+
+int process() {
+ int c1 = 0;
+ int c2 = 1;
+ int c3 = 0;
+ int c4 = 1;
+ int val = 0;
+
+ if (c1) {
+ if (c2)
+ val = 1;
+ } else {
+ if (c3) {
+ val = 2;
+ } else {
+ if (c4) {
+ val = 3;
+ }
+ }
+ }
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.hlsl
new file mode 100644
index 00000000000000..75e7e0b8452054
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.plain.hlsl
@@ -0,0 +1,39 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=2
+
+int process() {
+ int c = 0;
+ int val = 0;
+
+ // Both then and else
+ if (c) {
+ val = val + 1;
+ } else {
+ val = val + 2;
+ }
+
+ // No else
+ if (c)
+ val = 1;
+
+ // Empty then
+ if (c) {
+ } else {
+ val = 2;
+ }
+
+ // Null body
+ if (c)
+ ;
+
+ if (int d = val) {
+ c = true;
+ }
+
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.hlsl
new file mode 100644
index 00000000000000..4564e9d22bb44f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-and.hlsl
@@ -0,0 +1,27 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=1
+
+int fn() { return true; }
+
+int process() {
+ // Use in control flow
+ int a = 0;
+ int b = 0;
+ int val = 0;
+ if (a && b) val++;
+
+ // Operand with side effects
+ if (fn() && fn()) val++;
+
+ if (a && fn())
+ val++;
+
+ if (fn() && b)
+ val++;
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.hlsl
new file mode 100644
index 00000000000000..9dcdcc97dd040c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.logical-or.hlsl
@@ -0,0 +1,24 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=3
+
+int fn() { return true; }
+
+int process() {
+ int a = 0;
+ int b = 0;
+ int val = 0;
+
+ // Use in control flow
+ if (a || b) val++;
+
+ // Operand with side effects
+ if (fn() || fn()) val++;
+ if (a || fn()) val++;
+ if (fn() || b) val++;
+ return val;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.hlsl
new file mode 100644
index 00000000000000..b2bb5f90b110da
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.hlsl
@@ -0,0 +1,58 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=0
+
+int process() {
+ int cond = 1;
+ int value = 0;
+
+ while(value < 10) {
+ switch(value) {
+ case 1:
+ value = 1;
+ return value;
+ case 2: {
+ value = 3;
+ {return value;} // Return from function.
+ value = 4; // No SPIR-V should be emitted for this statement.
+ break; // No SPIR-V should be emitted for this statement.
+ }
+ case 5 : {
+ value = 5;
+ {{return value;}} // Return from function.
+ value = 6; // No SPIR-V should be emitted for this statement.
+ }
+ default:
+ for (int i=0; i<10; ++i) {
+ if (cond) {
+ return value; // Return from function.
+ return value; // No SPIR-V should be emitted for this statement.
+ continue; // No SPIR-V should be emitted for this statement.
+ break; // No SPIR-V should be emitted for this statement.
+ ++value; // No SPIR-V should be emitted for this statement.
+ } else {
+ return value; // Return from function
+ continue; // No SPIR-V should be emitted for this statement.
+ break; // No SPIR-V should be emitted for this statement.
+ ++value; // No SPIR-V should be emitted for this statement.
+ }
+ }
+
+ // Return from function.
+ // Even though this statement will never be executed [because both "if" and "else" above have return statements],
+ // SPIR-V code should be emitted for it as we do not analyze the logic.
+ return value;
+ }
+
+ // Return from function.
+ // Even though this statement will never be executed [because all "case" statements above contain a return statement],
+ // SPIR-V code should be emitted for it as we do not analyze the logic.
+ return value;
+ }
+
+ return value;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.simple.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.simple.hlsl
new file mode 100644
index 00000000000000..206a31beacb88a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.simple.hlsl
@@ -0,0 +1,20 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+[numthreads(1, 1, 1)]
+void main() {
+ int a, b;
+ int cond = 1;
+
+ while(cond) {
+ switch(b) {
+ default:
+ if (cond) {
+ if (cond)
+ return;
+ else
+ return;
+ }
+ }
+ return;
+ }
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.return.void.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.void.hlsl
new file mode 100644
index 00000000000000..c2132ca63d937d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.void.hlsl
@@ -0,0 +1,14 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+void A() {
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ // CHECK: [[type:%[0-9]+]] = OpTypeFunction %void
+ // CHECK: %src_main = OpFunction %void None [[type]]
+ // CHECK: {{%[0-9]+}} = OpFunctionCall %void %A
+ // CHECK: OpReturn
+ // CHECK: OpFunctionEnd
+ return A();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.hlsl
new file mode 100644
index 00000000000000..95c93da9942ab6
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.hlsl
@@ -0,0 +1,122 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=308
+
+int foo() { return 200; }
+
+int process() {
+ int a = 0;
+ int b = 0;
+ int c = 0;
+ const int r = 20;
+ const int s = 40;
+ const int t = 3*r+2*s;
+
+
+ ////////////////////////////////////////
+ // DefaultStmt is the first statement //
+ ////////////////////////////////////////
+ switch(a) {
+ default:
+ b += 0;
+ case 1:
+ b += 1;
+ break;
+ case 2:
+ b += 2;
+ }
+
+
+ //////////////////////////////////////////////
+ // DefaultStmt in the middle of other cases //
+ //////////////////////////////////////////////
+ switch(a) {
+ case 10:
+ b += 1;
+ default:
+ b += 0;
+ case 20:
+ b += 2;
+ break;
+ }
+
+ ///////////////////////////////////////////////
+ // Various CaseStmt and BreakStmt topologies //
+ // DefaultStmt is the last statement //
+ ///////////////////////////////////////////////
+ switch(int d = 5) {
+ case 1:
+ b += 1;
+ c += foo();
+ case 2:
+ b += 2;
+ break;
+ case 3:
+ {
+ b += 3;
+ break;
+ }
+ case t:
+ b += t;
+ case 4:
+ case 5:
+ b += 5;
+ break;
+ case 6: {
+ case 7:
+ break;}
+ default:
+ break;
+ }
+
+
+ //////////////////////////
+ // No Default statement //
+ //////////////////////////
+ switch(a) {
+ case 100:
+ b += 100;
+ break;
+ }
+
+
+ /////////////////////////////////////////////////////////
+ // No cases. Only a default //
+ // This means the default body will always be executed //
+ /////////////////////////////////////////////////////////
+ switch(a) {
+ default:
+ b += 100;
+ c += 200;
+ break;
+ }
+
+
+ ////////////////////////////////////////////////////////////
+ // Nested Switch with branching //
+ // The two inner switch statements should be executed for //
+ // both cases of the outer switch (case 300 and case 400) //
+ ////////////////////////////////////////////////////////////
+ switch(a) {
+ case 300:
+ b += 300;
+ case 400:
+ switch(c) {
+ case 500:
+ b += 500;
+ break;
+ case 600:
+ switch(b) {
+ default:
+ a += 600;
+ b += 600;
+ }
+ }
+ }
+
+ return a + b + c;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.hlsl
new file mode 100644
index 00000000000000..9f5e3190d163bd
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple.hlsl
@@ -0,0 +1,25 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=5
+
+int process() {
+ int b = 0;
+ const int t = 50;
+
+ switch(int d = 5) {
+ case t:
+ b = t;
+ case 4:
+ case 5:
+ b = 5;
+ break;
+ default:
+ break;
+ }
+
+ return b;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.hlsl
new file mode 100644
index 00000000000000..a37e78f4c0cf54
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.hlsl
@@ -0,0 +1,45 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-sim --function=_Z7processv --wave=1 --expects=5
+
+int foo() { return 200; }
+
+int process() {
+ int a = 0;
+ int b = 0;
+ int c = 0;
+ const int r = 20;
+ const int s = 40;
+ const int t = 3*r+2*s;
+
+ switch(int d = 5) {
+ case 1:
+ b += 1;
+ c += foo();
+ case 2:
+ b += 2;
+ break;
+ case 3:
+ {
+ b += 3;
+ break;
+ }
+ case t:
+ b += t;
+ case 4:
+ case 5:
+ b += 5;
+ break;
+ case 6: {
+ case 7:
+ break;}
+ default:
+ break;
+ }
+
+ return a + b + c;
+}
+
+[numthreads(1, 1, 1)]
+void main() {
+ process();
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl
new file mode 100644
index 00000000000000..19d889064077cd
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl
@@ -0,0 +1,360 @@
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val
+
+int foo() { return 200; }
+
+[numthreads(1, 1, 1)]
+void main() {
+ int result;
+
+
+
+ ////////////////////////////
+ // The most basic case //
+ // Has a 'default' case //
+ // All cases have 'break' //
+ ////////////////////////////
+
+ int a = 0;
+// CHECK: [[a:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge None
+// CHECK-NEXT: OpSwitch [[a]] %switch_default -3 %switch_n3 0 %switch_0 1 %switch_1 2 %switch_2
+ switch(a) {
+// CHECK-NEXT: %switch_n3 = OpLabel
+// CHECK-NEXT: OpStore %result %int_n300
+// CHECK-NEXT: OpBranch %switch_merge
+ case -3:
+ result = -300;
+ break;
+// CHECK-NEXT: %switch_0 = OpLabel
+// CHECK-NEXT: OpStore %result %int_0
+// CHECK-NEXT: OpBranch %switch_merge
+ case 0:
+ result = 0;
+ break;
+// CHECK-NEXT: %switch_1 = OpLabel
+// CHECK-NEXT: OpStore %result %int_100
+// CHECK-NEXT: OpBranch %switch_merge
+ case 1:
+ result = 100;
+ break;
+// CHECK-NEXT: %switch_2 = OpLabel
+// CHECK-NEXT: [[foo:%[0-9]+]] = OpFunctionCall %int %foo
+// CHECK-NEXT: OpStore %result [[foo]]
+// CHECK-NEXT: OpBranch %switch_merge
+ case 2:
+ result = foo();
+ break;
+// CHECK-NEXT: %switch_default = OpLabel
+// CHECK-NEXT: OpStore %result %int_777
+// CHECK-NEXT: OpBranch %switch_merge
+ default:
+ result = 777;
+ break;
+ }
+// CHECK-NEXT: %switch_merge = OpLabel
+
+
+
+ ////////////////////////////////////
+ // The selector is a statement //
+ // Does not have a 'default' case //
+ // All cases have 'break' //
+ ////////////////////////////////////
+
+// CHECK-NEXT: [[a1:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpStore %c [[a1]]
+// CHECK-NEXT: [[c:%[0-9]+]] = OpLoad %int %c
+// CHECK-NEXT: OpSelectionMerge %switch_merge_0 None
+// CHECK-NEXT: OpSwitch [[c]] %switch_merge_0 -4 %switch_n4 4 %switch_4
+ switch(int c = a) {
+// CHECK-NEXT: %switch_n4 = OpLabel
+// CHECK-NEXT: OpStore %result %int_n400
+// CHECK-NEXT: OpBranch %switch_merge_0
+ case -4:
+ result = -400;
+ break;
+// CHECK-NEXT: %switch_4 = OpLabel
+// CHECK-NEXT: OpStore %result %int_400
+// CHECK-NEXT: OpBranch %switch_merge_0
+ case 4:
+ result = 400;
+ break;
+ }
+// CHECK-NEXT: %switch_merge_0 = OpLabel
+
+
+
+ ///////////////////////////////////
+ // All cases are fall-through //
+ // The last case is fall-through //
+ ///////////////////////////////////
+
+// CHECK-NEXT: [[a2:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge_1 None
+// CHECK-NEXT: OpSwitch [[a2]] %switch_merge_1 -5 %switch_n5 5 %switch_5
+ switch(a) {
+// CHECK-NEXT: %switch_n5 = OpLabel
+// CHECK-NEXT: OpStore %result %int_n500
+// CHECK-NEXT: OpBranch %switch_5
+ case -5:
+ result = -500;
+// CHECK-NEXT: %switch_5 = OpLabel
+// CHECK-NEXT: OpStore %result %int_500
+// CHECK-NEXT: OpBranch %switch_merge_1
+ case 5:
+ result = 500;
+ }
+// CHECK-NEXT: %switch_merge_1 = OpLabel
+
+
+
+ ///////////////////////////////////////
+ // Some cases are fall-through //
+ // The last case is not fall-through //
+ ///////////////////////////////////////
+
+// CHECK-NEXT: [[a3:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge_2 None
+// CHECK-NEXT: OpSwitch [[a3]] %switch_default_0 6 %switch_6 7 %switch_7 8 %switch_8
+ switch(a) {
+// CHECK-NEXT: %switch_6 = OpLabel
+// CHECK-NEXT: OpStore %result %int_600
+// CHECK-NEXT: OpBranch %switch_7
+ case 6:
+ result = 600;
+ case 7:
+// CHECK-NEXT: %switch_7 = OpLabel
+// CHECK-NEXT: OpStore %result %int_700
+// CHECK-NEXT: OpBranch %switch_8
+ result = 700;
+// CHECK-NEXT: %switch_8 = OpLabel
+// CHECK-NEXT: OpStore %result %int_800
+// CHECK-NEXT: OpBranch %switch_merge_2
+ case 8:
+ result = 800;
+ break;
+// CHECK-NEXT: %switch_default_0 = OpLabel
+// CHECK-NEXT: OpStore %result %int_777
+// CHECK-NEXT: OpBranch %switch_merge_2
+ default:
+ result = 777;
+ break;
+ }
+// CHECK-NEXT: %switch_merge_2 = OpLabel
+
+
+
+ ///////////////////////////////////////
+ // Fall-through cases with no body //
+ ///////////////////////////////////////
+
+// CHECK-NEXT: [[a4:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge_3 None
+// CHECK-NEXT: OpSwitch [[a4]] %switch_default_1 10 %switch_10 11 %switch_11 12 %switch_12
+ switch(a) {
+// CHECK-NEXT: %switch_10 = OpLabel
+// CHECK-NEXT: OpBranch %switch_11
+ case 10:
+// CHECK-NEXT: %switch_11 = OpLabel
+// CHECK-NEXT: OpBranch %switch_default_1
+ case 11:
+// CHECK-NEXT: %switch_default_1 = OpLabel
+// CHECK-NEXT: OpBranch %switch_12
+ default:
+// CHECK-NEXT: %switch_12 = OpLabel
+// CHECK-NEXT: OpStore %result %int_12
+// CHECK-NEXT: OpBranch %switch_merge_3
+ case 12:
+ result = 12;
+ }
+// CHECK-NEXT: %switch_merge_3 = OpLabel
+
+
+
+ ////////////////////////////////////////////////
+ // No-op. Two nested cases and a nested break //
+ ////////////////////////////////////////////////
+
+// CHECK-NEXT: [[a5:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge_4 None
+// CHECK-NEXT: OpSwitch [[a5]] %switch_merge_4 15 %switch_15 16 %switch_16
+ switch(a) {
+// CHECK-NEXT: %switch_15 = OpLabel
+// CHECK-NEXT: OpBranch %switch_16
+ case 15:
+// CHECK-NEXT: %switch_16 = OpLabel
+// CHECK-NEXT: OpBranch %switch_merge_4
+ case 16:
+ break;
+ }
+// CHECK-NEXT: %switch_merge_4 = OpLabel
+
+
+
+ ////////////////////////////////////////////////////////////////
+ // Using braces (compound statements) in various parts //
+ // Using breaks such that each AST configuration is different //
+ // Also uses 'forcecase' attribute //
+ ////////////////////////////////////////////////////////////////
+
+// CHECK-NEXT: [[a6:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge_5 None
+// CHECK-NEXT: OpSwitch [[a6]] %switch_merge_5 20 %switch_20 21 %switch_21 22 %switch_22 23 %switch_23 24 %switch_24 25 %switch_25 26 %switch_26 27 %switch_27 28 %switch_28 29 %switch_29
+ switch(a) {
+// CHECK-NEXT: %switch_20 = OpLabel
+// CHECK-NEXT: OpStore %result %int_20
+// CHECK-NEXT: OpBranch %switch_merge_5
+ case 20: {
+ result = 20;
+ break;
+ }
+// CHECK-NEXT: %switch_21 = OpLabel
+// CHECK-NEXT: OpStore %result %int_21
+// CHECK-NEXT: OpBranch %switch_merge_5
+ case 21:
+ result = 21;
+ break;
+// CHECK-NEXT: %switch_22 = OpLabel
+// CHECK-NEXT: OpBranch %switch_23
+// CHECK-NEXT: %switch_23 = OpLabel
+// CHECK-NEXT: OpBranch %switch_merge_5
+ case 22:
+ case 23:
+ break;
+// CHECK-NEXT: %switch_24 = OpLabel
+// CHECK-NEXT: OpBranch %switch_25
+// CHECK-NEXT: %switch_25 = OpLabel
+// CHECK-NEXT: OpStore %result %int_25
+// CHECK-NEXT: OpBranch %switch_merge_5
+ case 24:
+ case 25: { result = 25; }
+ break;
+// CHECK-NEXT: %switch_26 = OpLabel
+// CHECK-NEXT: OpBranch %switch_27
+// CHECK-NEXT: %switch_27 = OpLabel
+// CHECK-NEXT: OpBranch %switch_merge_5
+ case 26:
+ case 27: {
+ break;
+ }
+// CHECK-NEXT: %switch_28 = OpLabel
+// CHECK-NEXT: OpStore %result %int_28
+// CHECK-NEXT: OpBranch %switch_merge_5
+ case 28: {
+ result = 28;
+ {{break;}}
+ }
+// CHECK-NEXT: %switch_29 = OpLabel
+// CHECK-NEXT: OpStore %result %int_29
+// CHECK-NEXT: OpBranch %switch_merge_5
+ case 29: {
+ {
+ result = 29;
+ {break;}
+ }
+ }
+ }
+// CHECK-NEXT: %switch_merge_5 = OpLabel
+
+
+
+ ////////////////////////////////////////////////////////////////////////
+ // Nested Switch statements with mixed use of fall-through and braces //
+ ////////////////////////////////////////////////////////////////////////
+
+// CHECK-NEXT: [[a7:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge_7 None
+// CHECK-NEXT: OpSwitch [[a7]] %switch_merge_7 30 %switch_30
+ switch(a) {
+// CHECK-NEXT: %switch_30 = OpLabel
+ case 30: {
+// CHECK-NEXT: OpStore %result %int_30
+ result = 30;
+// CHECK-NEXT: [[result:%[0-9]+]] = OpLoad %int %result
+// CHECK-NEXT: OpSelectionMerge %switch_merge_6 None
+// CHECK-NEXT: OpSwitch [[result]] %switch_default_2 50 %switch_50 51 %switch_51 52 %switch_52 53 %switch_53 54 %switch_54
+ switch(result) {
+// CHECK-NEXT: %switch_default_2 = OpLabel
+// CHECK-NEXT: OpStore %a %int_55
+// CHECK-NEXT: OpBranch %switch_50
+ default:
+ a = 55;
+// CHECK-NEXT: %switch_50 = OpLabel
+// CHECK-NEXT: OpStore %a %int_50
+// CHECK-NEXT: OpBranch %switch_merge_6
+ case 50:
+ a = 50;
+ break;
+// CHECK-NEXT: %switch_51 = OpLabel
+// CHECK-NEXT: OpBranch %switch_52
+ case 51:
+// CHECK-NEXT: %switch_52 = OpLabel
+// CHECK-NEXT: OpStore %a %int_52
+// CHECK-NEXT: OpBranch %switch_53
+ case 52:
+ a = 52;
+// CHECK-NEXT: %switch_53 = OpLabel
+// CHECK-NEXT: OpStore %a %int_53
+// CHECK-NEXT: OpBranch %switch_merge_6
+ case 53:
+ a = 53;
+ break;
+// CHECK-NEXT: %switch_54 = OpLabel
+// CHECK-NEXT: OpStore %a %int_54
+// CHECK-NEXT: OpBranch %switch_merge_6
+ case 54 : {
+ a = 54;
+ break;
+ }
+ }
+// CHECK-NEXT: %switch_merge_6 = OpLabel
+// CHECK-NEXT: OpBranch %switch_merge_7
+ }
+ }
+// CHECK-NEXT: %switch_merge_7 = OpLabel
+
+
+
+ ///////////////////////////////////////////////
+ // Constant integer variables as case values //
+ ///////////////////////////////////////////////
+
+ const int r = 35;
+ const int s = 45;
+ const int t = 2*r + s; // evaluates to 115.
+
+// CHECK: [[a8:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpSelectionMerge %switch_merge_8 None
+// CHECK-NEXT: OpSwitch [[a8]] %switch_merge_8 35 %switch_35 115 %switch_115
+ switch(a) {
+// CHECK-NEXT: %switch_35 = OpLabel
+// CHECK-NEXT: [[r:%[0-9]+]] = OpLoad %int %r
+// CHECK-NEXT: OpStore %result [[r]]
+// CHECK-NEXT: OpBranch %switch_115
+ case r:
+ result = r;
+// CHECK-NEXT: %switch_115 = OpLabel
+// CHECK-NEXT: [[t:%[0-9]+]] = OpLoad %int %t
+// CHECK-NEXT: OpStore %result [[t]]
+// CHECK-NEXT: OpBranch %switch_merge_8
+ case t:
+ result = t;
+ break;
+// CHECK-NEXT: %switch_merge_8 = OpLabel
+ }
+
+
+ //////////////////////////////////////////////////////////////////
+ // Using float as selector results in multiple casts in the AST //
+ //////////////////////////////////////////////////////////////////
+ float sel;
+// CHECK: [[floatSelector:%[0-9]+]] = OpLoad %float %sel
+// CHECK-NEXT: [[sel:%[0-9]+]] = OpConvertFToS %int [[floatSelector]]
+// CHECK-NEXT: OpSelectionMerge %switch_merge_9 None
+// CHECK-NEXT: OpSwitch [[sel]] %switch_merge_9 0 %switch_0_0
+ switch ((int)sel) {
+ case 0:
+ result = 0;
+ break;
+ }
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl
new file mode 100644
index 00000000000000..09f8c2caae5f6f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl
@@ -0,0 +1,36 @@
+// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val
+
+[numthreads(1, 1, 1)]
+void main() {
+///////////////////////////////
+// 32-bit int literal switch //
+///////////////////////////////
+// CHECK: OpSelectionMerge %switch_merge None
+// CHECK: OpSwitch %int_0 %switch_default 0 %switch_0
+ switch (0) {
+// CHECK: %switch_0 = OpLabel
+ case 0:
+// CHECK: OpReturnValue %float_1
+ return;
+// CHECK: %switch_default = OpLabel
+ default:
+// CHECK: OpReturnValue %float_2
+ return;
+ }
+// CHECK: %switch_merge = OpLabel
+
+///////////////////////////////
+// 64-bit int literal switch //
+///////////////////////////////
+// CHECK: OpSelectionMerge %switch_merge_0 None
+// CHECK: OpSwitch %long_12345678910 %switch_merge_0 12345678910 %switch_12345678910
+ switch (12345678910) {
+// CHECK: %switch_12345678910 = OpLabel
+ case 12345678910:
+// CHECK: OpReturnValue %float_1
+ return;
+ }
+// CHECK: %switch_merge_0 = OpLabel
+
+ return;
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.simple.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.simple.hlsl
new file mode 100644
index 00000000000000..3bc8d786390ddb
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.simple.hlsl
@@ -0,0 +1,36 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+int foo() { return 200; }
+
+[numthreads(1, 1, 1)]
+void main() {
+ int result;
+
+ int a = 0;
+ switch(a) {
+ case -3:
+ result = -300;
+ break;
+ case 0:
+ result = 0;
+ break;
+ case 1:
+ result = 100;
+ break;
+ case 2:
+ result = foo();
+ break;
+ default:
+ result = 777;
+ break;
+ }
+
+ switch(int c = a) {
+ case -4:
+ result = -400;
+ break;
+ case 4:
+ result = 400;
+ break;
+ }
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.hlsl
new file mode 100644
index 00000000000000..612e3050831715
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.break.hlsl
@@ -0,0 +1,87 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+int foo() { return true; }
+
+[numthreads(1, 1, 1)]
+void main() {
+ int val = 0;
+ int i = 0;
+
+
+// CHECK: OpBranch %while_check
+// CHECK: %while_check = OpLabel
+// CHECK: [[i_lt_10:%[0-9]+]] = OpSLessThan %bool {{%[0-9]+}} %int_10
+// CHECK-NEXT: OpLoopMerge %while_merge %while_continue None
+// CHECK-NEXT: OpBranchConditional [[i_lt_10]] %while_body %while_merge
+ while (i < 10) {
+// CHECK-NEXT: %while_body = OpLabel
+ val = i;
+// CHECK: [[val_gt_5:%[0-9]+]] = OpSGreaterThan %bool {{%[0-9]+}} %int_5
+// CHECK-NEXT: OpSelectionMerge %if_merge None
+// CHECK-NEXT: OpBranchConditional [[val_gt_5]] %if_true %if_merge
+ if (val > 5) {
+// CHECK-NEXT: %if_true = OpLabel
+// CHECK-NEXT: OpBranch %while_merge
+ break;
+ }
+// CHECK-NEXT: %if_merge = OpLabel
+
+// CHECK: [[val_gt_6:%[0-9]+]] = OpSGreaterThan %bool {{%[0-9]+}} %int_6
+// CHECK-NEXT: OpSelectionMerge %if_merge_0 None
+// CHECK-NEXT: OpBranchConditional [[val_gt_6]] %if_true_0 %if_merge_0
+ if (val > 6) {
+// CHECK-NEXT: %if_true_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_merge
+ break;
+ break; // No SPIR-V should be emitted for this statement.
+ val++; // No SPIR-V should be emitted for this statement.
+ while(true); // No SPIR-V should be emitted for this statement.
+ --i; // No SPIR-V should be emitted for this statement.
+ }
+// CHECK-NEXT: %if_merge_0 = OpLabel
+
+// CHECK-NEXT: OpBranch %while_continue
+// CHECK-NEXT: %while_continue = OpLabel
+// CHECK-NEXT: OpBranch %while_check
+ }
+
+// CHECK-NEXT: %while_merge = OpLabel
+
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Nested while loops with break statements //
+ // Each break statement should branch to the corresponding loop's break block //
+ ////////////////////////////////////////////////////////////////////////////////
+
+// CHECK-NEXT: OpBranch %while_check_0
+// CHECK-NEXT: %while_check_0 = OpLabel
+// CHECK-NEXT: OpLoopMerge %while_merge_1 %while_continue_1 None
+// CHECK-NEXT: OpBranchConditional %true %while_body_0 %while_merge_1
+ while (true) {
+// CHECK-NEXT: %while_body_0 = OpLabel
+ i++;
+
+// CHECK: OpBranch %while_check_1
+// CHECK-NEXT: %while_check_1 = OpLabel
+// CHECK: [[i_lt_20:%[0-9]+]] = OpSLessThan %bool {{%[0-9]+}} %int_20
+// CHECK-NEXT: OpLoopMerge %while_merge_0 %while_continue_0 None
+// CHECK-NEXT: OpBranchConditional [[i_lt_20]] %while_body_1 %while_merge_0
+ while(i<20) {
+// CHECK-NEXT: %while_body_1 = OpLabel
+ val = i;
+// CHECK: OpBranch %while_merge_0
+ {{break;}}
+// CHECK-NEXT: %while_continue_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_1
+ }
+// CHECK-NEXT: %while_merge_0 = OpLabel
+ --i;
+// CHECK: OpBranch %while_merge_1
+ break;
+// CHECK-NEXT: %while_continue_1 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_0
+ }
+// CHECK-NEXT: %while_merge_1 = OpLabel
+
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.hlsl
new file mode 100644
index 00000000000000..e84f2dcf5ee26b
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.hlsl
@@ -0,0 +1,89 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+int foo() { return true; }
+
+[numthreads(1, 1, 1)]
+void main() {
+ int val = 0;
+ int i = 0;
+
+
+// CHECK: OpBranch %while_check
+// CHECK: %while_check = OpLabel
+// CHECK: [[i_lt_10:%[0-9]+]] = OpSLessThan %bool {{%[0-9]+}} %int_10
+// CHECK-NEXT: OpLoopMerge %while_merge %while_continue None
+// CHECK-NEXT: OpBranchConditional [[i_lt_10]] %while_body %while_merge
+ while (i < 10) {
+// CHECK-NEXT: %while_body = OpLabel
+ val = i;
+// CHECK: [[val_gt_5:%[0-9]+]] = OpSGreaterThan %bool {{%[0-9]+}} %int_5
+// CHECK-NEXT: OpSelectionMerge %if_merge None
+// CHECK-NEXT: OpBranchConditional [[val_gt_5]] %if_true %if_merge
+ if (val > 5) {
+// CHECK-NEXT: %if_true = OpLabel
+// CHECK-NEXT: OpBranch %while_continue
+ continue;
+ }
+// CHECK-NEXT: %if_merge = OpLabel
+
+// CHECK: [[val_gt_6:%[0-9]+]] = OpSGreaterThan %bool {{%[0-9]+}} %int_6
+// CHECK-NEXT: OpSelectionMerge %if_merge_0 None
+// CHECK-NEXT: OpBranchConditional [[val_gt_6]] %if_true_0 %if_merge_0
+ if (val > 6) {
+// CHECK-NEXT: %if_true_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_continue
+ {{continue;}}
+ val++; // No SPIR-V should be emitted for this statement.
+ continue; // No SPIR-V should be emitted for this statement.
+ while(true); // No SPIR-V should be emitted for this statement.
+ --i; // No SPIR-V should be emitted for this statement.
+ }
+// CHECK-NEXT: %if_merge_0 = OpLabel
+
+// CHECK-NEXT: OpBranch %while_continue
+// CHECK-NEXT: %while_continue = OpLabel
+// CHECK-NEXT: OpBranch %while_check
+ }
+
+// CHECK-NEXT: %while_merge = OpLabel
+
+
+
+ //////////////////////////////////////////////////////////////////////////////////////
+ // Nested while loops with continue statements //
+ // Each continue statement should branch to the corresponding loop's continue block //
+ //////////////////////////////////////////////////////////////////////////////////////
+
+// CHECK-NEXT: OpBranch %while_check_0
+// CHECK-NEXT: %while_check_0 = OpLabel
+// CHECK-NEXT: OpLoopMerge %while_merge_1 %while_continue_1 None
+// CHECK-NEXT: OpBranchConditional %true %while_body_0 %while_merge_1
+ while (true) {
+// CHECK-NEXT: %while_body_0 = OpLabel
+ i++;
+
+// CHECK: OpBranch %while_check_1
+// CHECK-NEXT: %while_check_1 = OpLabel
+// CHECK: [[i_lt_20:%[0-9]+]] = OpSLessThan %bool {{%[0-9]+}} %int_20
+// CHECK-NEXT: OpLoopMerge %while_merge_0 %while_continue_0 None
+// CHECK-NEXT: OpBranchConditional [[i_lt_20]] %while_body_1 %while_merge_0
+ while(i<20) {
+// CHECK-NEXT: %while_body_1 = OpLabel
+ val = i;
+// CHECK: OpBranch %while_continue_0
+ continue;
+// CHECK-NEXT: %while_continue_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_1
+ }
+// CHECK-NEXT: %while_merge_0 = OpLabel
+ --i;
+// CHECK: OpBranch %while_continue_1
+ continue;
+ continue; // No SPIR-V should be emitted for this statement.
+
+// CHECK-NEXT: %while_continue_1 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_0
+ }
+// CHECK-NEXT: %while_merge_1 = OpLabel
+
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.hlsl
new file mode 100644
index 00000000000000..93495053079c38
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.hlsl
@@ -0,0 +1,79 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+[numthreads(1, 1, 1)]
+void main() {
+ int val=0, i=0, j=0, k=0;
+
+// CHECK: OpBranch %while_check
+// CHECK-NEXT: %while_check = OpLabel
+// CHECK-NEXT: [[i0:%[0-9]+]] = OpLoad %int %i
+// CHECK-NEXT: [[i_lt_10:%[0-9]+]] = OpSLessThan %bool [[i0]] %int_10
+// CHECK-NEXT: OpLoopMerge %while_merge_1 %while_continue_1 DontUnroll
+// CHECK-NEXT: OpBranchConditional [[i_lt_10]] %while_body %while_merge_1
+ while (i < 10) {
+// CHECK-NEXT: %while_body = OpLabel
+// CHECK-NEXT: [[val1:%[0-9]+]] = OpLoad %int %val
+// CHECK-NEXT: [[i1:%[0-9]+]] = OpLoad %int %i
+// CHECK-NEXT: [[val_plus_i:%[0-9]+]] = OpIAdd %int [[val1]] [[i1]]
+// CHECK-NEXT: OpStore %val [[val_plus_i]]
+// CHECK-NEXT: OpBranch %while_check_0
+ val = val + i;
+// CHECK-NEXT: %while_check_0 = OpLabel
+// CHECK-NEXT: [[j0:%[0-9]+]] = OpLoad %int %j
+// CHECK-NEXT: [[j_lt_20:%[0-9]+]] = OpSLessThan %bool [[j0]] %int_20
+// CHECK-NEXT: OpLoopMerge %while_merge_0 %while_continue_0 Unroll
+// CHECK-NEXT: OpBranchConditional [[j_lt_20]] %while_body_0 %while_merge_0
+ while (j < 20) {
+// CHECK-NEXT: %while_body_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_1
+
+// CHECK-NEXT: %while_check_1 = OpLabel
+// CHECK-NEXT: [[k0:%[0-9]+]] = OpLoad %int %k
+// CHECK-NEXT: [[k_lt_30:%[0-9]+]] = OpSLessThan %bool [[k0]] %int_30
+// CHECK-NEXT: OpLoopMerge %while_merge %while_continue DontUnroll
+// CHECK-NEXT: OpBranchConditional [[k_lt_30]] %while_body_1 %while_merge
+ while (k < 30) {
+// CHECK-NEXT: %while_body_1 = OpLabel
+// CHECK-NEXT: [[val2:%[0-9]+]] = OpLoad %int %val
+// CHECK-NEXT: [[k2:%[0-9]+]] = OpLoad %int %k
+// CHECK-NEXT: [[val_plus_k:%[0-9]+]] = OpIAdd %int [[val2]] [[k2]]
+// CHECK-NEXT: OpStore %val [[val_plus_k]]
+ val = val + k;
+// CHECK-NEXT: [[k3:%[0-9]+]] = OpLoad %int %k
+// CHECK-NEXT: [[k_plus_1:%[0-9]+]] = OpIAdd %int [[k3]] %int_1
+// CHECK-NEXT: OpStore %k [[k_plus_1]]
+ ++k;
+// CHECK-NEXT: OpBranch %while_continue
+// CHECK-NEXT: %while_continue = OpLabel
+// CHECK-NEXT: OpBranch %while_check_1
+ }
+// CHECK-NEXT: %while_merge = OpLabel
+
+// CHECK-NEXT: [[val3:%[0-9]+]] = OpLoad %int %val
+// CHECK-NEXT: [[val_mult_2:%[0-9]+]] = OpIMul %int [[val3]] %int_2
+// CHECK-NEXT: OpStore %val [[val_mult_2]]
+ val = val * 2;
+// CHECK-NEXT: [[j1:%[0-9]+]] = OpLoad %int %j
+// CHECK-NEXT: [[j_plus_1:%[0-9]+]] = OpIAdd %int [[j1]] %int_1
+// CHECK-NEXT: OpStore %j [[j_plus_1]]
+ ++j;
+// CHECK-NEXT: OpBranch %while_continue_0
+// CHECK-NEXT: %while_continue_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_0
+ }
+// CHECK-NEXT: %while_merge_0 = OpLabel
+
+// CHECK-NEXT: [[i2:%[0-9]+]] = OpLoad %int %i
+// CHECK-NEXT: [[i_plus_1:%[0-9]+]] = OpIAdd %int [[i2]] %int_1
+// CHECK-NEXT: OpStore %i [[i_plus_1]]
+ ++i;
+// CHECK-NEXT: OpBranch %while_continue_1
+// CHECK-NEXT: %while_continue_1 = OpLabel
+// CHECK-NEXT: OpBranch %while_check
+ }
+// CHECK-NEXT: %while_merge_1 = OpLabel
+
+
+// CHECK-NEXT: OpReturn
+// CHECK-NEXT: OpFunctionEnd
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.hlsl
new file mode 100644
index 00000000000000..c96b931969b0d4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.hlsl
@@ -0,0 +1,101 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+int foo() { return true; }
+
+[numthreads(1, 1, 1)]
+void main() {
+ int val = 0;
+ int i = 0;
+
+ //////////////////////////
+ //// Basic while loop ////
+ //////////////////////////
+
+// CHECK: OpBranch %while_check
+// CHECK-NEXT: %while_check = OpLabel
+
+// CHECK-NEXT: [[i:%[0-9]+]] = OpLoad %int %i
+// CHECK-NEXT: [[i_lt_10:%[0-9]+]] = OpSLessThan %bool [[i]] %int_10
+// CHECK-NEXT: OpLoopMerge %while_merge %while_continue None
+// CHECK-NEXT: OpBranchConditional [[i_lt_10]] %while_body %while_merge
+ while (i < 10) {
+// CHECK-NEXT: %while_body = OpLabel
+// CHECK-NEXT: [[i1:%[0-9]+]] = OpLoad %int %i
+// CHECK-NEXT: OpStore %val [[i1]]
+ val = i;
+// CHECK-NEXT: OpBranch %while_continue
+// CHECK-NEXT: %while_continue = OpLabel
+// CHECK-NEXT: OpBranch %while_check
+ }
+// CHECK-NEXT: %while_merge = OpLabel
+
+
+
+ //////////////////////////
+ //// infinite loop ////
+ //////////////////////////
+
+// CHECK-NEXT: OpBranch %while_check_0
+// CHECK-NEXT: %while_check_0 = OpLabel
+// CHECK-NEXT: OpLoopMerge %while_merge_0 %while_continue_0 None
+// CHECK-NEXT: OpBranchConditional %true %while_body_0 %while_merge_0
+ while (true) {
+// CHECK-NEXT: %while_body_0 = OpLabel
+// CHECK-NEXT: OpStore %val %int_0
+ val = 0;
+// CHECK-NEXT: OpBranch %while_continue_0
+// CHECK-NEXT: %while_continue_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_0
+ }
+// CHECK-NEXT: %while_merge_0 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_1
+
+
+
+ //////////////////////////
+ //// Null Body ////
+ //////////////////////////
+
+// CHECK-NEXT: %while_check_1 = OpLabel
+// CHECK-NEXT: [[val1:%[0-9]+]] = OpLoad %int %val
+// CHECK-NEXT: [[val_lt_20:%[0-9]+]] = OpSLessThan %bool [[val1]] %int_20
+// CHECK-NEXT: OpLoopMerge %while_merge_1 %while_continue_1 None
+// CHECK-NEXT: OpBranchConditional [[val_lt_20]] %while_body_1 %while_merge_1
+ while (val < 20)
+// CHECK-NEXT: %while_body_1 = OpLabel
+// CHECK-NEXT: OpBranch %while_continue_1
+// CHECK-NEXT: %while_continue_1 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_1
+ ;
+// CHECK-NEXT: %while_merge_1 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_2
+
+
+
+ ////////////////////////////////////////////////////////////////
+ //// Condition variable has VarDecl ////
+ //// foo() returns an integer which must be cast to boolean ////
+ ////////////////////////////////////////////////////////////////
+
+// CHECK-NEXT: %while_check_2 = OpLabel
+// CHECK-NEXT: [[foo:%[0-9]+]] = OpFunctionCall %int %foo
+// CHECK-NEXT: OpStore %a [[foo]]
+// CHECK-NEXT: [[a:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: [[is_a_true:%[0-9]+]] = OpINotEqual %bool [[a]] %int_0
+// CHECK-NEXT: OpLoopMerge %while_merge_2 %while_continue_2 None
+// CHECK-NEXT: OpBranchConditional [[is_a_true]] %while_body_2 %while_merge_2
+ while (int a = foo()) {
+// CHECK-NEXT: %while_body_2 = OpLabel
+// CHECK-NEXT: [[a1:%[0-9]+]] = OpLoad %int %a
+// CHECK-NEXT: OpStore %val [[a1]]
+ val = a;
+// CHECK-NEXT: OpBranch %while_continue_2
+// CHECK-NEXT: %while_continue_2 = OpLabel
+// CHECK-NEXT: OpBranch %while_check_2
+ }
+// CHECK-NEXT: %while_merge_2 = OpLabel
+
+
+// CHECK-NEXT: OpReturn
+// CHECK-NEXT: OpFunctionEnd
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.hlsl
new file mode 100644
index 00000000000000..39300d2bdbab13
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.hlsl
@@ -0,0 +1,20 @@
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
+
+[numthreads(1, 1, 1)]
+void main() {
+ int a, b;
+ while (a && b) {
+ }
+
+ while (a || b) {
+ }
+ while (a && ((a || b) && b)) {
+ }
+
+ while (a ? a : b) {
+ }
+
+ int x, y;
+ while (x + (x && y)) {
+ }
+}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
new file mode 100644
index 00000000000000..0e54d3495b319d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
@@ -0,0 +1,128 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan-compute"
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @fn() #4 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @fn1() #4 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 0
+}
+
+; Function Attrs: convergent noinline nounwind optnone
+define spir_func noundef i32 @fn2() #4 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+define internal spir_func void @main() #0 {
+; CHECK: %[[#cond:]] = OpINotEqual %[[#bool_ty:]] %[[#a:]] %[[#b:]]
+; CHECK: OpSelectionMerge %[[#cond_end:]] None
+; CHECK: OpBranchConditional %[[#cond]] %[[#cond_true:]] %[[#cond_false:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ %c = alloca i32, align 4
+ %val = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ %1 = load i32, ptr %a, align 4
+ %tobool = icmp ne i32 %1, 0
+ br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:
+ %2 = load i32, ptr %b, align 4
+ br label %cond.end
+
+; CHECK: %[[#cond_false]] = OpLabel
+; CHECK: OpBranch %[[#cond_end]]
+cond.false:
+ %3 = load i32, ptr %c, align 4
+ br label %cond.end
+
+; CHECK: %[[#cond_end]] = OpLabel
+; CHECK: %[[#tmp:]] = OpPhi %[[#int_ty:]] %[[#load_cond_true:]] %[[#cond_true]] %[[#load_cond_false:]] %[[#cond_false:]]
+; CHECK: %[[#cond:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0:]]
+; CHECK: OpSelectionMerge %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#cond]] %[[#if_then:]] %[[#if_end]]
+cond.end:
+ %cond = phi i32 [ %2, %cond.true ], [ %3, %cond.false ]
+ %tobool1 = icmp ne i32 %cond, 0
+ br i1 %tobool1, label %if.then, label %if.end
+
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpSelectionMerge %[[#cond_end8:]] None
+; CHECK: OpBranchConditional %[[#tmp:]] %[[#cond4_true:]] %[[#cond_false6:]]
+if.end:
+ %call2 = call spir_func noundef i32 @fn() #4 [ "convergencectrl"(token %0) ]
+ %tobool3 = icmp ne i32 %call2, 0
+ br i1 %tobool3, label %cond.true4, label %cond.false6
+
+; CHECK: %[[#cond_false6]] = OpLabel
+; CHECK: OpBranch %[[#cond_end8]]
+cond.false6:
+ %call7 = call spir_func noundef i32 @fn2() #4 [ "convergencectrl"(token %0) ]
+ br label %cond.end8
+
+; CHECK: %[[#cond_end8]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_end13:]] None
+; CHECK: OpBranchConditional %[[#tmp:]] %[[#if_then11:]] %[[#if_end13]]
+cond.end8:
+ %cond9 = phi i32 [ %call5, %cond.true4 ], [ %call7, %cond.false6 ]
+ %tobool10 = icmp ne i32 %cond9, 0
+ br i1 %tobool10, label %if.then11, label %if.end13
+
+; CHECK: %[[#if_end13]] = OpLabel
+; CHECK: OpReturn
+if.end13:
+ ret void
+
+; CHECK: %[[#if_then11]] = OpLabel
+; CHECK: OpBranch %[[#if_end13]]
+if.then11:
+ %4 = load i32, ptr %val, align 4
+ %inc12 = add nsw i32 %4, 1
+ store i32 %inc12, ptr %val, align 4
+ br label %if.end13
+
+; CHECK: %[[#cond4_true]] = OpLabel
+; CHECK: OpBranch %[[#cond_end8]]
+cond.true4:
+ %call5 = call spir_func noundef i32 @fn1() #4 [ "convergencectrl"(token %0) ]
+ br label %cond.end8
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#if_end]]
+if.then:
+ %5 = load i32, ptr %val, align 4
+ %inc = add nsw i32 %5, 1
+ store i32 %inc, ptr %val, align 4
+ br label %if.end
+
+; CHECK: %[[#cond_true]] = OpLabel
+; CHECK: OpBranch %[[#cond_end]]
+}
+
+declare token @llvm.experimental.convergence.entry() #2
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent norecurse "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #3 = { convergent }
+attributes #4 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-break.ll
new file mode 100644
index 00000000000000..6673aa4713157a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-break.ll
@@ -0,0 +1,89 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define internal spir_func void @main() #1 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK: OpBranch %[[#do_body:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+; CHECK: %[[#do_body]] = OpLabel
+; CHECK: OpSelectionMerge %[[#do_end:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_end]] %[[#if_end:]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %2, 1
+ store i32 %inc, ptr %i, align 4
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp sgt i32 %3, 5
+ br i1 %cmp, label %if.then, label %if.end
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpBranch %[[#do_end]]
+if.end:
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %do.end
+
+; Block is removed.
+if.then:
+ br label %do.end
+
+; CHECK: %[[#do_end]] = OpLabel
+; CHECK: OpBranch %[[#do_body2:]]
+do.end:
+ br label %do.body2
+
+; CHECK: %[[#do_body2]] = OpLabel
+; CHECK: OpBranch %[[#do_body4:]]
+do.body2:
+ %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %7 = load i32, ptr %i, align 4
+ %inc3 = add nsw i32 %7, 1
+ store i32 %inc3, ptr %i, align 4
+ br label %do.body4
+
+; CHECK: %[[#do_body4]] = OpLabel
+; CHECK: OpBranch %[[#do_end8:]]
+do.body4:
+ %8 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
+ %9 = load i32, ptr %val, align 4
+ %inc5 = add nsw i32 %9, 1
+ store i32 %inc5, ptr %val, align 4
+ br label %do.end8
+
+; CHECK: %[[#do_end8]] = OpLabel
+; CHECK: OpBranch %[[#do_end11:]]
+do.end8:
+ %11 = load i32, ptr %i, align 4
+ %dec = add nsw i32 %11, -1
+ store i32 %dec, ptr %i, align 4
+ br label %do.end11
+
+; CHECK: %[[#do_end11]] = OpLabel
+; CHECK: OpReturn
+do.end11:
+ ret void
+
+}
+
+
+declare token @llvm.experimental.convergence.entry() #0
+declare token @llvm.experimental.convergence.loop() #0
+
+attributes #0 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #1 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
new file mode 100644
index 00000000000000..d547ad8eded976
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
@@ -0,0 +1,124 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define spir_func noundef i32 @_Z3foov() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+define internal spir_func void @main() #2 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK: OpBranch %[[#do_body:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+; CHECK: %[[#do_header:]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end:]] %[[#do_cond:]] None
+; CHECK: OpBranch %[[#do_body:]]
+
+; CHECK: %[[#do_body]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_then:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_then]] %[[#if_end:]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %2, 1
+ store i32 %inc, ptr %i, align 4
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp sgt i32 %3, 5
+ br i1 %cmp, label %if.then, label %if.end
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpBranch %[[#if_then]]
+if.end:
+ %4 = load i32, ptr %i, align 4
+ store i32 %4, ptr %val, align 4
+ br label %do.cond
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#do_cond]]
+if.then:
+ br label %do.cond
+
+; CHECK: %[[#do_cond]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_header]] %[[#do_end]]
+do.cond:
+ %5 = load i32, ptr %i, align 4
+ %cmp1 = icmp slt i32 %5, 10
+ br i1 %cmp1, label %do.body, label %do.end
+
+; CHECK: %[[#do_end]] = OpLabel
+; CHECK: OpBranch %[[#do_body2:]]
+do.end:
+ br label %do.body2
+
+; CHECK: %[[#do_body2]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end11:]] %[[#do_cond9:]] None
+; CHECK: OpBranch %[[#do_body4:]]
+do.body2:
+ %6 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %7 = load i32, ptr %i, align 4
+ %inc3 = add nsw i32 %7, 1
+ store i32 %inc3, ptr %i, align 4
+ br label %do.body4
+
+; CHECK: %[[#do_body4]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end8:]] %[[#do_cond6:]] None
+; CHECK: OpBranch %[[#do_cond6]]
+do.body4:
+ %8 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %6) ]
+ %9 = load i32, ptr %val, align 4
+ %inc5 = add nsw i32 %9, 1
+ store i32 %inc5, ptr %val, align 4
+ br label %do.cond6
+
+; CHECK: %[[#do_cond6]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body4]] %[[#do_end8]]
+do.cond6:
+ %10 = load i32, ptr %i, align 4
+ %cmp7 = icmp slt i32 %10, 10
+ br i1 %cmp7, label %do.body4, label %do.end8
+
+; CHECK: %[[#do_end8]] = OpLabel
+; CHECK: OpBranch %[[#do_cond9]]
+do.end8:
+ %11 = load i32, ptr %i, align 4
+ %dec = add nsw i32 %11, -1
+ store i32 %dec, ptr %i, align 4
+ br label %do.cond9
+
+; CHECK: %[[#do_cond9]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body2]] %[[#do_end11]]
+do.cond9:
+ %12 = load i32, ptr %val, align 4
+ %cmp10 = icmp slt i32 %12, 10
+ br i1 %cmp10, label %do.body2, label %do.end11
+
+; CHECK: %[[#do_end11]] = OpLabel
+; CHECK: OpReturn
+do.end11:
+ ret void
+}
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #3 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
new file mode 100644
index 00000000000000..a16eed5cdfb4ce
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
@@ -0,0 +1,102 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define internal spir_func void @main() #0 {
+; CHECK: %[[#entry:]] = OpLabel
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ %j = alloca i32, align 4
+ %k = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ store i32 0, ptr %j, align 4
+ store i32 0, ptr %k, align 4
+ br label %do.body
+
+; CHECK: %[[#do_1_header:]] = OpLabel
+; CHECK: OpLoopMerge %[[#end:]] %[[#do_1_latch:]] None
+; CHECK: OpBranch %[[#do_2_header:]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %val, align 4
+ %3 = load i32, ptr %i, align 4
+ %add = add nsw i32 %2, %3
+ store i32 %add, ptr %val, align 4
+ br label %do.body1
+
+; CHECK: %[[#do_2_header]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_2_end:]] %[[#do_2_latch:]] None
+; CHECK: OpBranch %[[#do_2_body:]]
+do.body1:
+ %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %1) ]
+ br label %do.body2
+
+; CHECK: %[[#do_2_body]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_3_end:]] %[[#do_3_header:]] None
+; CHECK: OpBranch %[[#do_3_header]]
+do.body2:
+ %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %4) ]
+ %6 = load i32, ptr %k, align 4
+ %inc = add nsw i32 %6, 1
+ store i32 %inc, ptr %k, align 4
+ br label %do.cond
+
+; CHECK: %[[#do_3_header]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_2_body]] %[[#do_3_end]]
+do.cond:
+ %7 = load i32, ptr %k, align 4
+ %cmp = icmp slt i32 %7, 30
+ br i1 %cmp, label %do.body2, label %do.end
+
+; CHECK: %[[#do_3_end]] = OpLabel
+; CHECK: OpBranch %[[#do_2_latch]]
+do.end:
+ %8 = load i32, ptr %j, align 4
+ %inc3 = add nsw i32 %8, 1
+ store i32 %inc3, ptr %j, align 4
+ br label %do.cond4
+
+; CHECK: %[[#do_2_latch]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_2_header]] %[[#do_2_end]]
+do.cond4:
+ %9 = load i32, ptr %j, align 4
+ %cmp5 = icmp slt i32 %9, 20
+ br i1 %cmp5, label %do.body1, label %do.end6
+
+; CHECK: %[[#do_2_end]] = OpLabel
+; CHECK: OpBranch %[[#do_1_latch]]
+do.end6:
+ %10 = load i32, ptr %i, align 4
+ %inc7 = add nsw i32 %10, 1
+ store i32 %inc7, ptr %i, align 4
+ br label %do.cond8
+
+; CHECK: %[[#do_1_latch]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_1_header]] %[[#end]]
+do.cond8:
+ %11 = load i32, ptr %i, align 4
+ %cmp9 = icmp slt i32 %11, 10
+ br i1 %cmp9, label %do.body, label %do.end10
+
+; CHECK: %[[#end]] = OpLabel
+; CHECK: OpReturn
+do.end10:
+ ret void
+}
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
new file mode 100644
index 00000000000000..6d4a0e591cf512
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
@@ -0,0 +1,98 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+
+define spir_func noundef i32 @_Z3foov() #0 {
+; CHECK: %[[#foo:]] = OpLabel
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+
+define internal spir_func void @main() #2 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK: OpBranch %[[#do_body:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %val = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ store i32 0, ptr %i, align 4
+ br label %do.body
+
+; CHECK: %[[#do_body]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end:]] %[[#do_cond:]] None
+; CHECK: OpBranch %[[#do_cond]]
+do.body:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %i, align 4
+ store i32 %2, ptr %val, align 4
+ br label %do.cond
+
+; CHECK: %[[#do_cond]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body]] %[[#do_end]]
+do.cond:
+ %3 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %3, 10
+ br i1 %cmp, label %do.body, label %do.end
+
+; CHECK: %[[#do_end]] = OpLabel
+; CHECK: OpBranch %[[#do_body1:]]
+do.end:
+ br label %do.body1
+
+; CHECK: %[[#do_body1]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end3:]] %[[#do_cond2:]] None
+; CHECK: OpBranch %[[#do_cond2]]
+do.body1:
+ %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ store i32 0, ptr %val, align 4
+ br label %do.cond2
+
+; CHECK: %[[#do_cond2]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body1]] %[[#do_end3]]
+do.cond2:
+ br i1 true, label %do.body1, label %do.end3
+
+; CHECK: %[[#do_end3]] = OpLabel
+; CHECK: OpBranch %[[#do_body4:]]
+do.end3:
+ br label %do.body4
+
+; CHECK: %[[#do_body4]] = OpLabel
+; CHECK: OpLoopMerge %[[#do_end7:]] %[[#do_cond5:]] None
+; CHECK: OpBranch %[[#do_cond5]]
+do.body4:
+ %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ br label %do.cond5
+
+; CHECK: %[[#do_cond5]] = OpLabel
+; CHECK: OpBranchConditional %[[#cond:]] %[[#do_body4]] %[[#do_end7]]
+do.cond5:
+ %6 = load i32, ptr %val, align 4
+ %cmp6 = icmp slt i32 %6, 20
+ br i1 %cmp6, label %do.body4, label %do.end7
+
+; CHECK: %[[#do_end7]] = OpLabel
+; CHECK: OpReturn
+do.end7:
+ ret void
+}
+
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
new file mode 100644
index 00000000000000..0bce3d20483a20
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
@@ -0,0 +1,93 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - --asm-verbose=0 | FileCheck %s --match-full-lines
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+; CHECK-DAG: OpName %[[#fn:]] "fn"
+; CHECK-DAG: OpName %[[#main:]] "main"
+; CHECK-DAG: OpName %[[#var_a:]] "a"
+; CHECK-DAG: OpName %[[#var_b:]] "b"
+
+; CHECK-DAG: %[[#bool:]] = OpTypeBool
+; CHECK-DAG: %[[#true:]] = OpConstantTrue %[[#bool]]
+
+; CHECK: %[[#fn]] = OpFunction %[[#param:]] DontInline %[[#ftype:]]
+define spir_func noundef i32 @fn() #0 {
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ ret i32 1
+}
+
+; CHECK: %[[#main]] = OpFunction %[[#param:]] DontInline %[[#ftype:]]
+
+define internal spir_func void @main() #3 {
+
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK-DAG: %[[#var_a]] = OpVariable %[[#type:]] Function
+; CHECK-DAG: %[[#var_b]] = OpVariable %[[#type:]] Function
+; CHECK: %[[#tmp:]] = OpLoad %[[#type:]] %[[#var_a]] Aligned 4
+; CHECK: %[[#cond:]] = OpINotEqual %[[#bool]] %[[#tmp]] %[[#const:]]
+; CHECK: OpSelectionMerge %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#true]] %[[#entry:]] %[[#dead:]]
+
+; CHECK: %[[#dead]] = OpLabel
+; CHECK-NEXT: OpUnreachable
+
+; CHECK: %[[#entry]] = OpLabel
+; CHECK: OpSelectionMerge %[[#new_exit:]] None
+; CHECK: OpBranchConditional %[[#cond]] %[[#if_then:]] %[[#lor_lhs_false:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ %val = alloca i32, align 4
+ store i32 0, ptr %val, align 4
+ %1 = load i32, ptr %a, align 4
+ %tobool = icmp ne i32 %1, 0
+ br i1 %tobool, label %if.then, label %lor.lhs.false
+
+
+; CHECK: %[[#lor_lhs_false]] = OpLabel
+; CHECK: %[[#tmp:]] = OpLoad %[[#type:]] %[[#var_b]] Aligned 4
+; CHECK: %[[#cond:]] = OpINotEqual %[[#bool]] %[[#tmp]] %[[#value:]]
+; CHECK: OpBranchConditional %[[#cond]] %[[#new_exit]] %[[#alias_exit:]]
+
+lor.lhs.false:
+ %2 = load i32, ptr %b, align 4
+ %tobool1 = icmp ne i32 %2, 0
+ br i1 %tobool1, label %if.then, label %if.end
+
+; CHECK: %[[#alias_exit]] = OpLabel
+; CHECK: OpBranch %[[#new_exit]]
+
+; CHECK: %[[#new_exit]] = OpLabel
+; CHECK: %[[#tmp:]] = OpPhi %[[#type:]] %[[#A:]] %[[#entry]] %[[#A:]] %[[#lor_lhs_false]] %[[#B:]] %[[#alias_exit]]
+; CHECK: %[[#cond:]] = OpIEqual %[[#bool]] %[[#A]] %[[#tmp]]
+; CHECK: OpBranchConditional %[[#cond]] %[[#if_then:]] %[[#if_end:]]
+
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpReturn
+if.end:
+ ret void
+
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK: OpBranch %[[#if_end]]
+if.then:
+ %8 = load i32, ptr %val, align 4
+ %inc = add nsw i32 %8, 1
+ store i32 %inc, ptr %val, align 4
+ br label %if.end
+}
+
+declare token @llvm.experimental.convergence.entry() #2
+
+attributes #0 = { convergent noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #3 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
index 55f85726cbdd2b..47069b9e374dc6 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll
@@ -1,3 +1,4 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
@@ -7,14 +8,15 @@ define internal spir_func void @main() #0 {
; CHECK: OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#pint_ty:]] = OpTypePointer Function %[[#int_ty]]
+; CHECK-DAG: %[[#int_fpty:]] = OpTypePointer Function %[[#int_ty]]
+; CHECK-DAG: %[[#int_ipty:]] = OpTypePointer Input %[[#int_ty]]
; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool
; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#int_ty]] 0
-; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#int_ty]] 1
; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#int_ty]] 10
+; CHECK-DAG: %[[#builtin]] = OpVariable %[[#int_ipty]] Input
; CHECK: %[[#entry:]] = OpLabel
-; CHECK: %[[#idx:]] = OpVariable %[[#pint_ty]] Function
+; CHECK: %[[#idx:]] = OpVariable %[[#int_fpty]] Function
; CHECK: OpStore %[[#idx]] %[[#int_0]] Aligned 4
; CHECK: OpBranch %[[#while_cond:]]
entry:
@@ -26,32 +28,28 @@ entry:
; CHECK: %[[#while_cond]] = OpLabel
; CHECK: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
; CHECK: %[[#cmp:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_10]]
-; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end:]]
+; CHECK: OpLoopMerge %[[#new_end:]] %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end]]
+
+; CHECK: %[[#new_end]] = OpLabel
+; CHECK: OpBranch %[[#while_end:]]
while.cond:
%1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
%2 = load i32, ptr %idx, align 4
%cmp = icmp ne i32 %2, 10
br i1 %cmp, label %while.body, label %while.end
-; CHECK: %[[#new_end]] = OpLabel
-; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_1]] %[[#while_cond]] %[[#int_0]] %[[#while_body]]
-; CHECK: OpSwitch %[[#route]] %[[#if_then:]] 1 %[[#while_end_loopexit:]]
-
-; CHECK: %[[#while_end_loopexit]] = OpLabel
-; CHECK: OpBranch %[[#while_end:]]
-
; CHECK: %[[#while_end]] = OpLabel
; CHECK: OpReturn
-
-; CHECK: %[[#if_then]] = OpLabel
-; CHECK: OpBranch %[[#while_end]]
+while.end:
+ ret void
; CHECK: %[[#while_body]] = OpLabel
; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
; CHECK-NEXT: %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-; CHECK: OpBranchConditional %[[#cmp1]] %[[#new_end]] %[[#if_end:]]
+; CHECK: OpBranchConditional %[[#cmp1]] %[[#new_end]] %[[#if_end]]
while.body:
%3 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %3, ptr %idx, align 4
@@ -59,17 +57,13 @@ while.body:
%cmp1 = icmp eq i32 %4, 0
br i1 %cmp1, label %if.then, label %if.end
-if.then:
- br label %while.end
-
; CHECK: %[[#if_end]] = OpLabel
; CHECK: OpBranch %[[#while_cond]]
if.end:
br label %while.cond
-while.end:
- ret void
-
+if.then:
+ br label %while.end
}
declare token @llvm.experimental.convergence.entry() #2
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
index 72ce6bdcba5ffd..6852108ae26f8c 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll
@@ -1,3 +1,4 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
@@ -6,11 +7,11 @@ target triple = "spirv-unknown-vulkan-compute"
define internal spir_func void @main() #0 {
; CHECK: OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId
+
; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#pint_ty:]] = OpTypePointer Function %[[#int_ty]]
; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool
; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#int_ty]] 0
-; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#int_ty]] 1
; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#int_ty]] 10
; CHECK: %[[#entry:]] = OpLabel
@@ -26,7 +27,8 @@ entry:
; CHECK: %[[#while_cond]] = OpLabel
; CHECK: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
; CHECK: %[[#cmp:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_10]]
-; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end:]]
+; CHECK: OpLoopMerge %[[#new_end:]] %[[#if_end:]] None
+; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end]]
while.cond:
%1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
%2 = load i32, ptr %idx, align 4
@@ -34,21 +36,19 @@ while.cond:
br i1 %cmp, label %while.body, label %while.end
; CHECK: %[[#new_end]] = OpLabel
-; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_0]] %[[#while_cond]] %[[#int_1]] %[[#tail:]]
-; CHECK: OpSwitch %[[#route]] %[[#while_end_loopexit:]] 1 %[[#while_end:]]
-
-; CHECK: %[[#while_end]] = OpLabel
-; CHECK: OpReturn
+; CHECK: OpBranch %[[#while_end:]]
-; CHECK: %[[#while_end_loopexit]] = OpLabel
-; CHECK: OpBranch %[[#while_end]]
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK: OpReturn
+while.end:
+ ret void
-; CHECK: %[[#while_body]] = OpLabel
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
-; CHECK-NEXT: %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-; CHECK: OpBranchConditional %[[#cmp1]] %[[#if_then:]] %[[#if_end:]]
+; CHECK: %[[#while_body]] = OpLabel
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
+; CHECK-NEXT: %[[#cmp1:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+; CHECK: OpBranchConditional %[[#cmp1]] %[[#if_then:]] %[[#if_end]]
while.body:
%3 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %3, ptr %idx, align 4
@@ -56,29 +56,19 @@ while.body:
%cmp1 = icmp eq i32 %4, 0
br i1 %cmp1, label %if.then, label %if.end
-; CHECK: %[[#if_end]] = OpLabel
-; CHECK: OpBranch %[[#while_cond]]
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK: OpBranch %[[#while_cond]]
+if.end:
+ br label %while.cond
-; CHECK: %[[#if_then]] = OpLabel
-; CHECK-NEXT: OpBranch %[[#tail]]
+; CHECK: %[[#if_then:]] = OpLabel
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK: OpBranch %[[#new_end:]]
if.then:
- br label %tail
-
-; CHECK: %[[#tail]] = OpLabel
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK: OpBranch %[[#new_end:]]
-tail:
%5 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %5, ptr %idx, align 4
br label %while.end
-
-if.end:
- br label %while.cond
-
-while.end:
- ret void
-
}
declare token @llvm.experimental.convergence.entry() #2
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
index 1768d6526f2ba6..c7c37aacf9d9bf 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll
@@ -1,3 +1,4 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
@@ -27,28 +28,25 @@ entry:
; CHECK: %[[#while_cond]] = OpLabel
; CHECK: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
; CHECK: %[[#cmp:]] = OpINotEqual %[[#bool_ty]] %[[#tmp]] %[[#int_10]]
-; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end:]]
+; CHECK: OpLoopMerge %[[#new_end:]] %[[#if_end2:]] None
+; CHECK: OpBranchConditional %[[#cmp]] %[[#while_body:]] %[[#new_end]]
while.cond:
%1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
%2 = load i32, ptr %idx, align 4
%cmp = icmp ne i32 %2, 10
br i1 %cmp, label %while.body, label %while.end
+; TODO: this OpSwitch is useless. Improve the "remove useless branches" step of the structurizer to
+; cleanup those.
; CHECK: %[[#new_end]] = OpLabel
-; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_2]] %[[#while_cond]] %[[#int_0]] %[[#while_body]] %[[#int_1]] %[[#if_end:]]
-; CHECK: OpSwitch %[[#route]] %[[#if_then:]] 1 %[[#if_then2:]] 2 %[[#while_end_loopexit:]]
+; CHECK: %[[#route:]] = OpPhi %[[#int_ty]] %[[#int_0]] %[[#while_cond]] %[[#int_1]] %[[#while_body]] %[[#int_2]] %[[#if_end:]]
+; CHECK: OpSwitch %[[#route]] %[[#while_end:]] 1 %[[#while_end:]] 2 %[[#while_end:]]
-; CHECK: %[[#while_end_loopexit]] = OpLabel
-; CHECK: OpBranch %[[#while_end:]]
-
-; CHECK: %[[#while_end]] = OpLabel
-; CHECK: OpReturn
-
-; CHECK: %[[#if_then2]] = OpLabel
-; CHECK: OpBranch %[[#while_end]]
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK: OpReturn
+while.end:
+ ret void
-; CHECK: %[[#if_then]] = OpLabel
-; CHECK: OpBranch %[[#while_end]]
; CHECK: %[[#while_body]] = OpLabel
; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
@@ -63,15 +61,12 @@ while.body:
%cmp1 = icmp eq i32 %4, 0
br i1 %cmp1, label %if.then, label %if.end
-if.then:
- br label %while.end
-
-; CHECK: %[[#if_end]] = OpLabel
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
-; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
-; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
-; CHECK-NEXT: %[[#cmp2:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
-; CHECK: OpBranchConditional %[[#cmp2]] %[[#new_end]] %[[#if_end2:]]
+; CHECK: %[[#if_end]] = OpLabel
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#builtin]] Aligned 1
+; CHECK-NEXT: OpStore %[[#idx]] %[[#tmp]] Aligned 4
+; CHECK-NEXT: %[[#tmp:]] = OpLoad %[[#int_ty]] %[[#idx]] Aligned 4
+; CHECK-NEXT: %[[#cmp2:]] = OpIEqual %[[#bool_ty]] %[[#tmp]] %[[#int_0]]
+; CHECK: OpBranchConditional %[[#cmp2]] %[[#new_end]] %[[#if_end2]]
if.end:
%5 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %1) ]
store i32 %5, ptr %idx, align 4
@@ -79,28 +74,25 @@ if.end:
%cmp2 = icmp eq i32 %6, 0
br i1 %cmp2, label %if.then2, label %if.end2
-if.then2:
- br label %while.end
-
-; CHECK: %[[#if_end2]] = OpLabel
-; CHECK: OpBranch %[[#while_cond:]]
+; CHECK: %[[#if_end2]] = OpLabel
+; CHECK: OpBranch %[[#while_cond:]]
if.end2:
br label %while.cond
+; Those blocks are removed by the structurizer.
+if.then:
+ br label %while.end
-while.end:
- ret void
-
+if.then2:
+ br label %while.end
}
-declare token @llvm.experimental.convergence.entry() #2
-declare token @llvm.experimental.convergence.loop() #2
-declare i32 @__hlsl_wave_get_lane_index() #3
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+declare i32 @__hlsl_wave_get_lane_index() convergent
attributes #0 = { convergent noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #1 = { convergent norecurse "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
-attributes #3 = { convergent }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
!llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll
index 755235b7012a3c..eccd36d02a0da8 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll
@@ -1,3 +1,4 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
@@ -21,8 +22,10 @@ while.cond:
%cmp = icmp ne i32 %2, 0
br i1 %cmp, label %while.body, label %while.end
-; CHECK: %[[#while_end]] = OpLabel
+; CHECK: %[[#while_end]] = OpLabel
; CHECK-NEXT: OpReturn
+while.end:
+ ret void
; CHECK: %[[#while_body]] = OpLabel
; CHECK: OpBranch %[[#while_cond]]
@@ -31,8 +34,6 @@ while.body:
store i32 %3, ptr %idx, align 4
br label %while.cond
-while.end:
- ret void
}
declare token @llvm.experimental.convergence.entry() #2
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
new file mode 100644
index 00000000000000..6c92eac23c9dad
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
@@ -0,0 +1,131 @@
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s --match-full-lines
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv-unknown-vulkan1.3-compute"
+
+define internal spir_func void @main() #0 {
+; CHECK: %[[#entry:]] = OpLabel
+; CHECK: OpBranch %[[#while_cond:]]
+entry:
+ %0 = call token @llvm.experimental.convergence.entry()
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ %cond = alloca i32, align 4
+ %i = alloca i32, align 4
+ store i32 1, ptr %cond, align 4
+ br label %while.cond
+
+; CHECK: %[[#while_cond]] = OpLabel
+; CHECK: OpSelectionMerge %[[#while_end:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#while_body:]] %[[#while_end]]
+while.cond:
+ %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+ %2 = load i32, ptr %cond, align 4
+ %tobool = icmp ne i32 %2, 0
+ br i1 %tobool, label %while.body, label %while.end
+
+; CHECK: %[[#while_end]] = OpLabel
+; CHECK: OpReturn
+while.end:
+ ret void
+
+; CHECK: %[[#while_body]] = OpLabel
+; CHECK: OpSelectionMerge %[[#switch_end:]] None
+; CHECK: OpSwitch %[[#cond:]] %[[#switch_end]] 1 %[[#case_1:]] 2 %[[#case_2:]] 5 %[[#case_5:]]
+while.body:
+ %3 = load i32, ptr %b, align 4
+ switch i32 %3, label %sw.default [
+ i32 1, label %sw.bb
+ i32 2, label %sw.bb1
+ i32 5, label %sw.bb2
+ ]
+
+; CHECK: %[[#case_5]] = OpLabel
+; CHECK: OpBranch %[[#switch_end]]
+sw.bb2:
+ store i32 5, ptr %a, align 4
+ br label %while.end
+
+; CHECK: %[[#case_2]] = OpLabel
+; CHECK: OpBranch %[[#switch_end]]
+sw.bb1:
+ store i32 3, ptr %a, align 4
+ br label %while.end
+
+; CHECK: %[[#case_1]] = OpLabel
+; CHECK: OpBranch %[[#switch_end]]
+sw.bb:
+ store i32 1, ptr %a, align 4
+ br label %while.end
+
+; CHECK: %[[#switch_end]] = OpLabel
+; CHECK: %[[#phi:]] = OpPhi %[[#type:]] %[[#A:]] %[[#while_body]] %[[#B:]] %[[#case_5]] %[[#B:]] %[[#case_2]] %[[#B:]] %[[#case_1]]
+; CHECK: %[[#tmp:]] = OpIEqual %[[#type:]] %[[#A]] %[[#phi]]
+; CHECK: OpBranchConditional %[[#tmp]] %[[#sw_default:]] %[[#while_end]]
+
+; CHECK: %[[#sw_default]] = OpLabel
+; CHECK: OpStore %[[#A:]] %[[#B:]] Aligned 4
+; CHECK: OpBranch %[[#for_cond:]]
+sw.default:
+ store i32 0, ptr %i, align 4
+ br label %for.cond
+
+; CHECK: %[[#for_cond]] = OpLabel
+; CHECK: OpSelectionMerge %[[#for_merge:]] None
+; CHECK-NEXT: OpBranchConditional %[[#cond:]] %[[#for_merge]] %[[#for_end:]]
+for.cond:
+ %4 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %1) ]
+ %5 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %5, 10
+ br i1 %cmp, label %for.body, label %for.end
+
+; CHECK: %[[#for_end]] = OpLabel
+; CHECK: OpBranch %[[#for_merge]]
+for.end:
+ br label %while.end
+
+; CHECK: %[[#for_merge]] = OpLabel
+; CHECK: %[[#phi:]] = OpPhi %[[#type:]] %[[#A:]] %[[#for_cond]] %[[#B:]] %[[#for_end]]
+; CHECK: %[[#tmp:]] = OpIEqual %[[#type:]] %[[#A]] %[[#phi]]
+; CHECK: OpBranchConditional %[[#tmp]] %[[#for_body:]] %[[#while_end]]
+
+
+; CHECK: %[[#for_body]] = OpLabel
+; CHECK: OpSelectionMerge %[[#if_merge:]] None
+; CHECK: OpBranchConditional %[[#cond:]] %[[#if_merge]] %[[#if_else:]]
+for.body:
+ %6 = load i32, ptr %cond, align 4
+ %tobool3 = icmp ne i32 %6, 0
+ br i1 %tobool3, label %if.then, label %if.else
+
+; CHECK: %[[#if_else]] = OpLabel
+; CHECK: OpBranch %[[#if_merge]]
+if.else:
+ br label %while.end
+
+; CHECK: %[[#if_merge]] = OpLabel
+; CHECK: OpBranch %[[#while_end]]
+if.then:
+ br label %while.end
+
+; CHECK-NOT: %[[#for_inc:]] = OpLabel
+; This block is not emitted since it's unreachable.
+for.inc:
+ %7 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %7, 1
+ store i32 %inc, ptr %i, align 4
+ br label %for.cond
+}
+
+declare token @llvm.experimental.convergence.entry() #1
+declare token @llvm.experimental.convergence.loop() #1
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{i32 7, !"frame-pointer", i32 2}
>From 82e9c149c58915fc0a573878e03b6c4b71ea8c1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Mon, 9 Sep 2024 11:27:50 +0200
Subject: [PATCH 2/7] typo fixes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Nathan Gauër <brioche at google.com>
---
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 4 +-
llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp | 45 ++++++++++-----------
2 files changed, 24 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 1784f00be600dd..abaee954705c54 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -816,8 +816,8 @@ static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
}
// For each instruction to fix, we replace all the G_BLOCK_ADDR operands by
- // the actual MBB it references. Once those references updated, we can cleanup
- // remaining G_BLOCK_ADDR references.
+ // the actual MBB it references. Once those references have been updated, we
+ // can cleanup remaining G_BLOCK_ADDR references.
SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
SmallPtrSet<MachineInstr *, 8> ToEraseMI;
MachineRegisterInfo &MRI = MF.getRegInfo();
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index f663b7f427e235..4dc2b26b64d56f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -220,7 +220,7 @@ const ConvergenceRegion *getRegionForHeader(const ConvergenceRegion *Node,
}
// Returns the single BasicBlock exiting the convergence region `CR`,
-// nullptr if no such exit exists. F must be the function CR belongs to.
+// nullptr if no such exit exists.
BasicBlock *getExitFor(const ConvergenceRegion *CR) {
std::unordered_set<BasicBlock *> ExitTargets;
for (BasicBlock *Exit : CR->Exits) {
@@ -285,7 +285,7 @@ bool hasLoopMergeInstruction(BasicBlock &BB) {
return false;
}
-// Returns truye is I is an OpSelectionMerge or OpLoopMerge instruction, false
+// Returns true is I is an OpSelectionMerge or OpLoopMerge instruction, false
// otherwise.
bool isMergeInstruction(Instruction *I) {
return getDesignatedMergeBlock(I) != nullptr;
@@ -413,7 +413,7 @@ void replaceIfBranchTargets(BasicBlock *BB, BasicBlock *OldTarget,
C->destroyConstant();
}
-// Replaces the branching instruction destination of |BB| by |NewTarget| if it
+// Replaces the target of branch instruction in |BB| with |NewTarget| if it
// was |OldTarget|. This function also fixes the associated merge instruction.
// Note: this function does not simplify branching instructions, it only updates
// targets. See also: simplifyBranches.
@@ -438,7 +438,7 @@ void replaceBranchTargets(BasicBlock *BB, BasicBlock *OldTarget,
}
// Replaces basic bloc operands |OldSrc| or OpPhi instructions in |BB| by
-// |NewSrc|. This function does not simplifies the OpPhi instruction once
+// |NewSrc|. This function does not simplify the OpPhi instruction once
// transformed.
void replacePhiTargets(BasicBlock *BB, BasicBlock *OldSrc, BasicBlock *NewSrc) {
for (PHINode &Phi : BB->phis()) {
@@ -462,8 +462,8 @@ class SPIRVStructurizer : public FunctionPass {
using ConstructList = std::vector<std::unique_ptr<DivergentConstruct>>;
// Represents a divergent construct in the SPIR-V sense.
- // Such construct is represented by a header (entry), a merge block (exit),
- // and possible a continue block (back-edge). Each construct can contain other
+ // Such constructs are represented by a header (entry), a merge block (exit),
+ // and possibly a continue block (back-edge). A construct can contain other
// constructs, but their boundaries do not cross.
struct DivergentConstruct {
BasicBlock *Header = nullptr;
@@ -491,13 +491,12 @@ class SPIRVStructurizer : public FunctionPass {
DT.recalculate(F);
}
- // Returns the list of blocks that belongs to a SPIR-V continue construct.
+ // Returns the list of blocks that belong to a SPIR-V continue construct.
std::vector<BasicBlock *> getContinueConstructBlocks(BasicBlock *Header,
BasicBlock *Continue) {
std::vector<BasicBlock *> Output;
Loop *L = LI.getLoopFor(Continue);
- BasicBlock *BackEdgeBlock = L->getLoopLatch();
- assert(BackEdgeBlock);
+ assert(L->getLoopLatch() != nullptr);
partialOrderVisit(*Continue, [&](BasicBlock *BB) {
if (BB == Header)
@@ -508,7 +507,7 @@ class SPIRVStructurizer : public FunctionPass {
return Output;
}
- // Returns the list of blocks that belongs to a SPIR-V loop construct.
+ // Returns the list of blocks that belong to a SPIR-V loop construct.
std::vector<BasicBlock *> getLoopConstructBlocks(BasicBlock *Header,
BasicBlock *Merge,
BasicBlock *Continue) {
@@ -525,7 +524,7 @@ class SPIRVStructurizer : public FunctionPass {
return Output;
}
- // Returns the list of blocks that belongs to a SPIR-V selection construct.
+ // Returns the list of blocks that belong to a SPIR-V selection construct.
std::vector<BasicBlock *>
getSelectionConstructBlocks(DivergentConstruct *Node) {
assert(DT.dominates(Node->Header, Node->Merge));
@@ -551,7 +550,7 @@ class SPIRVStructurizer : public FunctionPass {
return Output;
}
- // Returns the list of blocks that belongs to a SPIR-V switch construct.
+ // Returns the list of blocks that belong to a SPIR-V switch construct.
std::vector<BasicBlock *> getSwitchConstructBlocks(BasicBlock *Header,
BasicBlock *Merge) {
assert(DT.dominates(Header, Merge));
@@ -571,7 +570,7 @@ class SPIRVStructurizer : public FunctionPass {
return Output;
}
- // Returns the list of blocks that belongs to a SPIR-V case construct.
+ // Returns the list of blocks that belong to a SPIR-V case construct.
std::vector<BasicBlock *> getCaseConstructBlocks(BasicBlock *Target,
BasicBlock *Merge) {
assert(DT.dominates(Target, Merge));
@@ -597,12 +596,12 @@ class SPIRVStructurizer : public FunctionPass {
//
// clang-format off
//
- // In SPIR-V, construct must have a single exit/merge.
- // Given A, B nodes in the construct, a C a node outside, with the following edges.
+ // In SPIR-V, constructs must have a single exit/merge.
+ // Given nodes A and B in the construct, a node C outside, and the following edges.
// A -> C
// B -> C
//
- // In such cases, we must create a new exit node D, that belongs to the construct to make is viable:
+ // In such cases, we must create a new exit node D, that belong to the construct to make is viable:
// A -> D -> C
// B -> D -> C
//
@@ -839,11 +838,11 @@ class SPIRVStructurizer : public FunctionPass {
return Modified;
}
- // When a block has multiple OpSelectionMerge/OpLoopMerge, sorts those
- // instructions to but the "largest" first. A merge instruction is defined as
- // larger than another when its target merge block post-dominates the other
- // target's merge block.
- // (This ordering should match the nesting ordering of the source HLSL).
+ // When a block has multiple OpSelectionMerge/OpLoopMerge instructions, sorts
+ // them to put the "largest" first. A merge instruction is defined as larger
+ // than another when its target merge block post-dominates the other target's
+ // merge block. (This ordering should match the nesting ordering of the source
+ // HLSL).
bool sortSelectionMerge(Function &F, BasicBlock &Block) {
std::vector<Instruction *> MergeInstructions;
for (Instruction &I : Block)
@@ -1159,8 +1158,8 @@ class SPIRVStructurizer : public FunctionPass {
return Modified;
}
- // Makes sure every case target in |F| are unique. If 2 case branch to the
- // same basic block, one of the target is updated so it jumps to a new basic
+ // Makes sure every case target in |F| is unique. If 2 cases branch to the
+ // same basic block, one of the targets is updated so it jumps to a new basic
// block ending with a single unconditional branch to the original target.
bool splitSwitchCases(Function &F) {
bool Modified = false;
>From 35efc274641c3fc4f544b4c6c59b742e941520bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Mon, 9 Sep 2024 11:46:16 +0200
Subject: [PATCH 3/7] remove splitSwitchCases step
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Nathan Gauër <brioche at google.com>
---
llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp | 41 ---------------------
1 file changed, 41 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 4dc2b26b64d56f..61afb4e4fe61fc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -1158,42 +1158,6 @@ class SPIRVStructurizer : public FunctionPass {
return Modified;
}
- // Makes sure every case target in |F| is unique. If 2 cases branch to the
- // same basic block, one of the targets is updated so it jumps to a new basic
- // block ending with a single unconditional branch to the original target.
- bool splitSwitchCases(Function &F) {
- bool Modified = false;
-
- for (BasicBlock &BB : F) {
- SwitchInst *SI = dyn_cast<SwitchInst>(BB.getTerminator());
- if (!SI)
- continue;
-
- BlockSet Seen;
- Seen.insert(SI->getDefaultDest());
-
- auto It = SI->case_begin();
- while (It != SI->case_end()) {
- BasicBlock *Target = It->getCaseSuccessor();
- if (Seen.count(Target) == 0) {
- Seen.insert(Target);
- ++It;
- continue;
- }
-
- Modified = true;
- BasicBlock *NewTarget =
- BasicBlock::Create(F.getContext(), "new.sw.case", &F);
- IRBuilder<> Builder(NewTarget);
- Builder.CreateBr(Target);
- SI->addCase(It->getCaseValue(), NewTarget);
- It = SI->removeCase(It);
- }
- }
-
- return Modified;
- }
-
bool IsRequiredForPhiNode(BasicBlock *BB) {
for (BasicBlock *Successor : successors(BB)) {
for (PHINode &Phi : Successor->phis()) {
@@ -1321,11 +1285,6 @@ class SPIRVStructurizer : public FunctionPass {
virtual bool runOnFunction(Function &F) override {
bool Modified = false;
- // In LLVM, Switches are allowed to have several cases branching to the same
- // basic block. In SPIR-V, each target must be a distrinct block. This
- // function makes sure each target is unique.
- Modified |= splitSwitchCases(F);
-
// LLVM allows conditional branches to have both side jumping to the same
// block. It also allows switched to have a single default, or just one
// case. Cleaning this up now.
>From 47fa67ad3ffea8c1d533b6aa2b7d10141d100fda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Mon, 9 Sep 2024 11:55:18 +0200
Subject: [PATCH 4/7] prevent multiple map lookups
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Nathan Gauër <brioche at google.com>
---
llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 61afb4e4fe61fc..8d5acde5d62943 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -620,14 +620,14 @@ class SPIRVStructurizer : public FunctionPass {
std::vector<Edge> Output;
for (auto &[Src, Dst] : Edges) {
- if (Seen.count(Src) == 0) {
- Seen.emplace(Src, Dst);
+ auto [iterator, inserted] = Seen.insert({Src, Dst});
+ if (inserted) {
Output.emplace_back(Src, Dst);
continue;
}
// The exact same edge was already seen. Ignoring.
- if (Seen[Src] == Dst)
+ if (iterator->second == Dst)
continue;
// The same Src block branches to 2 distinct blocks. This will be an
>From c913cfb3e81d5c31d9ce019fbe49210ad7ca45a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Mon, 9 Sep 2024 11:58:53 +0200
Subject: [PATCH 5/7] change set into vector
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Nathan Gauër <brioche at google.com>
---
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 4 +--
llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp | 34 ++++++---------------
2 files changed, 11 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index abaee954705c54..f1b10e264781f2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -781,12 +781,12 @@ static void processSwitchesConstants(MachineFunction &MF,
// Some instructions are used during CodeGen but should never be emitted.
// Cleaning up those.
static void cleanupHelperInstructions(MachineFunction &MF) {
- SmallPtrSet<MachineInstr *, 8> ToEraseMI;
+ SmallVector<MachineInstr *, 8> ToEraseMI;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (isSpvIntrinsic(MI, Intrinsic::spv_track_constant) ||
MI.getOpcode() == TargetOpcode::G_BRINDIRECT)
- ToEraseMI.insert(&MI);
+ ToEraseMI.push_back(&MI);
}
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 8d5acde5d62943..5346ad27bb5e45 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -491,26 +491,10 @@ class SPIRVStructurizer : public FunctionPass {
DT.recalculate(F);
}
- // Returns the list of blocks that belong to a SPIR-V continue construct.
- std::vector<BasicBlock *> getContinueConstructBlocks(BasicBlock *Header,
- BasicBlock *Continue) {
- std::vector<BasicBlock *> Output;
- Loop *L = LI.getLoopFor(Continue);
- assert(L->getLoopLatch() != nullptr);
-
- partialOrderVisit(*Continue, [&](BasicBlock *BB) {
- if (BB == Header)
- return false;
- Output.push_back(BB);
- return true;
- });
- return Output;
- }
-
- // Returns the list of blocks that belong to a SPIR-V loop construct.
+ // Returns the list of blocks that belong to a SPIR-V loop construct,
+ // including the continue construct.
std::vector<BasicBlock *> getLoopConstructBlocks(BasicBlock *Header,
- BasicBlock *Merge,
- BasicBlock *Continue) {
+ BasicBlock *Merge) {
assert(DT.dominates(Header, Merge));
std::vector<BasicBlock *> Output;
partialOrderVisit(*Header, [&](BasicBlock *BB) {
@@ -776,10 +760,11 @@ class SPIRVStructurizer : public FunctionPass {
auto *Merge = getExitFor(CR);
// We are indeed in a loop, but there are no exits (infinite loop).
- // TODO: I see no value in having real infinite loops in vulkan shaders.
- // For now, I need to create a Merge block, and a structurally reachable
- // block for it, but maybe we'd want to raise an error, as locking up the
- // system is probably not wanted.
+ // This could be caused by a bad shader, but also could be an artifact
+ // from an earlier optimization. It is not always clear if structurally
+ // reachable means runtime reachable, so we cannot error-out. What we must
+ // do however is to make is legal on the SPIR-V point of view, hence
+ // adding an unreachable merge block.
if (Merge == nullptr) {
BranchInst *Br = cast<BranchInst>(BB.getTerminator());
assert(cast<BranchInst>(BB.getTerminator())->isUnconditional());
@@ -1021,8 +1006,7 @@ class SPIRVStructurizer : public FunctionPass {
assert(Node->Header && Node->Merge);
if (Node->Continue) {
- auto LoopBlocks =
- S.getLoopConstructBlocks(Node->Header, Node->Merge, Node->Continue);
+ auto LoopBlocks = S.getLoopConstructBlocks(Node->Header, Node->Merge);
return BlockSet(LoopBlocks.begin(), LoopBlocks.end());
}
>From e3cc406a163e3f4fa5743c27b5eac9bec5b0da05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Mon, 9 Sep 2024 16:28:49 +0200
Subject: [PATCH 6/7] fix tests relying on spirv-as
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Nathan Gauër <brioche at google.com>
---
llvm/test/CMakeLists.txt | 1 +
llvm/test/CodeGen/SPIRV/lit.local.cfg | 13 +++++++++++++
.../SPIRV/structurizer/cf.switch.opswitch.hlsl | 2 +-
.../structurizer/cf.switch.opswitch.literal.hlsl | 2 +-
llvm/tools/spirv-tools/CMakeLists.txt | 16 +++++++++++++---
5 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 7c52dd51cea818..c66075434f1583 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -230,6 +230,7 @@ endif()
if (LLVM_INCLUDE_SPIRV_TOOLS_TESTS)
list(APPEND LLVM_TEST_DEPENDS spirv-dis)
list(APPEND LLVM_TEST_DEPENDS spirv-val)
+ list(APPEND LLVM_TEST_DEPENDS spirv-as)
endif()
add_custom_target(llvm-test-depends DEPENDS ${LLVM_TEST_DEPENDS})
diff --git a/llvm/test/CodeGen/SPIRV/lit.local.cfg b/llvm/test/CodeGen/SPIRV/lit.local.cfg
index 00f50fb6f1cff7..7613e937c111e2 100644
--- a/llvm/test/CodeGen/SPIRV/lit.local.cfg
+++ b/llvm/test/CodeGen/SPIRV/lit.local.cfg
@@ -1,7 +1,20 @@
if not "SPIRV" in config.root.targets:
config.unsupported = True
+config.suffixes = [".ll", ".hlsl"]
+
+spirv_sim_root = os.path.join(config.llvm_src_root, "utils", "spirv-sim")
+
if config.spirv_tools_tests:
config.available_features.add("spirv-tools")
config.substitutions.append(("spirv-dis", os.path.join(config.llvm_tools_dir, "spirv-dis")))
config.substitutions.append(("spirv-val", os.path.join(config.llvm_tools_dir, "spirv-val")))
+ config.substitutions.append(("spirv-as", os.path.join(config.llvm_tools_dir, "spirv-as")))
+ config.substitutions.append( ('%clang', 'clang'))
+ config.substitutions.append(
+ (
+ "spirv-sim",
+ "'%s' %s"
+ % (config.python_executable, os.path.join(spirv_sim_root, "spirv-sim.py")),
+ )
+ )
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl
index 19d889064077cd..9b91de9ef90ad8 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
int foo() { return 200; }
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl
index 09f8c2caae5f6f..207c9ddf8a40c2 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.opswitch.literal.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val
+// RUN: %if spirv-tools %{ %clang --driver-mode=dxc -T cs_6_0 -fspv-target-env=vulkan1.3 %s -spirv | spirv-as --preserve-numeric-ids - -o - | spirv-val %}
[numthreads(1, 1, 1)]
void main() {
diff --git a/llvm/tools/spirv-tools/CMakeLists.txt b/llvm/tools/spirv-tools/CMakeLists.txt
index c0d4556de4c62d..57dfe3310c4596 100644
--- a/llvm/tools/spirv-tools/CMakeLists.txt
+++ b/llvm/tools/spirv-tools/CMakeLists.txt
@@ -12,7 +12,7 @@ endif ()
# SPIRV_DIS and SPIRV_VAL variables can be used to provide paths to existing
# spirv-dis and spirv-val binaries, respectively. Otherwise, build them from
# SPIRV-Tools source.
-if (NOT SPIRV_DIS OR NOT SPIRV_VAL)
+if (NOT SPIRV_DIS OR NOT SPIRV_VAL OR NOT SPIRV_AS)
include(ExternalProject)
set(BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/SPIRVTools-bin)
@@ -21,8 +21,8 @@ if (NOT SPIRV_DIS OR NOT SPIRV_VAL)
GIT_REPOSITORY https://github.com/KhronosGroup/SPIRV-Tools.git
GIT_TAG main
BINARY_DIR ${BINARY_DIR}
- BUILD_COMMAND ${CMAKE_COMMAND} --build ${BINARY_DIR} --target spirv-dis spirv-val
- BUILD_BYPRODUCTS ${BINARY_DIR}/tools/spirv-dis ${BINARY_DIR}/tools/spirv-val
+ BUILD_COMMAND ${CMAKE_COMMAND} --build ${BINARY_DIR} --target spirv-dis spirv-val spirv-as
+ BUILD_BYPRODUCTS ${BINARY_DIR}/tools/spirv-dis ${BINARY_DIR}/tools/spirv-val ${BINARY_DIR}/tools/spirv-as
DOWNLOAD_COMMAND git clone https://github.com/KhronosGroup/SPIRV-Tools.git SPIRVTools &&
cd SPIRVTools &&
${Python3_EXECUTABLE} utils/git-sync-deps
@@ -63,3 +63,13 @@ else ()
DEPENDS SPIRVTools
)
endif ()
+
+if (SPIRV_AS)
+ add_custom_target(spirv-as
+ COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} "${SPIRV_VAL}" "${LLVM_RUNTIME_OUTPUT_INTDIR}/spirv-as")
+else ()
+ add_custom_target(spirv-as
+ COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} "${BINARY_DIR}/tools/spirv-as${CMAKE_EXECUTABLE_SUFFIX}" "${LLVM_RUNTIME_OUTPUT_INTDIR}/spirv-as${CMAKE_EXECUTABLE_SUFFIX}"
+ DEPENDS SPIRVTools
+ )
+endif ()
>From 949e0c5154987765796255b90ec524e79374577c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche at google.com>
Date: Mon, 9 Sep 2024 17:27:38 +0200
Subject: [PATCH 7/7] Revert "remove splitSwitchCases step"
This reverts commit 35efc274641c3fc4f544b4c6c59b742e941520bf.
---
llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp | 41 +++++++++++++++++++++
1 file changed, 41 insertions(+)
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 5346ad27bb5e45..94a40b4225548a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -1142,6 +1142,42 @@ class SPIRVStructurizer : public FunctionPass {
return Modified;
}
+ // Makes sure every case target in |F| is unique. If 2 cases branch to the
+ // same basic block, one of the targets is updated so it jumps to a new basic
+ // block ending with a single unconditional branch to the original target.
+ bool splitSwitchCases(Function &F) {
+ bool Modified = false;
+
+ for (BasicBlock &BB : F) {
+ SwitchInst *SI = dyn_cast<SwitchInst>(BB.getTerminator());
+ if (!SI)
+ continue;
+
+ BlockSet Seen;
+ Seen.insert(SI->getDefaultDest());
+
+ auto It = SI->case_begin();
+ while (It != SI->case_end()) {
+ BasicBlock *Target = It->getCaseSuccessor();
+ if (Seen.count(Target) == 0) {
+ Seen.insert(Target);
+ ++It;
+ continue;
+ }
+
+ Modified = true;
+ BasicBlock *NewTarget =
+ BasicBlock::Create(F.getContext(), "new.sw.case", &F);
+ IRBuilder<> Builder(NewTarget);
+ Builder.CreateBr(Target);
+ SI->addCase(It->getCaseValue(), NewTarget);
+ It = SI->removeCase(It);
+ }
+ }
+
+ return Modified;
+ }
+
bool IsRequiredForPhiNode(BasicBlock *BB) {
for (BasicBlock *Successor : successors(BB)) {
for (PHINode &Phi : Successor->phis()) {
@@ -1269,6 +1305,11 @@ class SPIRVStructurizer : public FunctionPass {
virtual bool runOnFunction(Function &F) override {
bool Modified = false;
+ // In LLVM, Switches are allowed to have several cases branching to the same
+ // basic block. This is allowed in SPIR-V, but can make structurizing SPIR-V
+ // harder, so first remove edge cases.
+ Modified |= splitSwitchCases(F);
+
// LLVM allows conditional branches to have both side jumping to the same
// block. It also allows switched to have a single default, or just one
// case. Cleaning this up now.
More information about the cfe-commits
mailing list