[clang] [CIR] Implement FlattenCFG for coroutine AwaitOp, CoroBodyOp, and CoReturnOp (PR #203802)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 14 21:29:14 PDT 2026
https://github.com/Andres-Salamanca updated https://github.com/llvm/llvm-project/pull/203802
>From f27a5a713ec4b6138262e4267bb4b72cd6930a72 Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Sun, 14 Jun 2026 17:32:20 -0500
Subject: [PATCH 01/10] [CIR] Implement FlattenCFG for coroutine AwaitOp,
CoroBodyOp, and CoReturnOp
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 9 +-
clang/include/clang/CIR/MissingFeatures.h | 1 +
.../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 215 ++++++++-
.../CIR/CodeGenCoroutines/coro-flatten.cpp | 437 ++++++++++++++++++
4 files changed, 658 insertions(+), 4 deletions(-)
create mode 100644 clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 879184c6b6a437..643bee9cbffa44 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -766,6 +766,10 @@ def CIR_AllocaOp : CIR_Op<"alloca", [
The `cleanup_dest_slot` attribute indicates that this was a temporary
alloca generated by the compiler to handle cleanup exit dispatching.
+ The `coroutine_suspend_point` attribute indicates that this alloca is
+ used to track the coroutine suspend destination. The use of this alloca
+ identifies where control should continue when the coroutine suspends.
+
The result type is a pointer to the input's type.
Example:
@@ -786,6 +790,7 @@ def CIR_AllocaOp : CIR_Op<"alloca", [
UnitAttr:$init,
UnitAttr:$constant,
UnitAttr:$cleanup_dest_slot,
+ UnitAttr:$coroutine_suspend_point,
ConfinedAttr<I64Attr, [IntMinValue<1>]>:$alignment,
OptionalAttr<CIR_AnnotationArrayAttr>:$annotations
);
@@ -824,7 +829,8 @@ def CIR_AllocaOp : CIR_Op<"alloca", [
`align` `(` $alignment `)`
oilist( `init` $init
| `const` $constant
- | `cleanup_dest_slot` $cleanup_dest_slot)
+ | `cleanup_dest_slot` $cleanup_dest_slot
+ | `coroutine_suspend_point` $coroutine_suspend_point)
(`size` `(` $dynAllocSize^ `)`)? `:` qualified(type($addr))
($annotations^)? attr-dict
}];
@@ -4366,6 +4372,7 @@ def CIR_FuncOp : CIR_Op<"func", [
UnitAttr:$builtin,
UnitAttr:$coroutine,
OptionalAttr<CIR_InlineKindAttr>:$inline_kind,
+ UnitAttr:$flatten_coroutine,
UnitAttr:$lambda,
UnitAttr:$no_proto,
UnitAttr:$dso_local,
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index cd1e43b17ec3f0..d1a7a409bff9f8 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -127,6 +127,7 @@ struct MissingFeatures {
// Coroutines
static bool coroAwaitFullExprCleanups() { return false; }
static bool coroOutsideFrameMD() { return false; }
+ static bool coroutineGroManager() { return false; };
// Various handling of deferred processing in CIRGenModule.
static bool cgmRelease() { return false; }
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index 4005124a054ac2..f44a6a5b404a23 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -1083,7 +1083,7 @@ class CIRCleanupScopeOpFlattening
} else if (isa<cir::LoopOpInterface>(nestedOp)) {
collectExitsInLoop(nestedOp);
return mlir::WalkResult::skip();
- } else if (isa<cir::ContinueOp>(nestedOp) ||
+ } else if (isa<cir::ContinueOp, cir::CoReturnOp>(nestedOp) ||
isReturnThatExitsCleanup(nestedOp)) {
exits.emplace_back(nestedOp, nextId++);
} else if (isGotoThatExitsCleanup(nestedOp)) {
@@ -1106,7 +1106,8 @@ class CIRCleanupScopeOpFlattening
// the nested cleanup.
if (!ignoreBreak && isa<cir::BreakOp>(op)) {
exits.emplace_back(op, nextId++);
- } else if (isa<cir::ContinueOp>(op) || isReturnThatExitsCleanup(op)) {
+ } else if (isa<cir::ContinueOp, cir::CoReturnOp>(op) ||
+ isReturnThatExitsCleanup(op)) {
exits.emplace_back(op, nextId++);
} else if (isGotoThatExitsCleanup(op)) {
exits.emplace_back(op, nextId++);
@@ -1262,6 +1263,13 @@ class CIRCleanupScopeOpFlattening
cir::ContinueOp::create(rewriter, loc);
return mlir::success();
})
+ .Case<cir::CoReturnOp>([&](auto) {
+ // CoReturnOp does not carry a destination operand. The continuation
+ // block determines whether execution proceeds to another cleanup or
+ // to the coroutine's final suspend path.
+ cir::CoReturnOp::create(rewriter, loc);
+ return mlir::success();
+ })
.Case<cir::ReturnOp>([&](auto returnOp) {
// Return from the cleanup exit. Note, if this is a return inside a
// nested cleanup scope, the flattening of the outer scope will handle
@@ -2110,11 +2118,212 @@ class CIRTryOpFlattening : public mlir::OpRewritePattern<cir::TryOp> {
}
};
+static mlir::Block *getOrCreateBlockForSuspendPoint(
+ cir::FuncOp funcOp, mlir::PatternRewriter &rewriter, mlir::Location loc) {
+ mlir::Block &entryBlock = funcOp.getBody().front();
+
+ auto it = llvm::find_if(entryBlock, [](auto &op) {
+ return mlir::isa<AllocaOp>(&op) &&
+ mlir::cast<AllocaOp>(&op).getCoroutineSuspendPoint();
+ });
+
+ assert(it->hasOneUse() &&
+ "coroutine suspend point alloca must have exactly one use");
+ auto storeOp = cast<cir::StoreOp>(*it->getUses().begin()->getOwner());
+ auto suspendPoint = cast<cir::ConstantOp>(storeOp.getValue().getDefiningOp());
+ mlir::Block *suspendBlock = suspendPoint->getBlock();
+ if (&suspendBlock->front() == suspendPoint)
+ return suspendBlock;
+
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ mlir::Block *remainingBlock =
+ rewriter.splitBlock(suspendBlock, suspendPoint->getIterator());
+ rewriter.setInsertionPointToEnd(suspendBlock);
+ cir::BrOp::create(rewriter, loc, remainingBlock);
+ return remainingBlock;
+}
+
+class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
+public:
+ using OpRewritePattern<cir::AwaitOp>::OpRewritePattern;
+
+ mlir::LogicalResult
+ matchAndRewrite(cir::AwaitOp awaitOp,
+ mlir::PatternRewriter &rewriter) const override {
+ mlir::Block *awaitBlock = rewriter.getInsertionBlock();
+ mlir::Block *remainingOpsBlock =
+ rewriter.splitBlock(awaitBlock, rewriter.getInsertionPoint());
+
+ mlir::Location loc = awaitOp.getLoc();
+
+ mlir::Region &readyRegion = awaitOp.getReady();
+ mlir::Block &beforeReady = awaitOp.getReady().front();
+ mlir::Region &suspendRegion = awaitOp.getSuspend();
+ mlir::Region &resumeRegion = awaitOp.getResume();
+ auto conditionOp =
+ cast<cir::ConditionOp>(readyRegion.back().getTerminator());
+ {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ rewriter.setInsertionPoint(conditionOp);
+ rewriter.replaceOpWithNewOp<cir::BrCondOp>(
+ conditionOp, conditionOp.getCondition(), &resumeRegion.front(),
+ &suspendRegion.front());
+ }
+ rewriter.inlineRegionBefore(readyRegion, remainingOpsBlock);
+
+ {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ rewriter.setInsertionPointToEnd(awaitBlock);
+ cir::BrOp::create(rewriter, loc, mlir::ValueRange(), &beforeReady);
+ }
+
+ auto suspendYield =
+ cast<cir::YieldOp>(suspendRegion.back().getTerminator());
+ cir::LLVMIntrinsicCallOp coroSuspendIntri = nullptr;
+ {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ rewriter.setInsertionPoint(&suspendRegion.front().front());
+
+ // Insert coro.save at the beginning of the suspend region.
+ // This captures the current coroutine state before suspension.
+ auto voidPtrTy = cir::PointerType::get(cir::VoidType::get(getContext()));
+ auto nullPtr = cir::ConstantOp::create(
+ rewriter, loc,
+ cir::ConstPtrAttr::get(voidPtrTy, rewriter.getI64IntegerAttr(0)));
+ auto coroSaveIntri = cir::LLVMIntrinsicCallOp::create(
+ rewriter, loc, mlir::StringAttr::get(getContext(), "llvm.coro.save"),
+ cir::IntType::get(getContext(), 32, false),
+ mlir::ValueRange{nullPtr});
+ rewriter.setInsertionPoint(suspendYield);
+
+ bool isFinalSuspend = awaitOp.getKind() == cir::AwaitKind::Final;
+ auto isFinalCoroSuspend = cir::ConstantOp::create(
+ rewriter, loc, cir::BoolAttr::get(getContext(), isFinalSuspend));
+
+ // llvm.coro.suspend returns:
+ // -1 : coroutine suspended
+ // 0 : coroutine resumed
+ // 1 : coroutine destroyed
+ coroSuspendIntri = cir::LLVMIntrinsicCallOp::create(
+ rewriter, loc,
+ mlir::StringAttr::get(getContext(), "llvm.coro.suspend"),
+ cir::IntType::get(getContext(), 32, false),
+ mlir::ValueRange{coroSaveIntri.getResult(), isFinalCoroSuspend});
+ }
+ rewriter.inlineRegionBefore(suspendRegion, remainingOpsBlock);
+
+ auto func = awaitOp->getParentOfType<cir::FuncOp>();
+
+ {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ rewriter.setInsertionPoint(suspendYield);
+ llvm::SmallVector<mlir::APInt, 2> caseValues{mlir::APInt(32, 0),
+ mlir::APInt(32, 1)};
+
+ llvm::SmallVector<mlir::ValueRange, 8> caseOperands{
+ mlir::ValueRange(), mlir::ValueRange(), mlir::ValueRange()};
+
+ llvm::SmallVector<mlir::Block *, 8> caseDestinations;
+
+ // In Classic CodeGen, the destroy path reaches the coroutine cleanup by
+ // emitting an EmitBranchThroughCleanup(), ensuring that all nested
+ // cleanup scopes are executed before control reaches the coro.free
+ // cleanup.
+ //
+ // We achieve the same effect by creating a block that only contains a
+ // cir.yield. createExitTerminator() then propagates control through every
+ // enclosing cleanup scope until the parent coroutine cleanup (coro.free)
+ // is reached, after which execution continues to the return block.
+ mlir::Block *cleanupBlock = nullptr;
+ {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ cleanupBlock = rewriter.createBlock(remainingOpsBlock);
+ cir::YieldOp::create(rewriter, loc);
+ }
+ caseDestinations.push_back(&resumeRegion.front());
+ caseDestinations.push_back(cleanupBlock);
+
+ assert(!cir::MissingFeatures::coroutineGroManager());
+
+ // Default destination must be de suspend BB (the return block or the pre
+ // gro conv)
+ auto coroSuspendSwitch = cir::SwitchFlatOp::create(
+ rewriter, loc, coroSuspendIntri.getResult(),
+ getOrCreateBlockForSuspendPoint(func, rewriter, loc),
+ mlir::ValueRange(), caseValues, caseDestinations, caseOperands);
+
+ rewriter.replaceOp(suspendYield, coroSuspendSwitch);
+ }
+
+ auto resumeYield = cast<cir::YieldOp>(resumeRegion.back().getTerminator());
+ {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ rewriter.setInsertionPoint(resumeYield);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(resumeYield, remainingOpsBlock);
+ }
+ rewriter.inlineRegionBefore(resumeRegion, remainingOpsBlock);
+
+ rewriter.eraseOp(awaitOp);
+
+ func.setCoroutine(false);
+ func.setFlattenCoroutine(true);
+
+ return mlir::success();
+ }
+};
+
+class CIRCoroBodyOpFlattening : public mlir::OpRewritePattern<cir::CoroBodyOp> {
+public:
+ using OpRewritePattern<cir::CoroBodyOp>::OpRewritePattern;
+
+ mlir::LogicalResult
+ matchAndRewrite(cir::CoroBodyOp coroBodyOp,
+ mlir::PatternRewriter &rewriter) const override {
+
+ if (hasNestedOpsToFlatten(coroBodyOp.getBody()))
+ return mlir::failure();
+
+ llvm::SmallVector<cir::CoReturnOp> coReturns;
+ coroBodyOp.getBody().walk<mlir::WalkOrder::PreOrder>(
+ [&](cir::CoReturnOp op) {
+ coReturns.push_back(op);
+ return mlir::WalkResult::advance();
+ });
+
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ mlir::Location loc = coroBodyOp.getLoc();
+
+ mlir::Block *currentBlock = rewriter.getInsertionBlock();
+ mlir::Block *continueBlock =
+ rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
+
+ // Inline body region.
+ mlir::Block *beforeBody = &coroBodyOp.getBody().front();
+ rewriter.inlineRegionBefore(coroBodyOp.getBody(), continueBlock);
+
+ rewriter.setInsertionPointToEnd(currentBlock);
+ cir::BrOp::create(rewriter, loc, mlir::ValueRange(), beforeBody);
+
+ // In CIR CodeGen, the operation following CoroBodyOp is always the
+ // final-suspend path. Therefore, the continuation block created by the
+ // split corresponds to the final suspend point.
+ for (cir::CoReturnOp &coReturn : coReturns) {
+ rewriter.setInsertionPoint(coReturn);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(coReturn, continueBlock);
+ }
+
+ rewriter.replaceOp(coroBodyOp, continueBlock->getArguments());
+
+ return mlir::success();
+ }
+};
+
void populateFlattenCFGPatterns(RewritePatternSet &patterns) {
patterns
.add<CIRIfFlattening, CIRLoopOpInterfaceFlattening, CIRScopeOpFlattening,
CIRSwitchOpFlattening, CIRTernaryOpFlattening,
- CIRCleanupScopeOpFlattening, CIRTryOpFlattening>(
+ CIRCleanupScopeOpFlattening, CIRTryOpFlattening,
+ CIRAwaitOpFlattening, CIRCoroBodyOpFlattening>(
patterns.getContext());
}
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp b/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
new file mode 100644
index 00000000000000..fc5ca2e4d43ba4
--- /dev/null
+++ b/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
@@ -0,0 +1,437 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: cir-opt --cir-flatten-cfg %t.cir -o %t.flat.cir
+// RUN: FileCheck --input-file=%t.flat.cir %s -check-prefix=CIR-FLAT
+
+#include "Inputs/coroutine.h"
+
+using VoidTask = folly::coro::Task<void>;
+
+VoidTask silly_task() {
+ co_await std::suspend_always();
+}
+
+// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z10silly_taskv
+
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[NullPtr:.*]] = cir.const #cir.ptr<null>
+// CIR-FLAT: %[[Align:.*]] = cir.const #cir.int<16>
+// CIR-FLAT: %[[CoroId:.*]] = cir.call @__builtin_coro_id(%[[Align]], %[[NullPtr]], %[[NullPtr]], %[[NullPtr]])
+// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
+// CIR-FLAT: %[[SavedFrameAddr:.*]] = cir.alloca "__coro_frame_addr"
+// CIR-FLAT: %[[SuspendAlwaysAddr:.*]] = cir.alloca "ref.tmp0"
+// CIR-FLAT: %[[ShouldAlloc:.*]] = cir.call @__builtin_coro_alloc(%[[CoroId]]) : (!u32i) -> !cir.bool
+// CIR-FLAT: cir.store %[[NullPtr]], %[[SavedFrameAddr]]
+// CIR-FLAT: cir.brcond %[[ShouldAlloc]] ^[[CORO_ALLOC:.*]], ^[[CORO_INIT:.*]]
+// CIR-FLAT: ^[[CORO_ALLOC]]:
+// CIR-FLAT: %[[CoroSize:.*]] = cir.call @__builtin_coro_size()
+// CIR-FLAT: %[[AllocAddr:.*]] = cir.call @_Znwm(%[[CoroSize]])
+// CIR-FLAT: cir.store %[[AllocAddr]], %[[SavedFrameAddr]] : !cir.ptr<!void>, !cir.ptr<!cir.ptr<!void>>
+// CIR-FLAT: cir.br ^[[CORO_INIT]]
+// CIR-FLAT: ^[[CORO_INIT]]:
+// CIR-FLAT: %[[LOAD_CORO_FRAME:.*]] = cir.load %[[SavedFrameAddr]]
+// CIR-FLAT: %[[CoroFrameAddr:.*]] = cir.call @__builtin_coro_begin(%[[CoroId]], %[[LOAD_CORO_FRAME]])
+// CIR-FLAT: cir.br ^[[INIT_AWAIT_READY:.*]]
+// CIR-FLAT: ^[[INIT_AWAIT_READY]]:
+// CIR-FLAT: %[[RetObj:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type17get_return_objectEv(%[[VoidPromisseAddr:.*]])
+// CIR-FLAT: cir.store {{.*}} %[[RetObj]], %[[VoidTaskAddr:.*]]
+// CIR-FLAT: %[[Tmp0:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(%[[VoidPromisseAddr]])
+// CIR-FLAT: cir.store {{.*}} %[[Tmp0]], %[[SuspendAlwaysAddr]]
+// CIR-FLAT: cir.br ^[[INIT_AWAIT_READY_CONT:.*]]
+// CIR-FLAT: ^[[INIT_AWAIT_READY_CONT]]:
+// CIR-FLAT: %[[ShouldSuspend:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]])
+// CIR-FLAT: cir.brcond %[[ShouldSuspend]] ^[[AWAIT_INIT_RESUME:.*]], ^[[AWAIT_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_SUSPEND]]: // pred: ^bb4
+// CIR-FLAT: %[[NULLPTR2:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
+
+// TODO (cir): We should use a cir.token or mlir.token instead of returning !u32i.
+// CIR-FLAT: %[[SAVE_TOKEN:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR2]] : (!cir.ptr<!void>) -> !u32i
+// CIR-FLAT: %27 = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%21)
+// CIR-FLAT: cir.store align(1) %27, %10
+// CIR-FLAT: %28 = cir.load align(1) %10
+// CIR-FLAT: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%9, %28)
+// CIR-FLAT: %29 = cir.load align(1) %9
+// CIR-FLAT: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%8, %29)
+// CIR-FLAT: %[[IS_FINAL_SUSPEND:.*]] = cir.const #false
+// CIR-FLAT: %[[SUSPEND_RESULT:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN]], %[[IS_FINAL_SUSPEND]]
+// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT]] : !u32i, ^[[CORO_RET:.*]] [
+// CIR-FLAT: 0: ^[[AWAIT_INIT_RESUME]],
+// CIR-FLAT: 1: ^[[INIT_CLEANUP_DESTROY:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[INIT_CLEANUP_DESTROY]]:
+// CIR-FLAT: cir.const #cir.int<0>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
+// CIR-FLAT: ^[[AWAIT_INIT_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]])
+// CIR-FLAT: cir.br ^[[CORO_BODY:.*]]
+
+// The remaining await operations follow the same lowering pattern as above,
+// so we only check a few key instructions here instead of matching the entire IR
+
+// CIR-FLAT: ^[[CORO_BODY:.*]]:
+// CIR-FLAT: cir.br ^[[USER_AWAIT:.*]]
+// CIR-FLAT: ^[[USER_AWAIT]]:
+// CIR-FLAT: cir.br ^[[USER_AWAIT_READY_CONT:.*]]
+// CIR-FLAT: ^[[USER_AWAIT_READY_CONT]]:
+// CIR-FLAT: %[[ShouldSuspend2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
+// CIR-FLAT: cir.brcond %[[ShouldSuspend2]] ^[[AWAIT_USER_RESUME:.*]], ^[[AWAIT_USER_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_USER_SUSPEND]]
+// CIR-FLAT: %[[NULLPTR3:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
+// CIR-FLAT: %[[SAVE_TOKEN2:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR3]]
+// CIR-FLAT: %[[IS_FINAL_SUSPEND2:.*]] = cir.const #false
+// CIR-FLAT: %[[SUSPEND_RESULT2:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN2]], %[[IS_FINAL_SUSPEND2]]
+// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT2]] : !u32i, ^[[CORO_RET]] [
+// CIR-FLAT: 0: ^[[AWAIT_USER_RESUME]],
+// CIR-FLAT: 1: ^[[user_cleanup_destroy:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[user_cleanup_destroy]]:
+// CIR-FLAT: cir.const #cir.int<1>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
+// CIR-FLAT: ^[[AWAIT_USER_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: cir.br ^[[CO_RETURN:.*]]
+// CIR-FLAT: ^[[CO_RETURN]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[VoidPromisseAddr]])
+
+// This corresponds to the implicit cir.co_return path, which exits the
+// cir.coro.body and branches to the final suspend block.
+// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
+// CIR-FLAT: ^[[FINAL_SUSPEND_BB]]:
+// CIR-FLAT: %[[final_suspend:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[VoidPromisseAddr]])
+// CIR-FLAT: cir.store {{.*}} %[[final_suspend]], %{{.*}}
+// CIR-FLAT: cir.br ^[[FINAL_AWAIT_READY_CONT:.*]]
+// CIR-FLAT: ^[[FINAL_AWAIT_READY_CONT]]:
+// CIR-FLAT: %[[ShouldSuspend3:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
+// CIR-FLAT: cir.brcond %[[ShouldSuspend3]] ^[[AWAIT_FINAL_RESUME:.*]], ^[[AWAIT_FINAL_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_FINAL_SUSPEND]]:
+// CIR-FLAT: %[[NULLPTR4:.*]] = cir.const #cir.ptr<null>
+// CIR-FLAT: %[[SAVE_TOKEN3:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR4]]
+// CIR-FLAT: %[[IS_FINAL_SUSPEND3:.*]] = cir.const #true
+// CIR-FLAT: %[[SUSPEND_RESULT3:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN3]], %[[IS_FINAL_SUSPEND3]]
+// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT3]] : !u32i, ^[[CORO_RET]] [
+// CIR-FLAT: 0: ^[[AWAIT_FINAL_RESUME]],
+// CIR-FLAT: 1: ^[[final_cleanup_destroy:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[final_cleanup_destroy]]:
+// CIR-FLAT: cir.const #cir.int<2>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
+// CIR-FLAT: ^[[AWAIT_FINAL_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: cir.br ^[[FIANL_CLEANUP_EXIT:.*]]
+// CIR-FLAT: ^[[FIANL_CLEANUP_EXIT]]:
+// CIR-FLAT: cir.const #cir.int<3> : !s32i
+// CIR-FLAT: cir.store %{{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
+
+// Check whether llvm.coro.free returned a non-null pointer, indicating
+// that the coroutine frame must be deallocated.
+// CIR-FLAT: ^[[CLEANUP_CORO_FREE]]:
+// CIR-FLAT: %[[SHOULD_FREE:.*]] = cir.call @__builtin_coro_free(%[[CoroId]], %[[CoroFrameAddr]])
+// CIR-FLAT: %[[NULLPTR5:.*]] = cir.const #cir.ptr<null>
+// CIR-FLAT: %[[NEEDS_FREE:.*]] = cir.cmp ne %[[SHOULD_FREE]], %[[NULLPTR5]]
+// CIR-FLAT: cir.brcond %[[NEEDS_FREE]] ^[[FREE_FRAME:.*]], ^[[EXIT_CLEANUP:.*]]
+// CIR-FLAT: ^[[FREE_FRAME]]:
+// CIR-FLAT: %[[CoroSize2:.*]] = cir.call @__builtin_coro_size() : () -> (!u64i {llvm.noundef})
+// CIR-FLAT: cir.call @_ZdlPvm(%[[SHOULD_FREE]], %[[CoroSize2]])
+// CIR-FLAT: cir.br ^[[EXIT_CLEANUP]]
+// CIR-FLAT: ^[[EXIT_CLEANUP]]:
+// CIR-FLAT: cir.br ^[[EXIT_CLEANUP_SWITCH:.*]]
+// CIR-FLAT: ^[[EXIT_CLEANUP_SWITCH]]:
+// CIR-FLAT: %[[LOAD_DEST_SLOT:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.switch.flat %[[LOAD_DEST_SLOT]] : !s32i, ^[[DEFAULT_EXIT:.*]] [
+// CIR-FLAT: 0: ^[[EXIT1:.*]],
+// CIR-FLAT: 1: ^[[EXIT2:.*]],
+// CIR-FLAT: 2: ^[[EXIT3:.*]],
+// CIR-FLAT: 3: ^[[EXIT4:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[EXIT1]]:
+// CIR-FLAT: cir.br ^[[TO_RET:.*]]
+// CIR-FLAT: ^[[EXIT2]]:
+// CIR-FLAT: cir.br ^[[TO_RET]]
+// CIR-FLAT: ^[[EXIT3]]:
+// CIR-FLAT: cir.br ^[[TO_RET]]
+// CIR-FLAT: ^[[EXIT4]]:
+// CIR-FLAT: cir.br ^[[TO_RET]]
+// CIR-FLAT: ^[[DEFAULT_EXIT]]:
+// CIR-FLAT: cir.unreachable
+// CIR-FLAT: ^[[TO_RET]]:
+// CIR-FLAT: cir.br ^[[CORO_RET]]
+// CIR-FLAT: ^[[CORO_RET]]:
+// CIR-FLAT: %58 = cir.const #cir.int<1> : !s32i
+// CIR-FLAT: cir.store %58, %5 : !s32i, !cir.ptr<!s32i>
+// CIR-FLAT: %[[NULLPTR6:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
+// CIR-FLAT: %[[UNWIND:.*]] = cir.const #false
+// CIR-FLAT: cir.call @__builtin_coro_end(%[[NULLPTR6]], %[[UNWIND]])
+// CIR-FLAT: cir.return
+// CIR-FLAT: }
+
+struct HasDtor {
+ ~HasDtor();
+};
+
+VoidTask silly_task_with_dtor() {
+ HasDtor local;
+ co_await std::suspend_always();
+}
+
+// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z20silly_task_with_dtorv
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[SuspendAlwaysAddr:.*]] = cir.alloca "ref.tmp0"
+
+// CIR-FLAT: %[[ShouldSuspend:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]])
+// CIR-FLAT: cir.brcond %[[ShouldSuspend]] ^[[AWAIT_INIT_RESUME:.*]], ^[[AWAIT_INIT_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_INIT_SUSPEND]]:
+// CIR-FLAT: %[[NULLPTR:.*]] = cir.const #cir.ptr<null>
+// CIR-FLAT: %[[SAVE_TOKEN:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR]]
+// CIR-FLAT: %[[IS_FINAL_SUSPEND:.*]] = cir.const #false
+// CIR-FLAT: %[[SUSPEND_RESULT:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN]], %[[IS_FINAL_SUSPEND]]
+// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT]] : !u32i, ^[[CORO_RET:.*]] [
+// CIR-FLAT: 0: ^[[AWAIT_INIT_RESUME]],
+// CIR-FLAT: 1: ^[[INIT_CLEANUP_DESTROY:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[INIT_CLEANUP_DESTROY]]:
+// CIR-FLAT: cir.const #cir.int<0>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
+// CIR-FLAT: ^[[AWAIT_INIT_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]])
+// CIR-FLAT: cir.br ^[[AWAIT_INIT_RESUME_CONT:.*]]
+// CIR-FLAT: ^[[AWAIT_INIT_RESUME_CONT]]:
+// CIR-FLAT: cir.br ^[[CORO_BODY:.*]]
+// CIR-FLAT: ^[[CORO_BODY]]:
+// CIR-FLAT: cir.br ^[[HAS_DTOR_CLEANUP_SCOPE:.*]]
+// CIR-FLAT: ^[[HAS_DTOR_CLEANUP_SCOPE]]:
+// CIR-FLAT: cir.br ^[[USER_AWAIT_READY:.*]]
+// CIR-FLAT: ^[[USER_AWAIT_READY]]:
+// CIR-FLAT: %[[ShouldSuspend2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
+// CIR-FLAT: cir.brcond %[[ShouldSuspend2]] ^[[AWAIT_USER_RESUME:.*]], ^[[AWAIT_USER_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_USER_SUSPEND]]:
+// CIR-FLAT: %[[NULLPTR2:.*]] = cir.const #cir.ptr<null>
+// CIR-FLAT: %[[SAVE_TOKEN2:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR2]]
+// CIR-FLAT: %[[IS_FINAL_SUSPEND2:.*]] = cir.const #false
+// CIR-FLAT: %[[SUSPEND_RESULT2:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN2]], %[[IS_FINAL_SUSPEND2]]
+
+// The destroy branch cannot jump directly to the coroutine cleanup.
+// Since this await is nested within the lifetime of a local HasDtor object,
+// destruction must first run HasDtor::~HasDtor() before proceeding to the
+// coroutine-frame cleanup.
+//
+// Destroy path:
+// USER_CLEANUP_DESTROY -> HasDtor::~HasDtor() -> coro.free cleanup
+
+// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT2]] : !u32i, ^[[CORO_RET]] [
+// CIR-FLAT: 0: ^[[AWAIT_USER_RESUME]],
+// CIR-FLAT: 1: ^[[USER_CLEANUP_DESTROY:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[USER_CLEANUP_DESTROY]]:
+// CIR-FLAT: cir.const #cir.int<0>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[HAS_DTR:.*]]
+// CIR-FLAT: ^[[AWAIT_USER_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: cir.br ^[[CO_RETURN:.*]]
+// CIR-FLAT: ^[[CO_RETURN]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR-FLAT: cir.const #cir.int<1>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[HAS_DTR]]
+// CIR-FLAT: ^[[HAS_DTR]]:
+// CIR-FLAT: cir.call @_ZN7HasDtorD1Ev
+// CIR-FLAT: cir.br ^[[HAS_DTR_CONT:.*]]
+// CIR-FLAT: ^[[HAS_DTR_CONT]]:
+// CIR-FLAT: %[[LOAD_DEST_SLOT:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.switch.flat %[[LOAD_DEST_SLOT]] : !s32i, ^[[DEFAULT:.*]] [
+// CIR-FLAT: 0: ^[[FROM_USER_CLEANUP_DESTROY:.*]],
+// CIR-FLAT: 1: ^[[FROM_CO_RETURN:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[FROM_USER_CLEANUP_DESTROY]]:
+// CIR-FLAT: cir.br ^[[EXIT_TO_CORO_FREE:.*]]
+// CIR-FLAT: ^[[FROM_CO_RETURN]]:
+// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
+// CIR-FLAT: ^[[DEFAULT]]:
+// CIR-FLAT: cir.unreachable
+// CIR-FLAT: ^[[EXIT_TO_CORO_FREE]]:
+// CIR-FLAT: cir.const #cir.int<1>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
+// CIR-FLAT: ^[[FINAL_SUSPEND_BB:.*]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv
+// CIR-FLAT: cir.br ^[[FINAL_AWAIT_READY_CONT:.*]]
+// CIR-FLAT: cir.call @_ZNSt14suspend_always11await_readyEv
+// CIR-FLAT: cir.brcond {{.*}} ^[[AWAIT_FINAL_RESUME:.*]], ^[[AWAIT_FINAL_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_FINAL_SUSPEND]]:
+// CIR-FLAT: cir.call_llvm_intrinsic "llvm.coro.save"
+// CIR-FLAT: cir.const #true
+// CIR-FLAT: %[[SUSPEND_RESULT3:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend"
+// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT3]] : !u32i, ^[[CORO_RET]] [
+// CIR-FLAT: 0: ^[[AWAIT_FINAL_RESUME]],
+// CIR-FLAT: 1: ^[[FINAL_CLEANUP_DESTROY:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[FINAL_CLEANUP_DESTROY]]:
+// CIR-FLAT: cir.const #cir.int<2>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^bb28
+// CIR-FLAT: ^[[AWAIT_FINAL_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: cir.br ^[[CORO_BODY_EXIT:.*]]
+// CIR-FLAT: ^[[CORO_BODY_EXIT]]:
+// CIR-FLAT: cir.const #cir.int<3> : !s32i
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
+// CIR-FLAT: ^[[CLEANUP_CORO_FREE]]:
+// CIR-FLAT: %[[SHOULD_FREE:.*]] = cir.call @__builtin_coro_free
+// CIR-FLAT: %[[NULLPTR5:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
+// CIR-FLAT: %[[NEEDS_FREE:.*]] = cir.cmp ne %[[SHOULD_FREE]], %[[NULLPTR5]]
+// CIR-FLAT: cir.brcond %[[NEEDS_FREE]] ^[[FREE_FRAME:.*]], ^[[EXIT_CLEANUP:.*]]
+// CIR-FLAT: ^[[FREE_FRAME]]:
+// CIR-FLAT: %60 = cir.call @__builtin_coro_size() : () -> (!u64i {llvm.noundef})
+// CIR-FLAT: cir.call @_ZdlPvm(%57, %60) nothrow : (!cir.ptr<!void> {llvm.noundef}, !u64i {llvm.noundef}) -> ()
+// CIR-FLAT: cir.br ^[[EXIT_CLEANUP]]
+// CIR-FLAT: ^[[EXIT_CLEANUP]]:
+// CIR-FLAT: cir.br ^[[EXIT_CLEANUP_SWITCH:.*]]
+// CIR-FLAT: ^[[EXIT_CLEANUP_SWITCH]]:
+// CIR-FLAT: %[[LOAD_DEST_SLOT:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.switch.flat %[[LOAD_DEST_SLOT:.*]]
+// CIR-FLAT: ^[[CORO_RET]]:
+// CIR-FLAT: cir.return
+
+folly::coro::Task<int> co_returns(int flag) {
+ if (flag == 1) {
+ co_return 1;
+ } else if (flag == 2) {
+ co_return 2;
+ }
+ co_return 3;
+}
+
+// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z10co_returnsi
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
+
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: %[[LOAD_FLAG:.*]] = cir.load {{.*}} %[[FLAG:.*]]
+// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CIR-FLAT: %[[EQ_ONE:.*]] = cir.cmp eq %[[LOAD_FLAG]], %[[ONE]]
+// CIR-FLAT: cir.brcond %[[EQ_ONE]] ^[[IF_EQ_ONE:.*]], ^[[ELSE_IF:.*]]
+// CIR-FLAT: ^[[IF_EQ_ONE]]:
+// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[ONE]])
+// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
+// CIR-FLAT: ^[[ELSE_IF]]:
+// CIR-FLAT: cir.br ^[[ELSE_IF_CONT:.*]]
+// CIR-FLAT: ^[[ELSE_IF_CONT:.*]]:
+// CIR-FLAT: %[[LOAD_FLAG2:.*]] = cir.load {{.*}} %[[FLAG]]
+// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CIR-FLAT: %[[EQ_TWO:.*]] = cir.cmp eq %[[LOAD_FLAG2]], %[[TWO]]
+// CIR-FLAT: cir.brcond %[[EQ_TWO]] ^[[IF_EQ_TWO:.*]], ^[[ELSE:.*]]
+// CIR-FLAT: ^[[IF_EQ_TWO]]:
+// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[TWO]])
+// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB]]
+// CIR-FLAT: ^[[ELSE]]:
+// CIR-FLAT: cir.br ^[[ELSE_CONT:.*]]
+// CIR-FLAT: ^[[ELSE_CONT]]:
+// CIR-FLAT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[THREE]])
+// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB]]
+// CIR-FLAT: ^[[FINAL_SUSPEND_BB]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv
+
+
+
+folly::coro::Task<int> co_return_with_dtor(int flag) {
+ HasDtor local;
+ if (flag)
+ co_return 1; // local dtor must run here
+ co_return 2;
+}
+
+// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z19co_return_with_dtori
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
+// CIR-FLAT: %[[ShouldSuspend:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
+// CIR-FLAT: cir.brcond %[[ShouldSuspend]] ^[[AWAIT_INIT_RESUME:.*]], ^[[AWAIT_INIT_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_INIT_SUSPEND]]:
+// CIR-FLAT: cir.switch.flat {{.*}} : !u32i, ^[[CORO_RET:.*]] [
+// CIR-FLAT: 0: ^[[AWAIT_INIT_RESUME]],
+// CIR-FLAT: 1: ^[[INIT_CLEANUP_DESTROY:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[INIT_CLEANUP_DESTROY]]:
+// CIR-FLAT: cir.const #cir.int<0> : !s32i
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
+// CIR-FLAT: ^[[AWAIT_INIT_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%10)
+// CIR-FLAT: cir.br ^[[CORO_BODY:.*]]
+// CIR-FLAT: ^[[CORO_BODY]]:
+// CIR-FLAT: cir.br ^[[HAS_DTR_CLEANUP_SCOPE:.*]]
+// CIR-FLAT: ^[[HAS_DTR_CLEANUP_SCOPE]]:
+// CIR-FLAT: cir.br ^[[IF_SCOPE:.*]]
+// CIR-FLAT: ^[[IF_SCOPE]]:
+// CIR-FLAT: cir.br ^[[IF_SCOPE_CONT:.*]]
+// CIR-FLAT: ^[[IF_SCOPE_CONT]]:
+// CIR-FLAT: %[[LOAD_FLAG:.*]] = cir.load {{.*}} %[[FLAG_ARG:.*]]
+// CIR-FLAT: %[[CAST_TO_BOOL:.*]] = cir.cast int_to_bool %[[LOAD_FLAG]]
+// CIR-FLAT: cir.brcond %[[CAST_TO_BOOL]] ^[[IF_BODY:.*]], ^[[IF_CONT:.*]]
+// CIR-FLAT: ^[[IF_BODY]]:
+// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[ONE]])
+// CIR-FLAT: cir.const #cir.int<0>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+
+// This cir.co_return exits through the cleanup path. Control is transferred
+// to the destructor cleanup block before reaching the final suspend point.
+// co_return -> cleanup -> final suspend
+// CIR-FLAT: cir.br ^[[HAS_DTOR_CLEANUP:.*]]
+// CIR-FLAT: ^[[IF_CONT]]:
+// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[TWO]])
+// CIR-FLAT: cir.const #cir.int<1>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+
+// This cir.co_return exits through the cleanup path. Control is transferred
+// to the destructor cleanup block before reaching the final suspend point.
+// co_return -> cleanup -> final suspend
+// CIR-FLAT: cir.br ^[[HAS_DTOR_CLEANUP]]
+// CIR-FLAT: ^[[HAS_DTOR_CLEANUP]]:
+// CIR-FLAT: cir.call @_ZN7HasDtorD1Ev
+// CIR-FLAT: cir.br ^[[CLEANUP_EXIT:.*]]
+// CIR-FLAT: ^[[CLEANUP_EXIT]]:
+// CIR-FLAT: %[[LOAD_CLEANUP:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.switch.flat %[[LOAD_CLEANUP:.*]] : !s32i, ^[[DEFAULT:.*]] [
+// CIR-FLAT: 0: ^[[EXIT1:.*]],
+// CIR-FLAT: 1: ^[[EXIT2:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[EXIT1]]:
+// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
+// CIR-FLAT: ^[[EXIT2]]:
+// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB]]
+// CIR-FLAT: ^[[DEFAULT]]:
+// CIR-FLAT: cir.unreachable
+// CIR-FLAT: ^[[FINAL_SUSPEND_BB]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv
+// CIR-FLAT: %[[ShouldSuspend2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
+// CIR-FLAT: cir.brcond %[[ShouldSuspend2]] ^[[AWAIT_FINAL_RESUME:.*]], ^[[AWAIT_FINAL_SUSPEND:.*]]
+// CIR-FLAT: ^[[AWAIT_FINAL_SUSPEND]]:
+// CIR-FLAT: %[[IS_FINAL_SUSPEND:.*]] = cir.const #true
+// CIR-FLAT: cir.call_llvm_intrinsic "llvm.coro.suspend" {{.*}}, %[[IS_FINAL_SUSPEND]]
+// CIR-FLAT: cir.switch.flat {{.*}} : !u32i, ^[[CORO_RET]] [
+// CIR-FLAT: 0: ^[[AWAIT_FINAL_RESUME]],
+// CIR-FLAT: 1: ^[[FINAL_CLEANUP_DESTROY:.*]]
+// CIR-FLAT: ]
+// CIR-FLAT: ^[[FINAL_CLEANUP_DESTROY]]:
+// CIR-FLAT: cir.const #cir.int<2>
+// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
+// CIR-FLAT: ^[[AWAIT_FINAL_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+
+// CIR-FLAT: ^[[CLEANUP_CORO_FREE]]:
+// CIR-FLAT: cir.call @__builtin_coro_free
+// CIR-FLAT: cir.const #cir.ptr<null>
+// CIR-FLAT: cir.brcond %{{.*}} ^[[FREE_FRAME:.*]], ^[[EXIT_CLEANUP:.*]]
+// CIR-FLAT: ^[[FREE_FRAME]]:
+// CIR-FLAT: cir.call @__builtin_coro_size()
+// CIR-FLAT: cir.call @_ZdlPvm
>From 26843365e6d5df41a7a860159bb2069e05a76498 Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Tue, 16 Jun 2026 21:02:57 -0500
Subject: [PATCH 02/10] Address some review comments
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 4 +-
clang/lib/CIR/Dialect/IR/CIRDialect.cpp | 9 +++
.../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 67 +++++++++++++++----
.../CIR/CodeGenCoroutines/coro-flatten.cpp | 8 +--
4 files changed, 69 insertions(+), 19 deletions(-)
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 643bee9cbffa44..fa3338dfd673a0 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -4371,8 +4371,8 @@ def CIR_FuncOp : CIR_Op<"func", [
TypeAttrOf<CIR_FuncType>:$function_type,
UnitAttr:$builtin,
UnitAttr:$coroutine,
- OptionalAttr<CIR_InlineKindAttr>:$inline_kind,
- UnitAttr:$flatten_coroutine,
+ UnitAttr:$flattened_coroutine,
+ OptionalAttr<CIR_InlineKind>:$inline_kind,
UnitAttr:$lambda,
UnitAttr:$no_proto,
UnitAttr:$dso_local,
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 3088a39e5a4d0a..1293d978dfd445 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -2844,6 +2844,8 @@ ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) {
mlir::StringAttr builtinNameAttr = getBuiltinAttrName(state.name);
mlir::StringAttr coroutineNameAttr = getCoroutineAttrName(state.name);
+ mlir::StringAttr flattenedCoroutineNameAttr =
+ getFlattenedCoroutineAttrName(state.name);
mlir::StringAttr inlineKindNameAttr = getInlineKindAttrName(state.name);
mlir::StringAttr lambdaNameAttr = getLambdaAttrName(state.name);
mlir::StringAttr noProtoNameAttr = getNoProtoAttrName(state.name);
@@ -2861,6 +2863,10 @@ ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) {
parser.parseOptionalKeyword(coroutineNameAttr.strref())))
state.addAttribute(coroutineNameAttr, parser.getBuilder().getUnitAttr());
+ if (::mlir::succeeded(
+ parser.parseOptionalKeyword(flattenedCoroutineNameAttr.strref())))
+ state.addAttribute(flattenedCoroutineNameAttr,
+ parser.getBuilder().getUnitAttr());
// Parse optional inline kind attribute
cir::InlineKindAttr inlineKindAttr;
if (failed(parseInlineKindAttr(parser, inlineKindAttr)))
@@ -3206,6 +3212,9 @@ void cir::FuncOp::print(OpAsmPrinter &p) {
if (getCoroutine())
p << " coroutine";
+ if (getFlattenedCoroutine())
+ p << " flattened_coroutine";
+
printInlineKindAttr(p, getInlineKindAttr());
if (getLambda())
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index f44a6a5b404a23..b3261c4fd55ad9 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -2147,10 +2147,37 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
public:
using OpRewritePattern<cir::AwaitOp>::OpRewritePattern;
+ // Flatten `cir.await` into the following control flow:
+ //
+ // awaitBlock
+ // |
+ // v
+ // ready
+ // / \
+ // / \
+ // v v
+ // resume suspend
+ // | |
+ // | llvm.coro.suspend
+ // | |
+ // | +----+----+
+ // | | | |
+ // | | | v
+ // | | | suspend point
+ // | | |
+ // | | v
+ // | | destroy
+ // | |
+ // +--> resume
+ // |
+ // v
+ // continuation
mlir::LogicalResult
matchAndRewrite(cir::AwaitOp awaitOp,
mlir::PatternRewriter &rewriter) const override {
mlir::Block *awaitBlock = rewriter.getInsertionBlock();
+ // Split the current block before the AwaitOp to create the inlining
+ // point.
mlir::Block *remainingOpsBlock =
rewriter.splitBlock(awaitBlock, rewriter.getInsertionPoint());
@@ -2165,6 +2192,9 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(conditionOp);
+ // The condition is the result of `await_ready()`. If true, execution
+ // continues in the resume region otherwise, control transfers to the
+ // suspend region.
rewriter.replaceOpWithNewOp<cir::BrCondOp>(
conditionOp, conditionOp.getCondition(), &resumeRegion.front(),
&suspendRegion.front());
@@ -2174,6 +2204,8 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPointToEnd(awaitBlock);
+ // After inlining the ready region, branch from the original await block
+ // to the beginning of the inlined ready-region
cir::BrOp::create(rewriter, loc, mlir::ValueRange(), &beforeReady);
}
@@ -2190,12 +2222,16 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
auto nullPtr = cir::ConstantOp::create(
rewriter, loc,
cir::ConstPtrAttr::get(voidPtrTy, rewriter.getI64IntegerAttr(0)));
+ auto int32Ty = cir::IntType::get(getContext(), 32, false);
auto coroSaveIntri = cir::LLVMIntrinsicCallOp::create(
- rewriter, loc, mlir::StringAttr::get(getContext(), "llvm.coro.save"),
- cir::IntType::get(getContext(), 32, false),
+ rewriter, loc, rewriter.getStringAttr("llvm.coro.save"), int32Ty,
mlir::ValueRange{nullPtr});
rewriter.setInsertionPoint(suspendYield);
+ // The second argument to `llvm.coro.suspend` indicates whether this is
+ // the final suspend point. Coroutines suspended at a final suspend point
+ // are considered done (`llvm.coro.done` returns true) and may only be
+ // destroyed resuming them is undefined behavior.
bool isFinalSuspend = awaitOp.getKind() == cir::AwaitKind::Final;
auto isFinalCoroSuspend = cir::ConstantOp::create(
rewriter, loc, cir::BoolAttr::get(getContext(), isFinalSuspend));
@@ -2206,8 +2242,7 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
// 1 : coroutine destroyed
coroSuspendIntri = cir::LLVMIntrinsicCallOp::create(
rewriter, loc,
- mlir::StringAttr::get(getContext(), "llvm.coro.suspend"),
- cir::IntType::get(getContext(), 32, false),
+ mlir::StringAttr::get(getContext(), "llvm.coro.suspend"), int32Ty,
mlir::ValueRange{coroSaveIntri.getResult(), isFinalCoroSuspend});
}
rewriter.inlineRegionBefore(suspendRegion, remainingOpsBlock);
@@ -2217,13 +2252,10 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(suspendYield);
- llvm::SmallVector<mlir::APInt, 2> caseValues{mlir::APInt(32, 0),
- mlir::APInt(32, 1)};
-
- llvm::SmallVector<mlir::ValueRange, 8> caseOperands{
- mlir::ValueRange(), mlir::ValueRange(), mlir::ValueRange()};
+ llvm::SmallVector<mlir::APInt> caseValues{mlir::APInt(32, 0),
+ mlir::APInt(32, 1)};
- llvm::SmallVector<mlir::Block *, 8> caseDestinations;
+ llvm::SmallVector<mlir::ValueRange> caseOperands{3};
// In Classic CodeGen, the destroy path reaches the coroutine cleanup by
// emitting an EmitBranchThroughCleanup(), ensuring that all nested
@@ -2240,13 +2272,17 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
cleanupBlock = rewriter.createBlock(remainingOpsBlock);
cir::YieldOp::create(rewriter, loc);
}
+ // The destroy case dispatches to cleanupBlock, which propagates through
+ // enclosing cleanup scopes before reaching the coroutine-frame cleanup.
+ llvm::SmallVector<mlir::Block *> caseDestinations{&resumeRegion.front(),
+ cleanupBlock};
caseDestinations.push_back(&resumeRegion.front());
caseDestinations.push_back(cleanupBlock);
assert(!cir::MissingFeatures::coroutineGroManager());
- // Default destination must be de suspend BB (the return block or the pre
- // gro conv)
+ // Dispatch to the appropriate coroutine path based on the result of
+ // `llvm.coro.suspend`: resume, destroy, or suspend.
auto coroSuspendSwitch = cir::SwitchFlatOp::create(
rewriter, loc, coroSuspendIntri.getResult(),
getOrCreateBlockForSuspendPoint(func, rewriter, loc),
@@ -2259,14 +2295,19 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(resumeYield);
+ // Once the coroutine resumes, continue with the operations that
+ // originally followed the await.
rewriter.replaceOpWithNewOp<cir::BrOp>(resumeYield, remainingOpsBlock);
}
rewriter.inlineRegionBefore(resumeRegion, remainingOpsBlock);
rewriter.eraseOp(awaitOp);
+ // The coroutine regions have been flattened, so the original coroutine
+ // attribute no longer verifies. Preserve the information that this function
+ // originated from a coroutine for LLVM lowering.
func.setCoroutine(false);
- func.setFlattenCoroutine(true);
+ func.setFlattenedCoroutine(true);
return mlir::success();
}
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp b/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
index fc5ca2e4d43ba4..58babff2760ddc 100644
--- a/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
+++ b/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
@@ -10,7 +10,7 @@ VoidTask silly_task() {
co_await std::suspend_always();
}
-// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z10silly_taskv
+// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z10silly_taskv
// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
// CIR-FLAT: %[[NullPtr:.*]] = cir.const #cir.ptr<null>
@@ -176,7 +176,7 @@ VoidTask silly_task_with_dtor() {
co_await std::suspend_always();
}
-// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z20silly_task_with_dtorv
+// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z20silly_task_with_dtorv
// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
// CIR-FLAT: %[[SuspendAlwaysAddr:.*]] = cir.alloca "ref.tmp0"
@@ -306,7 +306,7 @@ folly::coro::Task<int> co_returns(int flag) {
co_return 3;
}
-// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z10co_returnsi
+// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z10co_returnsi
// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
@@ -348,7 +348,7 @@ folly::coro::Task<int> co_return_with_dtor(int flag) {
co_return 2;
}
-// CIR-FLAT: cir.func flatten-coroutine {{.*}} @_Z19co_return_with_dtori
+// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z19co_return_with_dtori
// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
// CIR-FLAT: %[[ShouldSuspend:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
>From 9e6874fb25d35d7505df35e556006b9839afce2f Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Thu, 1 Oct 2026 21:12:20 -0500
Subject: [PATCH 03/10] rebase and remove coroBody
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 6 +-
clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp | 2 +
.../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 68 +------------------
3 files changed, 6 insertions(+), 70 deletions(-)
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index fa3338dfd673a0..d8f04e819bfdbb 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -790,7 +790,6 @@ def CIR_AllocaOp : CIR_Op<"alloca", [
UnitAttr:$init,
UnitAttr:$constant,
UnitAttr:$cleanup_dest_slot,
- UnitAttr:$coroutine_suspend_point,
ConfinedAttr<I64Attr, [IntMinValue<1>]>:$alignment,
OptionalAttr<CIR_AnnotationArrayAttr>:$annotations
);
@@ -829,8 +828,7 @@ def CIR_AllocaOp : CIR_Op<"alloca", [
`align` `(` $alignment `)`
oilist( `init` $init
| `const` $constant
- | `cleanup_dest_slot` $cleanup_dest_slot
- | `coroutine_suspend_point` $coroutine_suspend_point)
+ | `cleanup_dest_slot` $cleanup_dest_slot)
(`size` `(` $dynAllocSize^ `)`)? `:` qualified(type($addr))
($annotations^)? attr-dict
}];
@@ -4372,7 +4370,7 @@ def CIR_FuncOp : CIR_Op<"func", [
UnitAttr:$builtin,
UnitAttr:$coroutine,
UnitAttr:$flattened_coroutine,
- OptionalAttr<CIR_InlineKind>:$inline_kind,
+ OptionalAttr<CIR_InlineKindAttr>:$inline_kind,
UnitAttr:$lambda,
UnitAttr:$no_proto,
UnitAttr:$dso_local,
diff --git a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
index f432788242043a..4c9878a68e8c6e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
@@ -566,6 +566,8 @@ CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) {
// Builds `exit`: coro.end(/*unwind*/ false) followed by the actual return
// to the caller.
+
+ assert(!cir::MissingFeatures::coroutineGroManager());
auto exitBuilder = [&]() {
cir::ConstantOp nullHandler =
builder.getNullPtr(builder.getVoidPtrTy(), openCurlyLoc);
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index b3261c4fd55ad9..d3cd0145221ee3 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -2122,25 +2122,7 @@ static mlir::Block *getOrCreateBlockForSuspendPoint(
cir::FuncOp funcOp, mlir::PatternRewriter &rewriter, mlir::Location loc) {
mlir::Block &entryBlock = funcOp.getBody().front();
- auto it = llvm::find_if(entryBlock, [](auto &op) {
- return mlir::isa<AllocaOp>(&op) &&
- mlir::cast<AllocaOp>(&op).getCoroutineSuspendPoint();
- });
-
- assert(it->hasOneUse() &&
- "coroutine suspend point alloca must have exactly one use");
- auto storeOp = cast<cir::StoreOp>(*it->getUses().begin()->getOwner());
- auto suspendPoint = cast<cir::ConstantOp>(storeOp.getValue().getDefiningOp());
- mlir::Block *suspendBlock = suspendPoint->getBlock();
- if (&suspendBlock->front() == suspendPoint)
- return suspendBlock;
-
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- mlir::Block *remainingBlock =
- rewriter.splitBlock(suspendBlock, suspendPoint->getIterator());
- rewriter.setInsertionPointToEnd(suspendBlock);
- cir::BrOp::create(rewriter, loc, remainingBlock);
- return remainingBlock;
+return {};
}
class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
@@ -2313,58 +2295,12 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
}
};
-class CIRCoroBodyOpFlattening : public mlir::OpRewritePattern<cir::CoroBodyOp> {
-public:
- using OpRewritePattern<cir::CoroBodyOp>::OpRewritePattern;
-
- mlir::LogicalResult
- matchAndRewrite(cir::CoroBodyOp coroBodyOp,
- mlir::PatternRewriter &rewriter) const override {
-
- if (hasNestedOpsToFlatten(coroBodyOp.getBody()))
- return mlir::failure();
-
- llvm::SmallVector<cir::CoReturnOp> coReturns;
- coroBodyOp.getBody().walk<mlir::WalkOrder::PreOrder>(
- [&](cir::CoReturnOp op) {
- coReturns.push_back(op);
- return mlir::WalkResult::advance();
- });
-
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- mlir::Location loc = coroBodyOp.getLoc();
-
- mlir::Block *currentBlock = rewriter.getInsertionBlock();
- mlir::Block *continueBlock =
- rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
-
- // Inline body region.
- mlir::Block *beforeBody = &coroBodyOp.getBody().front();
- rewriter.inlineRegionBefore(coroBodyOp.getBody(), continueBlock);
-
- rewriter.setInsertionPointToEnd(currentBlock);
- cir::BrOp::create(rewriter, loc, mlir::ValueRange(), beforeBody);
-
- // In CIR CodeGen, the operation following CoroBodyOp is always the
- // final-suspend path. Therefore, the continuation block created by the
- // split corresponds to the final suspend point.
- for (cir::CoReturnOp &coReturn : coReturns) {
- rewriter.setInsertionPoint(coReturn);
- rewriter.replaceOpWithNewOp<cir::BrOp>(coReturn, continueBlock);
- }
-
- rewriter.replaceOp(coroBodyOp, continueBlock->getArguments());
-
- return mlir::success();
- }
-};
-
void populateFlattenCFGPatterns(RewritePatternSet &patterns) {
patterns
.add<CIRIfFlattening, CIRLoopOpInterfaceFlattening, CIRScopeOpFlattening,
CIRSwitchOpFlattening, CIRTernaryOpFlattening,
CIRCleanupScopeOpFlattening, CIRTryOpFlattening,
- CIRAwaitOpFlattening, CIRCoroBodyOpFlattening>(
+ CIRAwaitOpFlattening>(
patterns.getContext());
}
>From e15829b02c2c09597f82ba95a8e664348898a9b9 Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Tue, 13 Oct 2026 22:53:56 -0500
Subject: [PATCH 04/10] [CIR] Implement flattenCFG for cir.await and
cir.coroutine
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 31 +
clang/include/clang/CIR/MissingFeatures.h | 1 +
.../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 301 +++--
.../CIR/CodeGenCoroutines/coro-flatten.cpp | 437 ------
.../test/CIR/CodeGenCoroutines/coro-task.cpp | 1185 ++++++++++++++++-
.../test/CIR/Transforms/flatten-coroutine.cir | 226 ++++
6 files changed, 1653 insertions(+), 528 deletions(-)
delete mode 100644 clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
create mode 100644 clang/test/CIR/Transforms/flatten-coroutine.cir
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index d8f04e819bfdbb..8c52638ca4cc91 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -5272,6 +5272,37 @@ def CIR_CoroSuspendOp : CIR_CoroIntrinsicOp<"suspend",
let llvmOp = "CoroSuspendOp";
}
+//===----------------------------------------------------------------------===//
+// Coroutine intrinsic SaveOp
+//===----------------------------------------------------------------------===//
+
+def CIR_CoroSaveOp : CIR_CoroIntrinsicOp<"save",
+ (ins CIR_VoidPtrType:$coroaddr),
+ (outs Token:$result), [TokenProducerTrait]> {
+ let summary = "Represents llvm.coro.save";
+ let description = [{
+ `cir.coro.intrinsic.save` represents the `llvm.coro.save` intrinsic.
+
+ It marks the point where a coroutine needs to update its state to
+ prepare for resumption, so that it is considered suspended (and thus
+ eligible for resumption).
+ }];
+ let llvmOp = "CoroSaveOp";
+}
+
+def CIR_CoroExitOp : CIR_Op<"coro.exit", [
+ Terminator
+]> {
+ let summary = "Branches to the enclosing cir.coroutine's exit region";
+ let description = [{
+ Placeholder emitted while flattening `cir.await`.
+ `cir.coroutine` flattening rewrites it into a `cir.br`
+ to the `exit` block. It must not survive flattening.
+ }];
+ let assemblyFormat = "attr-dict";
+ let hasLLVMLowering = false;
+}
+
//===----------------------------------------------------------------------===//
// Coroutine intrinsic PromiseOp
//===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index d1a7a409bff9f8..e30e16679883b9 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -128,6 +128,7 @@ struct MissingFeatures {
static bool coroAwaitFullExprCleanups() { return false; }
static bool coroOutsideFrameMD() { return false; }
static bool coroutineGroManager() { return false; };
+ static bool coroutineUnwindRegion() { return false; };
// Various handling of deferred processing in CIRGenModule.
static bool cgmRelease() { return false; }
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index d3cd0145221ee3..12f0537ce0c38f 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -2118,45 +2118,195 @@ class CIRTryOpFlattening : public mlir::OpRewritePattern<cir::TryOp> {
}
};
-static mlir::Block *getOrCreateBlockForSuspendPoint(
- cir::FuncOp funcOp, mlir::PatternRewriter &rewriter, mlir::Location loc) {
- mlir::Block &entryBlock = funcOp.getBody().front();
-
-return {};
+static void
+collectCoReturnsOps(mlir::Region ®ion,
+ llvm::SmallVectorImpl<cir::CoReturnOp> &coReturnOps) {
+ region.walk(
+ [&](cir::CoReturnOp coReturnOp) { coReturnOps.push_back(coReturnOp); });
}
+// Flatten `cir.coroutine`. It runs once every nested op (awaits, scopes,
+// cleanups, ...) has been flattened. The initialSuspend, body and finalSuspend
+// regions become the body of a cleanup scope whose cleanup is the destroy
+// region; the exit region follows the scope:
+//
+// cir.cleanup.scope {
+// initialSuspend ---> body ---> finalSuspend
+// | ^
+// +-------------+ cir.co_return
+// } cleanup all {
+// destroy coro.free
+// }
+// exit coro.end + cir.return
+//
+// Leaving the scope body always runs the destroy region. In particular, the
+// cir.yield emitted for an await's destroy case leaves through it. A
+// suspended coroutine (cir.coro.exit) must keep its frame, so it branches
+// straight to the exit region.
+class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
+ void collectCoroExitsOps(
+ cir::CoroutineOp &coroOp,
+ llvm::SmallVectorImpl<cir::CoroExitOp> &coroExitsOps) const {
+ // cir.coro.exit is produced by flattening a cir.await, which can only
+ // appear in the initialSuspend, body and finalSuspend regions.
+ auto collectCoroExit = [&](cir::CoroExitOp op) {
+ coroExitsOps.push_back(op);
+ };
+ coroOp.getInitialSuspend().walk(collectCoroExit);
+ coroOp.getBody().walk(collectCoroExit);
+ coroOp.getFinalSuspend().walk(collectCoroExit);
+ }
+
+public:
+ using OpRewritePattern<cir::CoroutineOp>::OpRewritePattern;
+
+ mlir::LogicalResult
+ matchAndRewrite(cir::CoroutineOp coroOp,
+ mlir::PatternRewriter &rewriter) const override {
+
+ assert(!cir::MissingFeatures::coroutineUnwindRegion());
+ for (mlir::Region ®ion : coroOp->getRegions())
+ if (hasNestedOpsToFlatten(region))
+ return mlir::failure();
+
+ // Collect the cir.coro.exit ops before the regions are moved. Each one
+ // becomes a branch to the entry of the exit region.
+ llvm::SmallVector<cir::CoroExitOp> coroExitsOps;
+ collectCoroExitsOps(coroOp, coroExitsOps);
+
+ mlir::Location loc = coroOp.getLoc();
+ mlir::Block *coroBlock = rewriter.getInsertionBlock();
+ // Split the current block before the CoroutineOp to create the inlining
+ // point.
+ mlir::Block *remainingOpsBlock =
+ rewriter.splitBlock(coroBlock, rewriter.getInsertionPoint());
+
+ rewriter.setInsertionPointToEnd(coroBlock);
+ cir::BrOp::create(rewriter, loc, mlir::ValueRange(), remainingOpsBlock);
+
+ // coroOp now lives in remainingOpsBlock. Build the cleanup scope in front
+ // of it.
+ rewriter.setInsertionPoint(coroOp);
+
+ auto emitYield = [](mlir::OpBuilder &b, mlir::Location l) {
+ cir::YieldOp::create(b, l);
+ };
+
+ auto scope = cir::CleanupScopeOp::create(rewriter, loc, CleanupKind::All,
+ emitYield, emitYield);
+ mlir::Block *bodyPlaceholder = &scope.getBodyRegion().front();
+
+ // Scope body: initialSuspend -> body -> finalSuspend.
+ mlir::Region &initialRegion = coroOp.getInitialSuspend();
+ mlir::Block *afterInitial = &initialRegion.back();
+ rewriter.inlineRegionBefore(initialRegion, bodyPlaceholder);
+
+ mlir::Region &bodyRegion = coroOp.getBody();
+ mlir::Block *beforeBody = &bodyRegion.front();
+ mlir::Block *afterBody = &bodyRegion.back();
+
+ rewriter.setInsertionPointToEnd(afterInitial);
+ auto initialYieldOp = cast<cir::YieldOp>(afterInitial->getTerminator());
+ rewriter.replaceOpWithNewOp<cir::BrOp>(
+ initialYieldOp, initialYieldOp.getArgs(), beforeBody);
+
+ llvm::SmallVector<cir::CoReturnOp> coReturnsOps;
+ collectCoReturnsOps(bodyRegion, coReturnsOps);
+
+ mlir::Region &finalRegion = coroOp.getFinalSuspend();
+ mlir::Block *beforeFinal = &finalRegion.front();
+
+ rewriter.inlineRegionBefore(bodyRegion, bodyPlaceholder);
+ rewriter.setInsertionPointToEnd(afterBody);
+
+ // The body either falls off the end (cir.yield) or ends in a
+ // cir.co_return. Both, like every other cir.co_return in the body, branch
+ // to the final suspend.
+ mlir::Operation *bodyTerminator = afterBody->getTerminator();
+ if (auto bodyYield = dyn_cast<cir::YieldOp>(bodyTerminator))
+ rewriter.replaceOpWithNewOp<cir::BrOp>(bodyYield, mlir::ValueRange{},
+ beforeFinal);
+
+ for (auto &coReturn : coReturnsOps) {
+ rewriter.setInsertionPoint(coReturn);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(coReturn, mlir::ValueRange{},
+ beforeFinal);
+ }
+
+ rewriter.inlineRegionBefore(finalRegion, bodyPlaceholder);
+ rewriter.eraseBlock(bodyPlaceholder);
+
+ // Scope cleanup: the destroy region.
+ mlir::Region &destroyRegion = coroOp.getDestroy();
+ mlir::Block *cleanupPlaceholder = &scope.getCleanupRegion().front();
+ rewriter.inlineRegionBefore(destroyRegion, cleanupPlaceholder);
+ rewriter.eraseBlock(cleanupPlaceholder);
+
+ mlir::Region &exitRegion = coroOp.getExit();
+ mlir::Block *beforeExit = &exitRegion.front();
+
+ for (auto &coroExit : coroExitsOps) {
+ // TODO(cir): once the GRO region is supported, this may need to branch
+ // there instead of to the exit region.
+ assert(!cir::MissingFeatures::coroutineGroManager());
+ rewriter.setInsertionPoint(coroExit);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(coroExit, mlir::ValueRange{},
+ beforeExit);
+ }
+
+ // Exit region: inline it right after the scope, which falls through to it.
+ rewriter.setInsertionPointAfter(scope);
+ mlir::Block *afterCleanup =
+ rewriter.splitBlock(remainingOpsBlock, rewriter.getInsertionPoint());
+ rewriter.inlineRegionBefore(exitRegion, afterCleanup);
+
+ rewriter.setInsertionPointAfter(scope);
+ cir::BrOp::create(rewriter, loc, mlir::ValueRange(), beforeExit);
+
+ auto func = coroOp->getParentOfType<cir::FuncOp>();
+ // The coroutine regions have been flattened, so the original coroutine
+ // attribute no longer verifies. Preserve the information that this function
+ // originated from a coroutine for LLVM lowering.
+ func.setCoroutine(false);
+ func.setFlattenedCoroutine(true);
+
+ rewriter.eraseOp(coroOp);
+ return mlir::success();
+ }
+};
+
class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
public:
using OpRewritePattern<cir::AwaitOp>::OpRewritePattern;
// Flatten `cir.await` into the following control flow:
//
- // awaitBlock
- // |
- // v
- // ready
- // / \
- // / \
- // v v
- // resume suspend
- // | |
- // | llvm.coro.suspend
- // | |
- // | +----+----+
- // | | | |
- // | | | v
- // | | | suspend point
- // | | |
- // | | v
- // | | destroy
- // | |
- // +--> resume
- // |
- // v
- // continuation
+ // awaitBlock
+ // |
+ // v
+ // ready await_ready()
+ // true / \ false
+ // | v
+ // | suspend coro.save, await_suspend(),
+ // | / | \ coro.suspend
+ // | 0/ 1| \-1 (default)
+ // v v v v
+ // resume destroy suspended
+ // | cir.yield cir.coro.exit
+ // v
+ // continuation ops after the await
+ //
+ // `destroy` is meant to leave through the enclosing cleanups to the
+ // coroutine's destroy region (see the FIXME below), and `suspended` goes
+ // straight to its exit region (see CIRCoroutineFlattening).
mlir::LogicalResult
matchAndRewrite(cir::AwaitOp awaitOp,
mlir::PatternRewriter &rewriter) const override {
+
+ for (mlir::Region ®ion : awaitOp->getRegions())
+ if (hasNestedOpsToFlatten(region))
+ return mlir::failure();
+
mlir::Block *awaitBlock = rewriter.getInsertionBlock();
// Split the current block before the AwaitOp to create the inlining
// point.
@@ -2191,86 +2341,92 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
cir::BrOp::create(rewriter, loc, mlir::ValueRange(), &beforeReady);
}
- auto suspendYield =
- cast<cir::YieldOp>(suspendRegion.back().getTerminator());
- cir::LLVMIntrinsicCallOp coroSuspendIntri = nullptr;
+ // TODO(cir): support an await_suspend() that returns bool (it can veto
+ // the suspension and resume right away) by flattening it into a brcond.
+ auto suspendPoint =
+ cast<cir::CoroSuspendPoint>(suspendRegion.back().getTerminator());
+ cir::CoroSuspendOp coroSuspendIntri = nullptr;
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(&suspendRegion.front().front());
- // Insert coro.save at the beginning of the suspend region.
- // This captures the current coroutine state before suspension.
+ // coro.save goes first in the suspend region, before await_suspend()
+ // runs, so the coroutine is already considered suspended (and can be
+ // resumed) from that point.
auto voidPtrTy = cir::PointerType::get(cir::VoidType::get(getContext()));
auto nullPtr = cir::ConstantOp::create(
rewriter, loc,
cir::ConstPtrAttr::get(voidPtrTy, rewriter.getI64IntegerAttr(0)));
- auto int32Ty = cir::IntType::get(getContext(), 32, false);
- auto coroSaveIntri = cir::LLVMIntrinsicCallOp::create(
- rewriter, loc, rewriter.getStringAttr("llvm.coro.save"), int32Ty,
- mlir::ValueRange{nullPtr});
- rewriter.setInsertionPoint(suspendYield);
+ auto coroSave = cir::CoroSaveOp::create(rewriter, loc, nullPtr);
+ rewriter.setInsertionPoint(suspendPoint);
// The second argument to `llvm.coro.suspend` indicates whether this is
// the final suspend point. Coroutines suspended at a final suspend point
// are considered done (`llvm.coro.done` returns true) and may only be
- // destroyed resuming them is undefined behavior.
+ // destroyed; resuming them is undefined behavior.
bool isFinalSuspend = awaitOp.getKind() == cir::AwaitKind::Final;
auto isFinalCoroSuspend = cir::ConstantOp::create(
rewriter, loc, cir::BoolAttr::get(getContext(), isFinalSuspend));
// llvm.coro.suspend returns:
- // -1 : coroutine suspended
+ // -1 : coroutine suspended
// 0 : coroutine resumed
// 1 : coroutine destroyed
- coroSuspendIntri = cir::LLVMIntrinsicCallOp::create(
+ coroSuspendIntri = cir::CoroSuspendOp::create(
rewriter, loc,
- mlir::StringAttr::get(getContext(), "llvm.coro.suspend"), int32Ty,
- mlir::ValueRange{coroSaveIntri.getResult(), isFinalCoroSuspend});
+ mlir::ValueRange{coroSave.getResult(), isFinalCoroSuspend});
}
rewriter.inlineRegionBefore(suspendRegion, remainingOpsBlock);
- auto func = awaitOp->getParentOfType<cir::FuncOp>();
-
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
- rewriter.setInsertionPoint(suspendYield);
- llvm::SmallVector<mlir::APInt> caseValues{mlir::APInt(32, 0),
- mlir::APInt(32, 1)};
+ rewriter.setInsertionPoint(suspendPoint);
+ llvm::SmallVector<mlir::APInt> caseValues{mlir::APInt(8, 0),
+ mlir::APInt(8, 1)};
- llvm::SmallVector<mlir::ValueRange> caseOperands{3};
+ llvm::SmallVector<mlir::ValueRange> caseOperands{caseValues.size()};
- // In Classic CodeGen, the destroy path reaches the coroutine cleanup by
- // emitting an EmitBranchThroughCleanup(), ensuring that all nested
- // cleanup scopes are executed before control reaches the coro.free
- // cleanup.
+ // Destroy case. Classic codegen uses EmitBranchThroughCleanup() so every
+ // enclosing cleanup runs before coro.free. Here the destroy block is a
+ // single cir.yield, meant to leave through the coroutine cleanup scope
+ // (coro.free) and then reach the exit region.
//
- // We achieve the same effect by creating a block that only contains a
- // cir.yield. createExitTerminator() then propagates control through every
- // enclosing cleanup scope until the parent coroutine cleanup (coro.free)
- // is reached, after which execution continues to the return block.
+ // FIXME: This only works when the await is directly in the coroutine
+ // body. Any enclosing cir.cleanup.scope or loop treats the cir.yield as
+ // a normal fallthrough, so after running its cleanups the destroy path
+ // continues into the rest of the body (or the next loop iteration)
+ // instead of reaching coro.free. This needs a dedicated terminator that
+ // createExitTerminator propagates through cleanup scopes (like
+ // cir.co_return) and that CIRCoroutineFlattening rewrites into a branch
+ // to the coroutine cleanup. See silly_task_with_dtor in
+ // clang/test/CIR/CodeGenCoroutines/coro-task.cpp.
mlir::Block *cleanupBlock = nullptr;
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
cleanupBlock = rewriter.createBlock(remainingOpsBlock);
cir::YieldOp::create(rewriter, loc);
}
- // The destroy case dispatches to cleanupBlock, which propagates through
- // enclosing cleanup scopes before reaching the coroutine-frame cleanup.
+
+ // Suspended case: cir.coro.exit, which CIRCoroutineFlattening turns into
+ // a branch to the exit region.
+ mlir::Block *retBlock = nullptr;
+ {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ assert(!cir::MissingFeatures::coroutineGroManager());
+ retBlock = rewriter.createBlock(remainingOpsBlock);
+ cir::CoroExitOp::create(rewriter, loc);
+ }
+ // 0 resumes, 1 destroys and the default (-1) stays suspended.
llvm::SmallVector<mlir::Block *> caseDestinations{&resumeRegion.front(),
cleanupBlock};
- caseDestinations.push_back(&resumeRegion.front());
- caseDestinations.push_back(cleanupBlock);
-
- assert(!cir::MissingFeatures::coroutineGroManager());
// Dispatch to the appropriate coroutine path based on the result of
// `llvm.coro.suspend`: resume, destroy, or suspend.
auto coroSuspendSwitch = cir::SwitchFlatOp::create(
- rewriter, loc, coroSuspendIntri.getResult(),
- getOrCreateBlockForSuspendPoint(func, rewriter, loc),
- mlir::ValueRange(), caseValues, caseDestinations, caseOperands);
+ rewriter, loc, coroSuspendIntri.getResult(), retBlock,
+ mlir::ValueRange{}, caseValues, caseDestinations, caseOperands);
- rewriter.replaceOp(suspendYield, coroSuspendSwitch);
+ rewriter.replaceOp(suspendPoint, coroSuspendSwitch);
}
auto resumeYield = cast<cir::YieldOp>(resumeRegion.back().getTerminator());
@@ -2285,12 +2441,6 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
rewriter.eraseOp(awaitOp);
- // The coroutine regions have been flattened, so the original coroutine
- // attribute no longer verifies. Preserve the information that this function
- // originated from a coroutine for LLVM lowering.
- func.setCoroutine(false);
- func.setFlattenedCoroutine(true);
-
return mlir::success();
}
};
@@ -2300,8 +2450,7 @@ void populateFlattenCFGPatterns(RewritePatternSet &patterns) {
.add<CIRIfFlattening, CIRLoopOpInterfaceFlattening, CIRScopeOpFlattening,
CIRSwitchOpFlattening, CIRTernaryOpFlattening,
CIRCleanupScopeOpFlattening, CIRTryOpFlattening,
- CIRAwaitOpFlattening>(
- patterns.getContext());
+ CIRAwaitOpFlattening, CIRCoroutineFlattening>(patterns.getContext());
}
namespace {
@@ -2364,7 +2513,7 @@ void CIRFlattenCFGPass::runOnOperation() {
llvm::SmallVector<Operation *, 16> ops;
getOperation()->walk<mlir::WalkOrder::PostOrder>([&](Operation *op) {
if (isa<IfOp, ScopeOp, SwitchOp, LoopOpInterface, TernaryOp,
- CleanupScopeOp, TryOp>(op))
+ CleanupScopeOp, TryOp, AwaitOp, CoroutineOp>(op))
ops.push_back(op);
});
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp b/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
deleted file mode 100644
index 58babff2760ddc..00000000000000
--- a/clang/test/CIR/CodeGenCoroutines/coro-flatten.cpp
+++ /dev/null
@@ -1,437 +0,0 @@
-// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
-// RUN: cir-opt --cir-flatten-cfg %t.cir -o %t.flat.cir
-// RUN: FileCheck --input-file=%t.flat.cir %s -check-prefix=CIR-FLAT
-
-#include "Inputs/coroutine.h"
-
-using VoidTask = folly::coro::Task<void>;
-
-VoidTask silly_task() {
- co_await std::suspend_always();
-}
-
-// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z10silly_taskv
-
-// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
-// CIR-FLAT: %[[NullPtr:.*]] = cir.const #cir.ptr<null>
-// CIR-FLAT: %[[Align:.*]] = cir.const #cir.int<16>
-// CIR-FLAT: %[[CoroId:.*]] = cir.call @__builtin_coro_id(%[[Align]], %[[NullPtr]], %[[NullPtr]], %[[NullPtr]])
-// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
-// CIR-FLAT: %[[SavedFrameAddr:.*]] = cir.alloca "__coro_frame_addr"
-// CIR-FLAT: %[[SuspendAlwaysAddr:.*]] = cir.alloca "ref.tmp0"
-// CIR-FLAT: %[[ShouldAlloc:.*]] = cir.call @__builtin_coro_alloc(%[[CoroId]]) : (!u32i) -> !cir.bool
-// CIR-FLAT: cir.store %[[NullPtr]], %[[SavedFrameAddr]]
-// CIR-FLAT: cir.brcond %[[ShouldAlloc]] ^[[CORO_ALLOC:.*]], ^[[CORO_INIT:.*]]
-// CIR-FLAT: ^[[CORO_ALLOC]]:
-// CIR-FLAT: %[[CoroSize:.*]] = cir.call @__builtin_coro_size()
-// CIR-FLAT: %[[AllocAddr:.*]] = cir.call @_Znwm(%[[CoroSize]])
-// CIR-FLAT: cir.store %[[AllocAddr]], %[[SavedFrameAddr]] : !cir.ptr<!void>, !cir.ptr<!cir.ptr<!void>>
-// CIR-FLAT: cir.br ^[[CORO_INIT]]
-// CIR-FLAT: ^[[CORO_INIT]]:
-// CIR-FLAT: %[[LOAD_CORO_FRAME:.*]] = cir.load %[[SavedFrameAddr]]
-// CIR-FLAT: %[[CoroFrameAddr:.*]] = cir.call @__builtin_coro_begin(%[[CoroId]], %[[LOAD_CORO_FRAME]])
-// CIR-FLAT: cir.br ^[[INIT_AWAIT_READY:.*]]
-// CIR-FLAT: ^[[INIT_AWAIT_READY]]:
-// CIR-FLAT: %[[RetObj:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type17get_return_objectEv(%[[VoidPromisseAddr:.*]])
-// CIR-FLAT: cir.store {{.*}} %[[RetObj]], %[[VoidTaskAddr:.*]]
-// CIR-FLAT: %[[Tmp0:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(%[[VoidPromisseAddr]])
-// CIR-FLAT: cir.store {{.*}} %[[Tmp0]], %[[SuspendAlwaysAddr]]
-// CIR-FLAT: cir.br ^[[INIT_AWAIT_READY_CONT:.*]]
-// CIR-FLAT: ^[[INIT_AWAIT_READY_CONT]]:
-// CIR-FLAT: %[[ShouldSuspend:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]])
-// CIR-FLAT: cir.brcond %[[ShouldSuspend]] ^[[AWAIT_INIT_RESUME:.*]], ^[[AWAIT_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_SUSPEND]]: // pred: ^bb4
-// CIR-FLAT: %[[NULLPTR2:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
-
-// TODO (cir): We should use a cir.token or mlir.token instead of returning !u32i.
-// CIR-FLAT: %[[SAVE_TOKEN:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR2]] : (!cir.ptr<!void>) -> !u32i
-// CIR-FLAT: %27 = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%21)
-// CIR-FLAT: cir.store align(1) %27, %10
-// CIR-FLAT: %28 = cir.load align(1) %10
-// CIR-FLAT: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%9, %28)
-// CIR-FLAT: %29 = cir.load align(1) %9
-// CIR-FLAT: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%8, %29)
-// CIR-FLAT: %[[IS_FINAL_SUSPEND:.*]] = cir.const #false
-// CIR-FLAT: %[[SUSPEND_RESULT:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN]], %[[IS_FINAL_SUSPEND]]
-// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT]] : !u32i, ^[[CORO_RET:.*]] [
-// CIR-FLAT: 0: ^[[AWAIT_INIT_RESUME]],
-// CIR-FLAT: 1: ^[[INIT_CLEANUP_DESTROY:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[INIT_CLEANUP_DESTROY]]:
-// CIR-FLAT: cir.const #cir.int<0>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
-// CIR-FLAT: ^[[AWAIT_INIT_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]])
-// CIR-FLAT: cir.br ^[[CORO_BODY:.*]]
-
-// The remaining await operations follow the same lowering pattern as above,
-// so we only check a few key instructions here instead of matching the entire IR
-
-// CIR-FLAT: ^[[CORO_BODY:.*]]:
-// CIR-FLAT: cir.br ^[[USER_AWAIT:.*]]
-// CIR-FLAT: ^[[USER_AWAIT]]:
-// CIR-FLAT: cir.br ^[[USER_AWAIT_READY_CONT:.*]]
-// CIR-FLAT: ^[[USER_AWAIT_READY_CONT]]:
-// CIR-FLAT: %[[ShouldSuspend2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
-// CIR-FLAT: cir.brcond %[[ShouldSuspend2]] ^[[AWAIT_USER_RESUME:.*]], ^[[AWAIT_USER_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_USER_SUSPEND]]
-// CIR-FLAT: %[[NULLPTR3:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
-// CIR-FLAT: %[[SAVE_TOKEN2:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR3]]
-// CIR-FLAT: %[[IS_FINAL_SUSPEND2:.*]] = cir.const #false
-// CIR-FLAT: %[[SUSPEND_RESULT2:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN2]], %[[IS_FINAL_SUSPEND2]]
-// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT2]] : !u32i, ^[[CORO_RET]] [
-// CIR-FLAT: 0: ^[[AWAIT_USER_RESUME]],
-// CIR-FLAT: 1: ^[[user_cleanup_destroy:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[user_cleanup_destroy]]:
-// CIR-FLAT: cir.const #cir.int<1>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
-// CIR-FLAT: ^[[AWAIT_USER_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
-// CIR-FLAT: cir.br ^[[CO_RETURN:.*]]
-// CIR-FLAT: ^[[CO_RETURN]]:
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[VoidPromisseAddr]])
-
-// This corresponds to the implicit cir.co_return path, which exits the
-// cir.coro.body and branches to the final suspend block.
-// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
-// CIR-FLAT: ^[[FINAL_SUSPEND_BB]]:
-// CIR-FLAT: %[[final_suspend:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[VoidPromisseAddr]])
-// CIR-FLAT: cir.store {{.*}} %[[final_suspend]], %{{.*}}
-// CIR-FLAT: cir.br ^[[FINAL_AWAIT_READY_CONT:.*]]
-// CIR-FLAT: ^[[FINAL_AWAIT_READY_CONT]]:
-// CIR-FLAT: %[[ShouldSuspend3:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
-// CIR-FLAT: cir.brcond %[[ShouldSuspend3]] ^[[AWAIT_FINAL_RESUME:.*]], ^[[AWAIT_FINAL_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_FINAL_SUSPEND]]:
-// CIR-FLAT: %[[NULLPTR4:.*]] = cir.const #cir.ptr<null>
-// CIR-FLAT: %[[SAVE_TOKEN3:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR4]]
-// CIR-FLAT: %[[IS_FINAL_SUSPEND3:.*]] = cir.const #true
-// CIR-FLAT: %[[SUSPEND_RESULT3:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN3]], %[[IS_FINAL_SUSPEND3]]
-// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT3]] : !u32i, ^[[CORO_RET]] [
-// CIR-FLAT: 0: ^[[AWAIT_FINAL_RESUME]],
-// CIR-FLAT: 1: ^[[final_cleanup_destroy:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[final_cleanup_destroy]]:
-// CIR-FLAT: cir.const #cir.int<2>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
-// CIR-FLAT: ^[[AWAIT_FINAL_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
-// CIR-FLAT: cir.br ^[[FIANL_CLEANUP_EXIT:.*]]
-// CIR-FLAT: ^[[FIANL_CLEANUP_EXIT]]:
-// CIR-FLAT: cir.const #cir.int<3> : !s32i
-// CIR-FLAT: cir.store %{{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
-
-// Check whether llvm.coro.free returned a non-null pointer, indicating
-// that the coroutine frame must be deallocated.
-// CIR-FLAT: ^[[CLEANUP_CORO_FREE]]:
-// CIR-FLAT: %[[SHOULD_FREE:.*]] = cir.call @__builtin_coro_free(%[[CoroId]], %[[CoroFrameAddr]])
-// CIR-FLAT: %[[NULLPTR5:.*]] = cir.const #cir.ptr<null>
-// CIR-FLAT: %[[NEEDS_FREE:.*]] = cir.cmp ne %[[SHOULD_FREE]], %[[NULLPTR5]]
-// CIR-FLAT: cir.brcond %[[NEEDS_FREE]] ^[[FREE_FRAME:.*]], ^[[EXIT_CLEANUP:.*]]
-// CIR-FLAT: ^[[FREE_FRAME]]:
-// CIR-FLAT: %[[CoroSize2:.*]] = cir.call @__builtin_coro_size() : () -> (!u64i {llvm.noundef})
-// CIR-FLAT: cir.call @_ZdlPvm(%[[SHOULD_FREE]], %[[CoroSize2]])
-// CIR-FLAT: cir.br ^[[EXIT_CLEANUP]]
-// CIR-FLAT: ^[[EXIT_CLEANUP]]:
-// CIR-FLAT: cir.br ^[[EXIT_CLEANUP_SWITCH:.*]]
-// CIR-FLAT: ^[[EXIT_CLEANUP_SWITCH]]:
-// CIR-FLAT: %[[LOAD_DEST_SLOT:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.switch.flat %[[LOAD_DEST_SLOT]] : !s32i, ^[[DEFAULT_EXIT:.*]] [
-// CIR-FLAT: 0: ^[[EXIT1:.*]],
-// CIR-FLAT: 1: ^[[EXIT2:.*]],
-// CIR-FLAT: 2: ^[[EXIT3:.*]],
-// CIR-FLAT: 3: ^[[EXIT4:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[EXIT1]]:
-// CIR-FLAT: cir.br ^[[TO_RET:.*]]
-// CIR-FLAT: ^[[EXIT2]]:
-// CIR-FLAT: cir.br ^[[TO_RET]]
-// CIR-FLAT: ^[[EXIT3]]:
-// CIR-FLAT: cir.br ^[[TO_RET]]
-// CIR-FLAT: ^[[EXIT4]]:
-// CIR-FLAT: cir.br ^[[TO_RET]]
-// CIR-FLAT: ^[[DEFAULT_EXIT]]:
-// CIR-FLAT: cir.unreachable
-// CIR-FLAT: ^[[TO_RET]]:
-// CIR-FLAT: cir.br ^[[CORO_RET]]
-// CIR-FLAT: ^[[CORO_RET]]:
-// CIR-FLAT: %58 = cir.const #cir.int<1> : !s32i
-// CIR-FLAT: cir.store %58, %5 : !s32i, !cir.ptr<!s32i>
-// CIR-FLAT: %[[NULLPTR6:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
-// CIR-FLAT: %[[UNWIND:.*]] = cir.const #false
-// CIR-FLAT: cir.call @__builtin_coro_end(%[[NULLPTR6]], %[[UNWIND]])
-// CIR-FLAT: cir.return
-// CIR-FLAT: }
-
-struct HasDtor {
- ~HasDtor();
-};
-
-VoidTask silly_task_with_dtor() {
- HasDtor local;
- co_await std::suspend_always();
-}
-
-// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z20silly_task_with_dtorv
-// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
-// CIR-FLAT: %[[SuspendAlwaysAddr:.*]] = cir.alloca "ref.tmp0"
-
-// CIR-FLAT: %[[ShouldSuspend:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]])
-// CIR-FLAT: cir.brcond %[[ShouldSuspend]] ^[[AWAIT_INIT_RESUME:.*]], ^[[AWAIT_INIT_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_INIT_SUSPEND]]:
-// CIR-FLAT: %[[NULLPTR:.*]] = cir.const #cir.ptr<null>
-// CIR-FLAT: %[[SAVE_TOKEN:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR]]
-// CIR-FLAT: %[[IS_FINAL_SUSPEND:.*]] = cir.const #false
-// CIR-FLAT: %[[SUSPEND_RESULT:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN]], %[[IS_FINAL_SUSPEND]]
-// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT]] : !u32i, ^[[CORO_RET:.*]] [
-// CIR-FLAT: 0: ^[[AWAIT_INIT_RESUME]],
-// CIR-FLAT: 1: ^[[INIT_CLEANUP_DESTROY:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[INIT_CLEANUP_DESTROY]]:
-// CIR-FLAT: cir.const #cir.int<0>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
-// CIR-FLAT: ^[[AWAIT_INIT_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]])
-// CIR-FLAT: cir.br ^[[AWAIT_INIT_RESUME_CONT:.*]]
-// CIR-FLAT: ^[[AWAIT_INIT_RESUME_CONT]]:
-// CIR-FLAT: cir.br ^[[CORO_BODY:.*]]
-// CIR-FLAT: ^[[CORO_BODY]]:
-// CIR-FLAT: cir.br ^[[HAS_DTOR_CLEANUP_SCOPE:.*]]
-// CIR-FLAT: ^[[HAS_DTOR_CLEANUP_SCOPE]]:
-// CIR-FLAT: cir.br ^[[USER_AWAIT_READY:.*]]
-// CIR-FLAT: ^[[USER_AWAIT_READY]]:
-// CIR-FLAT: %[[ShouldSuspend2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
-// CIR-FLAT: cir.brcond %[[ShouldSuspend2]] ^[[AWAIT_USER_RESUME:.*]], ^[[AWAIT_USER_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_USER_SUSPEND]]:
-// CIR-FLAT: %[[NULLPTR2:.*]] = cir.const #cir.ptr<null>
-// CIR-FLAT: %[[SAVE_TOKEN2:.*]] = cir.call_llvm_intrinsic "llvm.coro.save" %[[NULLPTR2]]
-// CIR-FLAT: %[[IS_FINAL_SUSPEND2:.*]] = cir.const #false
-// CIR-FLAT: %[[SUSPEND_RESULT2:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend" %[[SAVE_TOKEN2]], %[[IS_FINAL_SUSPEND2]]
-
-// The destroy branch cannot jump directly to the coroutine cleanup.
-// Since this await is nested within the lifetime of a local HasDtor object,
-// destruction must first run HasDtor::~HasDtor() before proceeding to the
-// coroutine-frame cleanup.
-//
-// Destroy path:
-// USER_CLEANUP_DESTROY -> HasDtor::~HasDtor() -> coro.free cleanup
-
-// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT2]] : !u32i, ^[[CORO_RET]] [
-// CIR-FLAT: 0: ^[[AWAIT_USER_RESUME]],
-// CIR-FLAT: 1: ^[[USER_CLEANUP_DESTROY:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[USER_CLEANUP_DESTROY]]:
-// CIR-FLAT: cir.const #cir.int<0>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[HAS_DTR:.*]]
-// CIR-FLAT: ^[[AWAIT_USER_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
-// CIR-FLAT: cir.br ^[[CO_RETURN:.*]]
-// CIR-FLAT: ^[[CO_RETURN]]:
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
-// CIR-FLAT: cir.const #cir.int<1>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[HAS_DTR]]
-// CIR-FLAT: ^[[HAS_DTR]]:
-// CIR-FLAT: cir.call @_ZN7HasDtorD1Ev
-// CIR-FLAT: cir.br ^[[HAS_DTR_CONT:.*]]
-// CIR-FLAT: ^[[HAS_DTR_CONT]]:
-// CIR-FLAT: %[[LOAD_DEST_SLOT:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.switch.flat %[[LOAD_DEST_SLOT]] : !s32i, ^[[DEFAULT:.*]] [
-// CIR-FLAT: 0: ^[[FROM_USER_CLEANUP_DESTROY:.*]],
-// CIR-FLAT: 1: ^[[FROM_CO_RETURN:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[FROM_USER_CLEANUP_DESTROY]]:
-// CIR-FLAT: cir.br ^[[EXIT_TO_CORO_FREE:.*]]
-// CIR-FLAT: ^[[FROM_CO_RETURN]]:
-// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
-// CIR-FLAT: ^[[DEFAULT]]:
-// CIR-FLAT: cir.unreachable
-// CIR-FLAT: ^[[EXIT_TO_CORO_FREE]]:
-// CIR-FLAT: cir.const #cir.int<1>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
-// CIR-FLAT: ^[[FINAL_SUSPEND_BB:.*]]:
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv
-// CIR-FLAT: cir.br ^[[FINAL_AWAIT_READY_CONT:.*]]
-// CIR-FLAT: cir.call @_ZNSt14suspend_always11await_readyEv
-// CIR-FLAT: cir.brcond {{.*}} ^[[AWAIT_FINAL_RESUME:.*]], ^[[AWAIT_FINAL_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_FINAL_SUSPEND]]:
-// CIR-FLAT: cir.call_llvm_intrinsic "llvm.coro.save"
-// CIR-FLAT: cir.const #true
-// CIR-FLAT: %[[SUSPEND_RESULT3:.*]] = cir.call_llvm_intrinsic "llvm.coro.suspend"
-// CIR-FLAT: cir.switch.flat %[[SUSPEND_RESULT3]] : !u32i, ^[[CORO_RET]] [
-// CIR-FLAT: 0: ^[[AWAIT_FINAL_RESUME]],
-// CIR-FLAT: 1: ^[[FINAL_CLEANUP_DESTROY:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[FINAL_CLEANUP_DESTROY]]:
-// CIR-FLAT: cir.const #cir.int<2>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^bb28
-// CIR-FLAT: ^[[AWAIT_FINAL_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
-// CIR-FLAT: cir.br ^[[CORO_BODY_EXIT:.*]]
-// CIR-FLAT: ^[[CORO_BODY_EXIT]]:
-// CIR-FLAT: cir.const #cir.int<3> : !s32i
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
-// CIR-FLAT: ^[[CLEANUP_CORO_FREE]]:
-// CIR-FLAT: %[[SHOULD_FREE:.*]] = cir.call @__builtin_coro_free
-// CIR-FLAT: %[[NULLPTR5:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
-// CIR-FLAT: %[[NEEDS_FREE:.*]] = cir.cmp ne %[[SHOULD_FREE]], %[[NULLPTR5]]
-// CIR-FLAT: cir.brcond %[[NEEDS_FREE]] ^[[FREE_FRAME:.*]], ^[[EXIT_CLEANUP:.*]]
-// CIR-FLAT: ^[[FREE_FRAME]]:
-// CIR-FLAT: %60 = cir.call @__builtin_coro_size() : () -> (!u64i {llvm.noundef})
-// CIR-FLAT: cir.call @_ZdlPvm(%57, %60) nothrow : (!cir.ptr<!void> {llvm.noundef}, !u64i {llvm.noundef}) -> ()
-// CIR-FLAT: cir.br ^[[EXIT_CLEANUP]]
-// CIR-FLAT: ^[[EXIT_CLEANUP]]:
-// CIR-FLAT: cir.br ^[[EXIT_CLEANUP_SWITCH:.*]]
-// CIR-FLAT: ^[[EXIT_CLEANUP_SWITCH]]:
-// CIR-FLAT: %[[LOAD_DEST_SLOT:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.switch.flat %[[LOAD_DEST_SLOT:.*]]
-// CIR-FLAT: ^[[CORO_RET]]:
-// CIR-FLAT: cir.return
-
-folly::coro::Task<int> co_returns(int flag) {
- if (flag == 1) {
- co_return 1;
- } else if (flag == 2) {
- co_return 2;
- }
- co_return 3;
-}
-
-// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z10co_returnsi
-// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
-// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
-
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
-// CIR-FLAT: %[[LOAD_FLAG:.*]] = cir.load {{.*}} %[[FLAG:.*]]
-// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1>
-// CIR-FLAT: %[[EQ_ONE:.*]] = cir.cmp eq %[[LOAD_FLAG]], %[[ONE]]
-// CIR-FLAT: cir.brcond %[[EQ_ONE]] ^[[IF_EQ_ONE:.*]], ^[[ELSE_IF:.*]]
-// CIR-FLAT: ^[[IF_EQ_ONE]]:
-// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1>
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[ONE]])
-// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
-// CIR-FLAT: ^[[ELSE_IF]]:
-// CIR-FLAT: cir.br ^[[ELSE_IF_CONT:.*]]
-// CIR-FLAT: ^[[ELSE_IF_CONT:.*]]:
-// CIR-FLAT: %[[LOAD_FLAG2:.*]] = cir.load {{.*}} %[[FLAG]]
-// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2>
-// CIR-FLAT: %[[EQ_TWO:.*]] = cir.cmp eq %[[LOAD_FLAG2]], %[[TWO]]
-// CIR-FLAT: cir.brcond %[[EQ_TWO]] ^[[IF_EQ_TWO:.*]], ^[[ELSE:.*]]
-// CIR-FLAT: ^[[IF_EQ_TWO]]:
-// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2>
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[TWO]])
-// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB]]
-// CIR-FLAT: ^[[ELSE]]:
-// CIR-FLAT: cir.br ^[[ELSE_CONT:.*]]
-// CIR-FLAT: ^[[ELSE_CONT]]:
-// CIR-FLAT: %[[THREE:.*]] = cir.const #cir.int<3>
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[THREE]])
-// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB]]
-// CIR-FLAT: ^[[FINAL_SUSPEND_BB]]:
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv
-
-
-
-folly::coro::Task<int> co_return_with_dtor(int flag) {
- HasDtor local;
- if (flag)
- co_return 1; // local dtor must run here
- co_return 2;
-}
-
-// CIR-FLAT: cir.func flattened_coroutine {{.*}} @_Z19co_return_with_dtori
-// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
-// CIR-FLAT: %[[SUSPEND_POINT:.*]] = cir.alloca "__coroutine_suspend_point"
-// CIR-FLAT: %[[ShouldSuspend:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
-// CIR-FLAT: cir.brcond %[[ShouldSuspend]] ^[[AWAIT_INIT_RESUME:.*]], ^[[AWAIT_INIT_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_INIT_SUSPEND]]:
-// CIR-FLAT: cir.switch.flat {{.*}} : !u32i, ^[[CORO_RET:.*]] [
-// CIR-FLAT: 0: ^[[AWAIT_INIT_RESUME]],
-// CIR-FLAT: 1: ^[[INIT_CLEANUP_DESTROY:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[INIT_CLEANUP_DESTROY]]:
-// CIR-FLAT: cir.const #cir.int<0> : !s32i
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE:.*]]
-// CIR-FLAT: ^[[AWAIT_INIT_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%10)
-// CIR-FLAT: cir.br ^[[CORO_BODY:.*]]
-// CIR-FLAT: ^[[CORO_BODY]]:
-// CIR-FLAT: cir.br ^[[HAS_DTR_CLEANUP_SCOPE:.*]]
-// CIR-FLAT: ^[[HAS_DTR_CLEANUP_SCOPE]]:
-// CIR-FLAT: cir.br ^[[IF_SCOPE:.*]]
-// CIR-FLAT: ^[[IF_SCOPE]]:
-// CIR-FLAT: cir.br ^[[IF_SCOPE_CONT:.*]]
-// CIR-FLAT: ^[[IF_SCOPE_CONT]]:
-// CIR-FLAT: %[[LOAD_FLAG:.*]] = cir.load {{.*}} %[[FLAG_ARG:.*]]
-// CIR-FLAT: %[[CAST_TO_BOOL:.*]] = cir.cast int_to_bool %[[LOAD_FLAG]]
-// CIR-FLAT: cir.brcond %[[CAST_TO_BOOL]] ^[[IF_BODY:.*]], ^[[IF_CONT:.*]]
-// CIR-FLAT: ^[[IF_BODY]]:
-// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1>
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[ONE]])
-// CIR-FLAT: cir.const #cir.int<0>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-
-// This cir.co_return exits through the cleanup path. Control is transferred
-// to the destructor cleanup block before reaching the final suspend point.
-// co_return -> cleanup -> final suspend
-// CIR-FLAT: cir.br ^[[HAS_DTOR_CLEANUP:.*]]
-// CIR-FLAT: ^[[IF_CONT]]:
-// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2>
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[TWO]])
-// CIR-FLAT: cir.const #cir.int<1>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-
-// This cir.co_return exits through the cleanup path. Control is transferred
-// to the destructor cleanup block before reaching the final suspend point.
-// co_return -> cleanup -> final suspend
-// CIR-FLAT: cir.br ^[[HAS_DTOR_CLEANUP]]
-// CIR-FLAT: ^[[HAS_DTOR_CLEANUP]]:
-// CIR-FLAT: cir.call @_ZN7HasDtorD1Ev
-// CIR-FLAT: cir.br ^[[CLEANUP_EXIT:.*]]
-// CIR-FLAT: ^[[CLEANUP_EXIT]]:
-// CIR-FLAT: %[[LOAD_CLEANUP:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.switch.flat %[[LOAD_CLEANUP:.*]] : !s32i, ^[[DEFAULT:.*]] [
-// CIR-FLAT: 0: ^[[EXIT1:.*]],
-// CIR-FLAT: 1: ^[[EXIT2:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[EXIT1]]:
-// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB:.*]]
-// CIR-FLAT: ^[[EXIT2]]:
-// CIR-FLAT: cir.br ^[[FINAL_SUSPEND_BB]]
-// CIR-FLAT: ^[[DEFAULT]]:
-// CIR-FLAT: cir.unreachable
-// CIR-FLAT: ^[[FINAL_SUSPEND_BB]]:
-// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv
-// CIR-FLAT: %[[ShouldSuspend2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv
-// CIR-FLAT: cir.brcond %[[ShouldSuspend2]] ^[[AWAIT_FINAL_RESUME:.*]], ^[[AWAIT_FINAL_SUSPEND:.*]]
-// CIR-FLAT: ^[[AWAIT_FINAL_SUSPEND]]:
-// CIR-FLAT: %[[IS_FINAL_SUSPEND:.*]] = cir.const #true
-// CIR-FLAT: cir.call_llvm_intrinsic "llvm.coro.suspend" {{.*}}, %[[IS_FINAL_SUSPEND]]
-// CIR-FLAT: cir.switch.flat {{.*}} : !u32i, ^[[CORO_RET]] [
-// CIR-FLAT: 0: ^[[AWAIT_FINAL_RESUME]],
-// CIR-FLAT: 1: ^[[FINAL_CLEANUP_DESTROY:.*]]
-// CIR-FLAT: ]
-// CIR-FLAT: ^[[FINAL_CLEANUP_DESTROY]]:
-// CIR-FLAT: cir.const #cir.int<2>
-// CIR-FLAT: cir.store {{.*}}, %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[CLEANUP_CORO_FREE]]
-// CIR-FLAT: ^[[AWAIT_FINAL_RESUME]]:
-// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
-
-// CIR-FLAT: ^[[CLEANUP_CORO_FREE]]:
-// CIR-FLAT: cir.call @__builtin_coro_free
-// CIR-FLAT: cir.const #cir.ptr<null>
-// CIR-FLAT: cir.brcond %{{.*}} ^[[FREE_FRAME:.*]], ^[[EXIT_CLEANUP:.*]]
-// CIR-FLAT: ^[[FREE_FRAME]]:
-// CIR-FLAT: cir.call @__builtin_coro_size()
-// CIR-FLAT: cir.call @_ZdlPvm
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-task.cpp b/clang/test/CIR/CodeGenCoroutines/coro-task.cpp
index b0970c5cc6d590..b5b97fd5af8d27 100644
--- a/clang/test/CIR/CodeGenCoroutines/coro-task.cpp
+++ b/clang/test/CIR/CodeGenCoroutines/coro-task.cpp
@@ -1,5 +1,7 @@
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: cir-opt --cir-flatten-cfg %t.cir -o %t.flat.cir
+// RUN: FileCheck --input-file=%t.flat.cir %s -check-prefix=CIR-FLAT --implicit-check-not=cir.coroutine --implicit-check-not=cir.await --implicit-check-not=cir.co_return --implicit-check-not=cir.coro.exit --implicit-check-not=cir.coro.suspend_point
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm -disable-llvm-passes %s -o %t-cir.ll
// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=OGCG
@@ -233,6 +235,185 @@ VoidTask silly_task() {
// OGCG: call void @llvm.coro.end(ptr null, i1 false, token none)
// OGCG: ret void
+// OGCG: define {{.*}} @_Z10silly_taskv.__await_suspend_wrapper__init(ptr noundef nonnull %[[Awaiter:.*]], ptr noundef %[[Handle:.*]])
+// OGCG: entry:
+// OGCG: %[[AwaiterAddr:.*]] = alloca ptr
+// OGCG: %[[HandleAddr:.*]] = alloca ptr
+// OGCG: %[[CoroHandleVoidAddr:.*]] = alloca %"struct.std::coroutine_handle"
+// OGCG: store ptr %[[Awaiter:.*]], ptr %[[AwaiterAddr]]
+// OGCG: store ptr %[[Handle:.*]], ptr %[[HandleAddr]]
+// OGCG: %[[AwaiterReload:.*]] = load ptr, ptr %[[AwaiterAddr]]
+// OGCG: %[[CoroFrameAddr:.*]] = load ptr, ptr %[[HandleAddr]]
+// OGCG: call void @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(ptr noundef %[[CoroFrameAddr]])
+// OGCG: call void @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(ptr noundef nonnull align 1 dereferenceable(1) %[[CoroHandleVoidAddr]])
+// OGCG: call void @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(ptr noundef nonnull align 1 dereferenceable(1) %[[AwaiterReload]])
+// OGCG: ret void
+// OGCG: }
+
+// After FlattenCFG, cir.coroutine is inlined into the function: the
+// initialSuspend, body and finalSuspend regions become the body of a cleanup
+// scope whose cleanup is the destroy region (coro.free), followed by the exit
+// region. Each cir.await becomes ready/suspend/resume blocks where suspend
+// dispatches on llvm.coro.suspend: 0 resumes, 1 destroys (runs the coroutine
+// cleanup), and the default suspends (jumps straight to coro.end).
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z10silly_taskv()
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot : !cir.ptr<!s32i>
+// CIR-FLAT: %[[FRAME_ADDR:.*]] = cir.alloca "__coro_frame_addr"
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[INIT_AWAITER:.*]] = cir.alloca "ref.tmp0"
+// CIR-FLAT: %[[USER_AWAITER:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: %[[FINAL_AWAITER:.*]] = cir.alloca "ref.tmp2"
+
+// The frame allocation cir.if is flattened into coro.alloc/coro.init blocks.
+
+// CIR-FLAT: %[[NULL:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
+// CIR-FLAT: %[[ALIGN:.*]] = cir.const #cir.int<16> : !u32i
+// CIR-FLAT: %[[CORO_ID:.*]] = cir.coro.intrinsic.id(%[[ALIGN]], %[[NULL]], %[[NULL]], %[[NULL]])
+// CIR-FLAT: %[[SHOULD_ALLOC:.*]] = cir.coro.intrinsic.alloc(%[[CORO_ID]])
+// CIR-FLAT: cir.store %[[NULL]], %[[FRAME_ADDR]]
+// CIR-FLAT: cir.brcond %[[SHOULD_ALLOC]] ^[[CORO_ALLOC:bb[0-9]+]], ^[[CORO_INIT:bb[0-9]+]]
+// CIR-FLAT: ^[[CORO_ALLOC]]:
+// CIR-FLAT: %[[SIZE:.*]] = cir.coro.intrinsic.size()
+// CIR-FLAT: %[[MEM:.*]] = cir.call @_Znwm(%[[SIZE]])
+// CIR-FLAT: cir.store %[[MEM]], %[[FRAME_ADDR]]
+// CIR-FLAT: cir.br ^[[CORO_INIT]]
+// CIR-FLAT: ^[[CORO_INIT]]:
+// CIR-FLAT: %[[MEM_RELOAD:.*]] = cir.load %[[FRAME_ADDR]]
+// CIR-FLAT: %[[FRAME:.*]] = cir.coro.intrinsic.begin(%[[CORO_ID]], %[[MEM_RELOAD]])
+
+// Initial suspend.
+
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type17get_return_objectEv(%[[PROMISE]])
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(%[[PROMISE]])
+// CIR-FLAT: cir.br ^[[INIT_READY:bb[0-9]+]]
+// CIR-FLAT: ^[[INIT_READY]]:
+// CIR-FLAT: %[[INIT_IS_READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[INIT_AWAITER]])
+// CIR-FLAT: cir.brcond %[[INIT_IS_READY]] ^[[INIT_RESUME:bb[0-9]+]], ^[[INIT_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[INIT_SUSPEND]]:
+// CIR-FLAT: %[[INIT_SAVE_ARG:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
+// CIR-FLAT: %[[INIT_SAVE:.*]] = cir.coro.intrinsic.save(%[[INIT_SAVE_ARG]]) : (!cir.ptr<!void>) -> token
+// CIR-FLAT: cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%[[FRAME]])
+// CIR-FLAT: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[INIT_AWAITER]])
+// CIR-FLAT: %[[INIT_IS_FINAL:.*]] = cir.const #false
+// CIR-FLAT: %[[INIT_SUSPEND_RES:.*]] = cir.coro.intrinsic.suspend(%[[INIT_SAVE]], %[[INIT_IS_FINAL]]) : (token, !cir.bool) -> !s8i
+// CIR-FLAT: cir.switch.flat %[[INIT_SUSPEND_RES]] : !s8i, ^[[INIT_SUSPENDED:bb[0-9]+]] [
+// CIR-FLAT-NEXT: 0: ^[[INIT_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[INIT_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[INIT_DESTROY]]:
+// CIR-FLAT: %[[DEST_0:.*]] = cir.const #cir.int<0> : !s32i
+// CIR-FLAT: cir.store %[[DEST_0]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CORO_CLEANUP:bb[0-9]+]]
+// CIR-FLAT: ^[[INIT_SUSPENDED]]:
+// CIR-FLAT: cir.br ^[[CORO_RET:bb[0-9]+]]
+// CIR-FLAT: ^[[INIT_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[INIT_AWAITER]])
+
+// User written co_await.
+
+// CIR-FLAT: %[[USER_IS_READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[USER_AWAITER]])
+// CIR-FLAT: cir.brcond %[[USER_IS_READY]] ^[[USER_RESUME:bb[0-9]+]], ^[[USER_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_SUSPEND]]:
+// CIR-FLAT: %[[USER_SAVE:.*]] = cir.coro.intrinsic.save(
+// CIR-FLAT: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[USER_AWAITER]])
+// CIR-FLAT: %[[USER_IS_FINAL:.*]] = cir.const #false
+// CIR-FLAT: %[[USER_SUSPEND_RES:.*]] = cir.coro.intrinsic.suspend(%[[USER_SAVE]], %[[USER_IS_FINAL]])
+// CIR-FLAT: cir.switch.flat %[[USER_SUSPEND_RES]] : !s8i, ^[[USER_SUSPENDED:bb[0-9]+]] [
+// CIR-FLAT-NEXT: 0: ^[[USER_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[USER_DESTROY]]:
+// CIR-FLAT: %[[DEST_1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT: cir.store %[[DEST_1]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CORO_CLEANUP]]
+// CIR-FLAT: ^[[USER_SUSPENDED]]:
+// CIR-FLAT: cir.br ^[[CORO_RET]]
+// CIR-FLAT: ^[[USER_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[USER_AWAITER]])
+
+// The cir.co_return is replaced by a branch to the final suspend.
+
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[PROMISE]])
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[PROMISE]])
+
+// Final suspend: same as above, but llvm.coro.suspend is marked final.
+
+// CIR-FLAT: %[[FINAL_IS_READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[FINAL_AWAITER]])
+// CIR-FLAT: cir.brcond %[[FINAL_IS_READY]] ^[[FINAL_RESUME:bb[0-9]+]], ^[[FINAL_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL_SUSPEND]]:
+// CIR-FLAT: %[[FINAL_SAVE:.*]] = cir.coro.intrinsic.save(
+// CIR-FLAT: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[FINAL_AWAITER]])
+// CIR-FLAT: %[[FINAL_IS_FINAL:.*]] = cir.const #true
+// CIR-FLAT: %[[FINAL_SUSPEND_RES:.*]] = cir.coro.intrinsic.suspend(%[[FINAL_SAVE]], %[[FINAL_IS_FINAL]])
+// CIR-FLAT: cir.switch.flat %[[FINAL_SUSPEND_RES]] : !s8i, ^[[FINAL_SUSPENDED:bb[0-9]+]] [
+// CIR-FLAT-NEXT: 0: ^[[FINAL_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[FINAL_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[FINAL_DESTROY]]:
+// CIR-FLAT: %[[DEST_2:.*]] = cir.const #cir.int<2> : !s32i
+// CIR-FLAT: cir.store %[[DEST_2]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CORO_CLEANUP]]
+// CIR-FLAT: ^[[FINAL_SUSPENDED]]:
+// CIR-FLAT: cir.br ^[[CORO_RET]]
+// CIR-FLAT: ^[[FINAL_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[FINAL_AWAITER]])
+// CIR-FLAT: cir.br ^[[FINAL_DONE:bb[0-9]+]]
+
+// Falling off the end of the final suspend also runs the coroutine cleanup.
+
+// CIR-FLAT: ^[[FINAL_DONE]]:
+// CIR-FLAT: %[[DEST_3:.*]] = cir.const #cir.int<3> : !s32i
+// CIR-FLAT: cir.store %[[DEST_3]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CORO_CLEANUP]]
+
+// Coroutine cleanup (the destroy region): free the frame if
+// llvm.coro.free returns non-null, then leave through the exit region.
+
+// CIR-FLAT: ^[[CORO_CLEANUP]]:
+// CIR-FLAT: %[[MEM_TO_FREE:.*]] = cir.coro.intrinsic.free(%[[CORO_ID]], %[[FRAME]])
+// CIR-FLAT: %[[NULL2:.*]] = cir.const #cir.ptr<null>
+// CIR-FLAT: %[[NEEDS_FREE:.*]] = cir.cmp ne %[[MEM_TO_FREE]], %[[NULL2]]
+// CIR-FLAT: cir.brcond %[[NEEDS_FREE]] ^[[CORO_FREE:bb[0-9]+]], ^[[AFTER_CORO_FREE:bb[0-9]+]]
+// CIR-FLAT: ^[[CORO_FREE]]:
+// CIR-FLAT: %[[SIZE2:.*]] = cir.coro.intrinsic.size()
+// CIR-FLAT: cir.call @_ZdlPvm(%[[MEM_TO_FREE]], %[[SIZE2]])
+// CIR-FLAT: cir.br ^[[AFTER_CORO_FREE]]
+// CIR-FLAT: ^[[AFTER_CORO_FREE]]:
+// CIR-FLAT: cir.br ^[[CLEANUP_EXIT:bb[0-9]+]]
+// CIR-FLAT: ^[[CLEANUP_EXIT]]:
+// CIR-FLAT: %[[DEST:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.switch.flat %[[DEST]] : !s32i, ^[[DEST_DEFAULT:bb[0-9]+]] [
+// CIR-FLAT-NEXT: 0: ^[[FROM_INIT_DESTROY:bb[0-9]+]],
+// CIR-FLAT-NEXT: 1: ^[[FROM_USER_DESTROY:bb[0-9]+]],
+// CIR-FLAT-NEXT: 2: ^[[FROM_FINAL_DESTROY:bb[0-9]+]],
+// CIR-FLAT-NEXT: 3: ^[[FROM_FINAL_DONE:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[FROM_INIT_DESTROY]]:
+// CIR-FLAT-NEXT: cir.br ^[[EXIT:bb[0-9]+]]
+// CIR-FLAT: ^[[FROM_USER_DESTROY]]:
+// CIR-FLAT-NEXT: cir.br ^[[EXIT]]
+// CIR-FLAT: ^[[FROM_FINAL_DESTROY]]:
+// CIR-FLAT-NEXT: cir.br ^[[EXIT]]
+// CIR-FLAT: ^[[FROM_FINAL_DONE]]:
+// CIR-FLAT-NEXT: cir.br ^[[EXIT]]
+// CIR-FLAT: ^[[DEST_DEFAULT]]:
+// CIR-FLAT-NEXT: cir.unreachable
+// CIR-FLAT: ^[[EXIT]]:
+// CIR-FLAT-NEXT: cir.br ^[[CORO_RET]]
+
+// The exit region; every suspend's default destination lands here.
+
+// CIR-FLAT: ^[[CORO_RET]]:
+// CIR-FLAT: %[[END_ARG0:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!void>
+// CIR-FLAT: %[[END_ARG1:.*]] = cir.const #false
+// CIR-FLAT: %[[END_TK:.*]] = cir.token.none
+// CIR-FLAT: cir.coro.intrinsic.end(%[[END_ARG0]], %[[END_ARG1]], %[[END_TK]])
+// CIR-FLAT: cir.return
+// CIR-FLAT: }
+
folly::coro::Task<int> byRef(const std::string& s) {
co_return s.size();
}
@@ -293,6 +474,47 @@ folly::coro::Task<int> byRef(const std::string& s) {
// CIR: }, destroy : {
// CIR: }, exit : {
+// The body has no user await; co_return branches straight to final suspend.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z5byRefRKSt6string
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: %[[SIZE:.*]] = cir.try_call @_ZNKSt6string4sizeEv({{.*}}) ^{{.*}}, ^[[UNWIND:bb[0-9]+]]
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[SIZE]]) ^[[RET_VAL_CONT:bb[0-9]+]], ^[[UNWIND]]
+// CIR-FLAT: ^[[RET_VAL_CONT]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+
+// CIR-FLAT: ^[[UNWIND]]:
+// CIR-FLAT: %[[EH_TOKEN:.*]] = cir.eh.initiate cleanup : !cir.eh_token
+// CIR-FLAT: cir.br ^[[EH_CLEANUP:bb[0-9]+]](%[[EH_TOKEN]] : !cir.eh_token)
+// CIR-FLAT: ^[[EH_CLEANUP]](%[[EH_TOKEN_ARG:.*]]: !cir.eh_token):
+// CIR-FLAT: %[[CLEANUP_TOKEN:.*]] = cir.begin_cleanup %[[EH_TOKEN_ARG]]
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.call @_ZdlPvm
+// CIR-FLAT: cir.end_cleanup %[[CLEANUP_TOKEN]]
+// CIR-FLAT: cir.resume %[[EH_TOKEN_ARG]] : !cir.eh_token
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// Classic codegen has no unwind edges here: this test is built without
+// -fexceptions, so every call is a plain call (no invoke/landingpad), while
+// CIR-FLAT above emits cir.try_call plus an EH copy of the coroutine cleanup.
+
+// OGCG: define {{.*}} void @_Z5byRefRKSt6string(
+// OGCG: cleanup.cont:
+// OGCG: %[[SIZE:.*]] = call noundef i32 @_ZNKSt6string4sizeEv(
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(ptr {{.*}} %__promise, i32 noundef %[[SIZE]])
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
+
folly::coro::Task<void> silly_coro() {
std::optional<folly::coro::Task<int>> task;
{
@@ -324,6 +546,33 @@ folly::coro::Task<void> silly_coro() {
// CIR: }, destroy : {
// CIR: }, exit : {
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z10silly_corov()
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: cir.try_call @_ZNSt6stringC1EPKc
+// CIR-FLAT: cir.try_call @_Z5byRefRKSt6string
+// CIR-FLAT: cir.try_call @_ZN5folly4coro12blockingWaitIiEET_ONS0_4TaskIS2_EE
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z10silly_corov()
+// OGCG: cleanup.cont:
+// OGCG: call void @_ZNSt6stringC1EPKc(
+// OGCG: call void @_Z5byRefRKSt6string(
+// OGCG: call noundef i32 @_ZN5folly4coro12blockingWaitIiEET_ONS0_4TaskIS2_EE(
+// OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(
+// OGCG: br label %coro.final
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
+
folly::coro::Task<void> yield();
folly::coro::Task<void> yield1() {
auto t = yield();
@@ -428,7 +677,69 @@ folly::coro::Task<void> yield1() {
// CIR: cir.return
// CIR: }
-// CHECK: }
+// co_yield is flattened like a user co_await (a non-final suspend).
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z6yield1v()
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[YIELD_AWAITER:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11yield_valueES2_(%[[PROMISE]])
+// CIR-FLAT: %[[READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[YIELD_AWAITER]])
+// CIR-FLAT: cir.brcond %[[READY]] ^[[YIELD_RESUME:bb[0-9]+]], ^[[YIELD_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[YIELD_SUSPEND]]:
+// CIR-FLAT: %[[SAVE:.*]] = cir.coro.intrinsic.save(
+// CIR-FLAT: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[YIELD_AWAITER]])
+// CIR-FLAT: %[[IS_FINAL:.*]] = cir.const #false
+// CIR-FLAT: %[[RES:.*]] = cir.coro.intrinsic.suspend(%[[SAVE]], %[[IS_FINAL]])
+// CIR-FLAT: cir.switch.flat %[[RES]] : !s8i, ^[[YIELD_SUSPENDED:bb[0-9]+]] [
+// CIR-FLAT-NEXT: 0: ^[[YIELD_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[YIELD_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[YIELD_DESTROY]]:
+// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT: cir.store %[[ONE]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CORO_CLEANUP:bb[0-9]+]]
+// CIR-FLAT: ^[[YIELD_SUSPENDED]]:
+// CIR-FLAT-NEXT: cir.br ^[[CORO_RET:bb[0-9]+]]
+// CIR-FLAT: ^[[YIELD_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[YIELD_AWAITER]])
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[PROMISE]])
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: ^[[CORO_CLEANUP]]:
+// CIR-FLAT-NEXT: cir.coro.intrinsic.free
+// CIR-FLAT: ^[[CORO_RET]]:
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z6yield1v()
+// OGCG: cleanup.cont:
+// OGCG: call void @_Z5yieldv()
+// OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type11yield_valueES2_(
+// OGCG: %[[READY:.*]] = call noundef zeroext i1 @_ZNSt14suspend_always11await_readyEv(
+// OGCG: br i1 %[[READY]], label %yield.ready, label %yield.suspend
+// OGCG: yield.suspend:
+// OGCG: %[[SAVE:.*]] = call token @llvm.coro.save(ptr null)
+// OGCG: call void @llvm.coro.await.suspend.void({{.*}}, ptr @_Z6yield1v.__await_suspend_wrapper__yield)
+// OGCG: %[[SUSP:.*]] = call i8 @llvm.coro.suspend(token %[[SAVE]], i1 false)
+// OGCG: switch i8 %[[SUSP]], label %coro.ret [
+// OGCG-NEXT: i8 0, label %yield.ready
+// OGCG-NEXT: i8 1, label %yield.cleanup
+// OGCG-NEXT: ]
+// OGCG: yield.ready:
+// OGCG: call void @_ZNSt14suspend_always12await_resumeEv(
+// OGCG: %[[SLOT:.*]] = phi i32 [ 0, %yield.ready ], [ 2, %yield.cleanup ]
+// OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
folly::coro::Task<int> go(int const& val);
folly::coro::Task<int> go1() {
@@ -473,6 +784,62 @@ folly::coro::Task<int> go1() {
// CIR: }, destroy : {
// CIR: }, exit : {
+// The value produced in the resume region is spilled and reloaded after the
+// await, so it is still available in the continuation block.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z3go1v()
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[TASK:.*]] = cir.alloca "task"
+// CIR-FLAT: %[[RESUME_RVAL:.*]] = cir.alloca "__coawait_resume_rval"
+// CIR-FLAT: cir.try_call @_Z2goRKi
+// CIR-FLAT: %[[READY:.*]] = cir.call @_ZN5folly4coro4TaskIiE11await_readyEv(%[[TASK]])
+// CIR-FLAT: cir.brcond %[[READY]] ^[[RESUME:bb[0-9]+]], ^[[SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[SUSPEND]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE13await_suspendESt16coroutine_handleIvE(%[[TASK]])
+// CIR-FLAT: cir.const #false
+// CIR-FLAT: cir.coro.intrinsic.suspend
+// CIR-FLAT: ^[[RESUME]]:
+// CIR-FLAT: %[[RESUME_VAL:.*]] = cir.try_call @_ZN5folly4coro4TaskIiE12await_resumeEv(%[[TASK]]) ^[[RESUME_CONT:bb[0-9]+]], ^{{.*}}
+// CIR-FLAT: ^[[RESUME_CONT]]:
+// CIR-FLAT: cir.store %[[RESUME_VAL]], %[[RESUME_RVAL]]
+// CIR-FLAT: %[[V:.*]] = cir.load %[[RESUME_RVAL]]
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[V]])
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// Classic codegen passes the await_resume result straight to return_value
+// (no __coawait_resume_rval spill). After the co_return it jumps to
+// coro.final through the cleanup dispatch (slot 3).
+
+// OGCG: define {{.*}} void @_Z3go1v()
+// OGCG: cleanup.cont:
+// OGCG: store i32 1, ptr %[[REF:.*]], align 4
+// OGCG: call void @_Z2goRKi(ptr {{.*}} %[[REF]])
+// OGCG: br i1 %{{.*}}, label %await.ready, label %await.suspend
+// OGCG: await.suspend:
+// OGCG: %[[SAVE:.*]] = call token @llvm.coro.save(ptr null)
+// OGCG: %[[SUSP:.*]] = call i8 @llvm.coro.suspend(token %[[SAVE]], i1 false)
+// OGCG: switch i8 %[[SUSP]], label %coro.ret [
+// OGCG-NEXT: i8 0, label %await.ready
+// OGCG-NEXT: i8 1, label %await.cleanup
+// OGCG-NEXT: ]
+// OGCG: await.ready:
+// OGCG-NEXT: %[[RVAL:.*]] = call noundef i32 @_ZN5folly4coro4TaskIiE12await_resumeEv(
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(ptr {{.*}} %__promise, i32 noundef %[[RVAL]])
+// OGCG: %[[SLOT:.*]] = phi i32 [ 3, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT]], label %unreachable [
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: i32 3, label %coro.final
+// OGCG-NEXT: ]
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
+
folly::coro::Task<int> go1_lambda() {
auto task = []() -> folly::coro::Task<int> {
co_return 1;
@@ -524,6 +891,70 @@ folly::coro::Task<int> go1_lambda() {
// CIR: cir.yield
// CIR: }, destroy : {
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z10go1_lambdav()
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[TASK:.*]] = cir.alloca "task"
+// CIR-FLAT: %[[RESUME_RVAL:.*]] = cir.alloca "__coawait_resume_rval"
+// CIR-FLAT: cir.try_call @_ZZ10go1_lambdavENK3$_0clEv
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE11await_readyEv(%[[TASK]])
+// CIR-FLAT: cir.coro.intrinsic.suspend
+// CIR-FLAT: %[[RESUME_VAL:.*]] = cir.try_call @_ZN5folly4coro4TaskIiE12await_resumeEv(%[[TASK]])
+// CIR-FLAT: cir.store %[[RESUME_VAL]], %[[RESUME_RVAL]]
+// CIR-FLAT: %[[V:.*]] = cir.load %[[RESUME_RVAL]]
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[V]])
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: cir.return
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_ZZ10go1_lambdavENK3$_0clEv
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[ONE]]) ^[[RET_VAL_CONT:bb[0-9]+]],
+// CIR-FLAT: ^[[RET_VAL_CONT]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z10go1_lambdav()
+// OGCG: cleanup.cont:
+// OGCG: call void @"_ZZ10go1_lambdavENK3$_0clEv"(
+// OGCG: br i1 %{{.*}}, label %await.ready, label %await.suspend
+// OGCG: await.suspend:
+// OGCG: %[[SAVE:.*]] = call token @llvm.coro.save(ptr null)
+// OGCG: %[[SUSP:.*]] = call i8 @llvm.coro.suspend(token %[[SAVE]], i1 false)
+// OGCG: switch i8 %[[SUSP]], label %coro.ret [
+// OGCG-NEXT: i8 0, label %await.ready
+// OGCG-NEXT: i8 1, label %await.cleanup
+// OGCG-NEXT: ]
+// OGCG: await.ready:
+// OGCG-NEXT: %[[RVAL:.*]] = call noundef i32 @_ZN5folly4coro4TaskIiE12await_resumeEv(
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(ptr {{.*}} %__promise, i32 noundef %[[RVAL]])
+// OGCG: %[[SLOT:.*]] = phi i32 [ 3, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT]], label %unreachable [
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: i32 3, label %coro.final
+// OGCG-NEXT: ]
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
+
+// OGCG: define internal void @"_ZZ10go1_lambdavENK3$_0clEv"(
+// OGCG: cleanup.cont:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(ptr {{.*}} %__promise, i32 noundef 1)
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
+
folly::coro::Task<int> go4() {
auto* fn = +[](int const& i) -> folly::coro::Task<int> { co_return i; };
auto task = fn(3);
@@ -572,20 +1003,56 @@ folly::coro::Task<int> go4() {
// CIR: },)
// CIR: }, destroy : {
-// OGCG: define {{.*}}__await_suspend_wrapper__init(ptr noundef nonnull %[[Awaiter:.*]], ptr noundef %[[Handle:.*]])
-// OGCG: entry:
-// OGCG: %[[AwaiterAddr:.*]] = alloca ptr
-// OGCG: %[[HandleAddr:.*]] = alloca ptr
-// OGCG: %[[CoroHandleVoidAddr:.*]] = alloca %"struct.std::coroutine_handle"
-// OGCG: store ptr %[[Awaiter:.*]], ptr %[[AwaiterAddr]]
-// OGCG: store ptr %[[Handle:.*]], ptr %[[HandleAddr]]
-// OGCG: %[[AwaiterReload:.*]] = load ptr, ptr %[[AwaiterAddr]]
-// OGCG: %[[CoroFrameAddr:.*]] = load ptr, ptr %[[HandleAddr]]
-// OGCG: call void @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(ptr noundef %[[CoroFrameAddr]])
-// OGCG: call void @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(ptr noundef nonnull align 1 dereferenceable(1) %[[CoroHandleVoidAddr]])
-// OGCG: call void @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(ptr noundef nonnull align 1 dereferenceable(1) %[[AwaiterReload]])
-// OGCG: ret void
-// OGCG: }
+// Calls that may throw become cir.try_call; every one of them unwinds into the
+// same EH copy of the coroutine cleanup.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z3go4v()
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[TASK:.*]] = cir.alloca "task"
+// CIR-FLAT: cir.call @_ZZ3go4vENK3$_0cvPFN5folly4coro4TaskIiEERKiEEv
+// CIR-FLAT: %[[FN:.*]] = cir.cast bitcast
+// CIR-FLAT: cir.try_call %[[FN]]({{.*}}) ^{{.*}}, ^[[UNWIND:bb[0-9]+]]
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE11await_readyEv(%[[TASK]])
+// CIR-FLAT: cir.coro.intrinsic.suspend
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12await_resumeEv(%[[TASK]]) ^{{.*}}, ^[[UNWIND]]
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], {{.*}}) ^{{.*}}, ^[[UNWIND]]
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: ^[[UNWIND]]:
+// CIR-FLAT-NEXT: %[[EH_TOKEN:.*]] = cir.eh.initiate cleanup : !cir.eh_token
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.resume
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z3go4v()
+// OGCG: cleanup.cont:
+// OGCG: %[[CONV:.*]] = call noundef ptr @"_ZZ3go4vENK3$_0cvPFN5folly4coro4TaskIiEERKiEEv"(
+// OGCG: store ptr %[[CONV]], ptr %[[FN_ADDR:.*]], align 8
+// OGCG: %[[FN:.*]] = load ptr, ptr %[[FN_ADDR]], align 8
+// OGCG: store i32 3, ptr %[[REF:.*]], align 4
+// OGCG: call void %[[FN]](ptr {{.*}} %[[REF]])
+// OGCG: br i1 %{{.*}}, label %await.ready, label %await.suspend
+// OGCG: await.suspend:
+// OGCG: %[[SAVE:.*]] = call token @llvm.coro.save(ptr null)
+// OGCG: %[[SUSP:.*]] = call i8 @llvm.coro.suspend(token %[[SAVE]], i1 false)
+// OGCG: switch i8 %[[SUSP]], label %coro.ret [
+// OGCG-NEXT: i8 0, label %await.ready
+// OGCG-NEXT: i8 1, label %await.cleanup
+// OGCG-NEXT: ]
+// OGCG: await.ready:
+// OGCG-NEXT: %[[RVAL:.*]] = call noundef i32 @_ZN5folly4coro4TaskIiE12await_resumeEv(
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(ptr {{.*}} %__promise, i32 noundef %[[RVAL]])
+// OGCG: %[[SLOT:.*]] = phi i32 [ 3, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT]], label %unreachable [
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: i32 3, label %coro.final
+// OGCG-NEXT: ]
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
folly::coro::Task<int> co_returns(int x) {
if (x < 0)
@@ -654,6 +1121,58 @@ folly::coro::Task<int> co_returns(int x) {
// OGCG: final.suspend:
// OGCG: final.ready:
+// Every cir.co_return, including the ones nested in cir.if, branches to the
+// same final suspend block.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z10co_returnsi
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: cir.alloca "__coro_frame_addr"
+// CIR-FLAT: %[[X:.*]] = cir.alloca "x"
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: %[[LT:.*]] = cir.cmp lt
+// CIR-FLAT: cir.brcond %[[LT]] ^[[IF_LT:bb[0-9]+]], ^[[ELSE:bb[0-9]+]]
+// CIR-FLAT: ^[[IF_LT]]:
+// CIR-FLAT: %[[MINUS_ONE:.*]] = cir.const #cir.int<-1> : !s32i
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[MINUS_ONE]]) ^[[RET1:bb[0-9]+]],
+// CIR-FLAT: ^[[RET1]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[ELSE]]:
+// CIR-FLAT: %[[EQ:.*]] = cir.cmp eq
+// CIR-FLAT: cir.brcond %[[EQ]] ^[[IF_EQ:bb[0-9]+]], ^[[AFTER_IF:bb[0-9]+]]
+// CIR-FLAT: ^[[IF_EQ]]:
+// CIR-FLAT: %[[MINUS_TWO:.*]] = cir.const #cir.int<-2> : !s32i
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[MINUS_TWO]]) ^[[RET2:bb[0-9]+]],
+// CIR-FLAT: ^[[RET2]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
+// CIR-FLAT: ^[[AFTER_IF]]:
+
+// The user co_await between the early returns and the last co_return.
+
+// CIR-FLAT: cir.call @_ZNSt14suspend_always11await_readyEv
+// CIR-FLAT: cir.coro.intrinsic.suspend
+// CIR-FLAT: cir.switch.flat %{{.*}} : !s8i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[USER_RESUME:bb[0-9]+]],
+// CIR-FLAT-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[USER_DESTROY]]:
+// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT: cir.store %[[ONE]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[CORO_CLEANUP:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_RESUME]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: %[[X_LOAD:.*]] = cir.load {{.*}} %[[X]]
+// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+// CIR-FLAT: %[[MUL:.*]] = cir.mul nsw %[[X_LOAD]], %[[TWO]]
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[MUL]]) ^[[RET3:bb[0-9]+]],
+// CIR-FLAT: ^[[RET3]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: ^[[CORO_CLEANUP]]:
+// CIR-FLAT-NEXT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.return
+
folly::coro::Task<int> implicit_yield_terminator(int x) {
if (x < 0)
co_return -1;
@@ -670,6 +1189,45 @@ folly::coro::Task<int> implicit_yield_terminator(int x) {
// CIR: cir.yield
// CIR: }, finalSuspend : {
+// Both branches end in a cir.co_return; the implicit cir.yield after the
+// cir.if is unreachable but still branches to the final suspend.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z25implicit_yield_terminatori
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: cir.brcond %{{.*}} ^[[THEN:bb[0-9]+]], ^[[ELSE:bb[0-9]+]]
+// CIR-FLAT: ^[[THEN]]:
+// CIR-FLAT: %[[MINUS_ONE:.*]] = cir.const #cir.int<-1> : !s32i
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[MINUS_ONE]]) ^[[RET1:bb[0-9]+]],
+// CIR-FLAT: ^[[RET1]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[ELSE]]:
+// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[TWO]]) ^[[RET2:bb[0-9]+]],
+// CIR-FLAT: ^[[RET2]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z25implicit_yield_terminatori(
+// OGCG: cleanup.cont:
+// OGCG: %[[CMP:.*]] = icmp slt i32 %{{.*}}, 0
+// OGCG: br i1 %[[CMP]], label %if.then, label %if.else
+// OGCG: if.then:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(ptr {{.*}} %__promise, i32 noundef -1)
+// OGCG-NEXT: br label %coro.final
+// OGCG: if.else:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(ptr {{.*}} %__promise, i32 noundef 2)
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
+
struct HasDtor {
~ HasDtor();
};
@@ -735,6 +1293,574 @@ folly::coro::Task<int> co_return_with_dtor(int flag) {
// OGCG: call void @_ZN7HasDtorD1Ev({{.*}} %[[LOCAL]])
// OGCG: br label %coro.final
+// A cir.co_return nested in a cleanup scope exits through the cleanup: it
+// records its destination in the cleanup dest slot, runs ~HasDtor, and only
+// then continues to the final suspend.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z19co_return_with_dtori
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[LOCAL:.*]] = cir.alloca "local"
+// CIR-FLAT: cir.call @_ZNSt14suspend_always12await_resumeEv
+// CIR-FLAT: %[[FLAG:.*]] = cir.cast int_to_bool
+// CIR-FLAT: cir.brcond %[[FLAG]] ^[[IF_BODY:bb[0-9]+]], ^[[IF_END:bb[0-9]+]]
+// CIR-FLAT: ^[[IF_BODY]]:
+// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[ONE]]) ^[[RET1:bb[0-9]+]],
+// CIR-FLAT: ^[[RET1]]:
+// CIR-FLAT: %[[DEST_0:.*]] = cir.const #cir.int<0> : !s32i
+// CIR-FLAT: cir.store %[[DEST_0]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[DTOR:bb[0-9]+]]
+// CIR-FLAT: ^[[IF_END]]:
+// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[TWO]]) ^[[RET2:bb[0-9]+]],
+// CIR-FLAT: ^[[RET2]]:
+// CIR-FLAT: %[[DEST_1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT: cir.store %[[DEST_1]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT: cir.br ^[[DTOR]]
+// CIR-FLAT: ^[[DTOR]]:
+// CIR-FLAT-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[LOCAL]])
+// CIR-FLAT-NEXT: cir.br ^[[DTOR_EXIT:bb[0-9]+]]
+// CIR-FLAT: ^[[DTOR_EXIT]]:
+// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.switch.flat %[[DEST]] : !s32i, ^[[DEFAULT:bb[0-9]+]] [
+// CIR-FLAT-NEXT: 0: ^[[FROM_RET1:bb[0-9]+]],
+// CIR-FLAT-NEXT: 1: ^[[FROM_RET2:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[FROM_RET1]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FROM_RET2]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
+// CIR-FLAT: ^[[DEFAULT]]:
+// CIR-FLAT-NEXT: cir.unreachable
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.return
+
+VoidTask silly_task_with_dtor() {
+ HasDtor local;
+ co_await std::suspend_always();
+}
+
+// CIR: cir.func coroutine {{.*}} @_Z20silly_task_with_dtorv
+// CIR: %[[LOCAL:.*]] = cir.alloca "local"
+// CIR: }, body : {
+// CIR: cir.cleanup.scope {
+// CIR: cir.await(user, ready : {
+// CIR: }, suspend : {
+// CIR: }, resume : {
+// CIR: },)
+// CIR: cir.yield
+// CIR: } cleanup normal {
+// CIR: cir.call @_ZN7HasDtorD1Ev(%[[LOCAL]])
+// CIR: cir.yield
+// CIR: }
+// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR: cir.co_return
+// CIR: }, finalSuspend : {
+
+// The user await is nested in the lifetime of `local`, so both of its exits
+// run ~HasDtor first and then dispatch on the cleanup dest slot
+// (0 = destroy, 1 = resume).
+//
+// FIXME: the destroy path is wrong. After ~HasDtor, both cases branch to the
+// rest of the body (return_void + final_suspend), so a destroyed coroutine
+// keeps running. Classic codegen (OGCG below) sends the destroy case straight
+// to %coro.cleanup (coro.free). The cause is that CIRAwaitOpFlattening ends
+// the destroy block with a cir.yield, which every enclosing cir.cleanup.scope
+// (and loop) treats as a normal fallthrough; it only reaches coro.free when
+// the await sits directly in the coroutine body. The same bug makes an await
+// inside a loop continue with the next iteration on destroy. A dedicated
+// terminator that createExitTerminator propagates through cleanup scopes
+// (like cir.co_return) and that CIRCoroutineFlattening turns into a branch to
+// the coroutine cleanup should fix it. Once fixed, DTOR_TO_DESTROY should
+// branch to CORO_CLEANUP instead of AFTER_DTOR.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z20silly_task_with_dtorv()
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[LOCAL:.*]] = cir.alloca "local"
+// CIR-FLAT: %[[USER_AWAITER:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: %[[READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[USER_AWAITER]])
+// CIR-FLAT: cir.brcond %[[READY]] ^[[USER_RESUME:bb[0-9]+]], ^[[USER_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_SUSPEND]]:
+// CIR-FLAT: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[USER_AWAITER]])
+// CIR-FLAT: %[[RES:.*]] = cir.coro.intrinsic.suspend
+// CIR-FLAT: cir.switch.flat %[[RES]] : !s8i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[USER_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[USER_DESTROY]]:
+// CIR-FLAT-NEXT: %[[DEST_DESTROY:.*]] = cir.const #cir.int<0> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_DESTROY]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_RESUME]]:
+// CIR-FLAT-NEXT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[USER_AWAITER]])
+// CIR-FLAT: %[[DEST_NORMAL:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_NORMAL]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR]]
+// CIR-FLAT: ^[[DTOR]]:
+// CIR-FLAT-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[LOCAL]])
+// CIR-FLAT-NEXT: cir.br ^[[DTOR_EXIT:bb[0-9]+]]
+// CIR-FLAT: ^[[DTOR_EXIT]]:
+// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.switch.flat %[[DEST]] : !s32i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[DTOR_TO_DESTROY:bb[0-9]+]],
+// CIR-FLAT-NEXT: 1: ^[[DTOR_TO_BODY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+
+// Both cases continue into the rest of the body (see FIXME above).
+// CIR-FLAT: ^[[DTOR_TO_DESTROY]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_DTOR:bb[0-9]+]]
+// CIR-FLAT: ^[[DTOR_TO_BODY]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_DTOR]]
+// CIR-FLAT: ^[[AFTER_DTOR]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[PROMISE]])
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[PROMISE]])
+
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z20silly_task_with_dtorv
+// OGCG: %[[LOCAL:.*]] = alloca %struct.HasDtor
+// OGCG: await.suspend:
+// OGCG: switch i8 %{{.*}}, label %coro.ret [
+// OGCG-NEXT: i8 0, label %await.ready
+// OGCG-NEXT: i8 1, label %await.cleanup
+// OGCG-NEXT: ]
+// OGCG: await.cleanup:
+// OGCG-NEXT: br label %[[AWAIT_CLEANUP:cleanup[0-9]+]]
+// OGCG: await.ready:
+// OGCG-NEXT: call void @_ZNSt14suspend_always12await_resumeEv(
+// OGCG-NEXT: br label %[[AWAIT_CLEANUP]]
+
+// Slot 0 = normal, slot 2 = destroy. Both reach the ~HasDtor cleanup.
+// OGCG: [[AWAIT_CLEANUP]]:
+// OGCG-NEXT: %[[SLOT1:.*]] = phi i32 [ 0, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT1]], label %[[DTOR:cleanup[0-9]+]] [
+// OGCG-NEXT: i32 0, label %[[AWAIT_CONT:cleanup.cont[0-9]+]]
+// OGCG-NEXT: ]
+// OGCG: [[AWAIT_CONT]]:
+// OGCG-NEXT: br label %[[DTOR]]
+// OGCG: [[DTOR]]:
+// OGCG-NEXT: %[[SLOT2:.*]] = phi i32 [ %[[SLOT1]], %[[AWAIT_CLEANUP]] ], [ 0, %[[AWAIT_CONT]] ]
+// OGCG-NEXT: call void @_ZN7HasDtorD1Ev({{.*}} %[[LOCAL]])
+// OGCG: switch i32 %[[SLOT2]], label %unreachable [
+// OGCG-NEXT: i32 0, label %[[DTOR_CONT:cleanup.cont[0-9]+]]
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: ]
+
+// Normal: return_void + final suspend. Destroy: %coro.cleanup (coro.free).
+// OGCG: [[DTOR_CONT]]:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.final:
+// OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+// OGCG: coro.ret:
+// OGCG-NEXT: call void @llvm.coro.end(ptr null, i1 false, token none)
+// OGCG-NEXT: ret void
+
+VoidTask await_in_loop(int n) {
+ for (int i = 0; i < n; ++i)
+ co_await std::suspend_always();
+}
+
+// CIR: cir.func coroutine {{.*}} @_Z13await_in_loopi
+// CIR: }, body : {
+// CIR: cir.scope {
+// CIR: cir.for : cond {
+// CIR: } body {
+// CIR: cir.await(user, ready : {
+// CIR: }, suspend : {
+// CIR: cir.coro.suspend_point
+// CIR: }, resume : {
+// CIR: },)
+// CIR: cir.yield
+// CIR: } step {
+// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR: cir.co_return
+// CIR: }, finalSuspend : {
+
+// FIXME: same destroy-path bug as silly_task_with_dtor. The destroy case of
+// the user await ends in a cir.yield, which the loop flattening treats as the
+// end of the loop body, so a destroyed coroutine runs the step (++i) and keeps
+// iterating. Classic codegen (OGCG below) leaves the loop and goes to
+// %coro.cleanup. Once fixed, USER_DESTROY should reach coro.free instead of
+// STEP.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z13await_in_loopi(
+// CIR-FLAT: %[[I:.*]] = cir.alloca "i"
+// CIR-FLAT: %[[USER_AWAITER:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: cir.store {{.*}} %{{.*}}, %[[I]]
+// CIR-FLAT-NEXT: cir.br ^[[COND:bb[0-9]+]]
+// CIR-FLAT: ^[[COND]]:
+// CIR-FLAT: %[[CMP:.*]] = cir.cmp lt
+// CIR-FLAT-NEXT: cir.brcond %[[CMP]] ^{{.*}}, ^[[LOOP_EXIT:bb[0-9]+]]
+// CIR-FLAT: %[[READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[USER_AWAITER]])
+// CIR-FLAT-NEXT: cir.brcond %[[READY]] ^[[USER_RESUME:bb[0-9]+]], ^[[USER_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_SUSPEND]]:
+// CIR-FLAT: %[[RES:.*]] = cir.coro.intrinsic.suspend
+// CIR-FLAT: cir.switch.flat %[[RES]] : !s8i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[USER_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[USER_DESTROY]]:
+// CIR-FLAT-NEXT: cir.br ^[[STEP:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_RESUME]]:
+// CIR-FLAT-NEXT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[USER_AWAITER]])
+// CIR-FLAT: ^[[STEP]]:
+// CIR-FLAT-NEXT: %[[I_VAL:.*]] = cir.load {{.*}}%[[I]]
+// CIR-FLAT-NEXT: %[[INC:.*]] = cir.inc nsw %[[I_VAL]]
+// CIR-FLAT-NEXT: cir.store {{.*}}%[[INC]], %[[I]]
+// CIR-FLAT-NEXT: cir.br ^[[COND]]
+// CIR-FLAT: ^[[LOOP_EXIT]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// In classic codegen the destroy case (slot 2) leaves the loop through the
+// loop cleanup and reaches %coro.cleanup; only the normal case (slot 0)
+// continues to for.inc.
+
+// OGCG: define {{.*}} void @_Z13await_in_loopi(
+// OGCG: await.suspend:
+// OGCG: switch i8 %{{.*}}, label %coro.ret [
+// OGCG-NEXT: i8 0, label %await.ready
+// OGCG-NEXT: i8 1, label %await.cleanup
+// OGCG-NEXT: ]
+// OGCG: await.cleanup:
+// OGCG-NEXT: br label %[[AWAIT_CLEANUP:cleanup[0-9]+]]
+// OGCG: [[AWAIT_CLEANUP]]:
+// OGCG-NEXT: %[[SLOT1:.*]] = phi i32 [ 0, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT1]], label %[[LOOP_CLEANUP:cleanup[0-9]+]] [
+// OGCG-NEXT: i32 0, label %[[AWAIT_CONT:cleanup.cont[0-9]+]]
+// OGCG-NEXT: ]
+// OGCG: [[AWAIT_CONT]]:
+// OGCG-NEXT: br label %for.inc
+// OGCG: [[LOOP_CLEANUP]]:
+// OGCG-NEXT: %[[SLOT2:.*]] = phi i32 [ %[[SLOT1]], %[[AWAIT_CLEANUP]] ], [ 4, %for.cond.cleanup ]
+// OGCG: switch i32 %[[SLOT2]], label %unreachable [
+// OGCG-NEXT: i32 4, label %for.end
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: ]
+// OGCG: for.end:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+
+VoidTask await_in_if(bool b) {
+ if (b)
+ co_await std::suspend_always();
+}
+
+// CIR: cir.func coroutine {{.*}} @_Z11await_in_ifb
+// CIR: }, body : {
+// CIR: cir.scope {
+// CIR: cir.if %{{.*}} {
+// CIR: cir.await(user, ready : {
+// CIR: }, suspend : {
+// CIR: cir.coro.suspend_point
+// CIR: }, resume : {
+// CIR: },)
+// CIR: }
+// CIR: }
+// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR: cir.co_return
+// CIR: }, finalSuspend : {
+
+// The destroy path is correct here, but only because cir.if flattening
+// rewrites just the terminator of the last block in each branch, so the
+// destroy block's cir.yield survives until the coroutine cleanup scope is
+// flattened. Adding a local with a destructor breaks it (see
+// await_in_if_with_dtor). Keep this as a regression test for the
+// destroy-path fix.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z11await_in_ifb(
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: cir.alloca "__coro_frame_addr"
+// CIR-FLAT: %[[B:.*]] = cir.alloca "b"
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[COND:.*]] = cir.load {{.*}}%[[B]]
+// CIR-FLAT-NEXT: cir.brcond %[[COND]] ^[[THEN:bb[0-9]+]], ^[[IF_END:bb[0-9]+]]
+// CIR-FLAT: ^[[THEN]]:
+// CIR-FLAT-NEXT: %[[USER_AWAITER:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: %[[READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[USER_AWAITER]])
+// CIR-FLAT-NEXT: cir.brcond %[[READY]] ^[[USER_RESUME:bb[0-9]+]], ^[[USER_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_SUSPEND]]:
+// CIR-FLAT: %[[RES:.*]] = cir.coro.intrinsic.suspend
+// CIR-FLAT: cir.switch.flat %[[RES]] : !s8i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[USER_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[USER_DESTROY]]:
+// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[CORO_CLEANUP:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_RESUME]]:
+// CIR-FLAT-NEXT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[USER_AWAITER]])
+// CIR-FLAT: ^[[IF_END]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_IF:bb[0-9]+]]
+// CIR-FLAT: ^[[AFTER_IF]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[PROMISE]])
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: ^[[CORO_CLEANUP]]:
+// CIR-FLAT-NEXT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z11await_in_ifb(
+// OGCG: cleanup.cont:
+// OGCG: br i1 %{{.*}}, label %if.then, label %if.end
+// OGCG: await.suspend:
+// OGCG: switch i8 %{{.*}}, label %coro.ret [
+// OGCG-NEXT: i8 0, label %await.ready
+// OGCG-NEXT: i8 1, label %await.cleanup
+// OGCG-NEXT: ]
+// OGCG: await.cleanup:
+// OGCG-NEXT: br label %[[AWAIT_CLEANUP:cleanup[0-9]+]]
+// OGCG: [[AWAIT_CLEANUP]]:
+// OGCG-NEXT: %[[SLOT:.*]] = phi i32 [ 0, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT]], label %unreachable [
+// OGCG-NEXT: i32 0, label %[[AWAIT_CONT:cleanup.cont[0-9]+]]
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: ]
+// OGCG: [[AWAIT_CONT]]:
+// OGCG-NEXT: br label %if.end
+// OGCG: if.end:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+
+VoidTask await_in_loop_with_dtor(int n) {
+ for (int i = 0; i < n; ++i) {
+ HasDtor local;
+ co_await std::suspend_always();
+ }
+}
+
+// CIR: cir.func coroutine {{.*}} @_Z23await_in_loop_with_dtori
+// CIR: }, body : {
+// CIR: cir.for : cond {
+// CIR: } body {
+// CIR: cir.cleanup.scope {
+// CIR: cir.await(user, ready : {
+// CIR: },)
+// CIR: cir.yield
+// CIR: } cleanup normal {
+// CIR: cir.call @_ZN7HasDtorD1Ev
+// CIR: cir.yield
+// CIR: }
+// CIR: } step {
+// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR: cir.co_return
+// CIR: }, finalSuspend : {
+
+// FIXME: same destroy-path bug as silly_task_with_dtor and await_in_loop.
+// After ~HasDtor both slots (0 = destroy, 1 = resume) continue to the end of
+// the loop body and then to the step (++i), so a destroyed coroutine keeps
+// iterating. Classic codegen (OGCG below) leaves the loop and goes to
+// %coro.cleanup. Once fixed, DTOR_TO_DESTROY should reach coro.free instead
+// of AFTER_DTOR.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z23await_in_loop_with_dtori(
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: %[[I:.*]] = cir.alloca "i"
+// CIR-FLAT: cir.store {{.*}} %{{.*}}, %[[I]]
+// CIR-FLAT-NEXT: cir.br ^[[COND:bb[0-9]+]]
+// CIR-FLAT: ^[[COND]]:
+// CIR-FLAT: %[[CMP:.*]] = cir.cmp lt
+// CIR-FLAT-NEXT: cir.brcond %[[CMP]] ^{{.*}}, ^[[LOOP_EXIT:bb[0-9]+]]
+// CIR-FLAT: %[[LOCAL:.*]] = cir.alloca "local"
+// CIR-FLAT-NEXT: %[[USER_AWAITER:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: %[[READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[USER_AWAITER]])
+// CIR-FLAT-NEXT: cir.brcond %[[READY]] ^[[USER_RESUME:bb[0-9]+]], ^[[USER_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_SUSPEND]]:
+// CIR-FLAT: %[[RES:.*]] = cir.coro.intrinsic.suspend
+// CIR-FLAT: cir.switch.flat %[[RES]] : !s8i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[USER_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[USER_DESTROY]]:
+// CIR-FLAT-NEXT: %[[DEST_DESTROY:.*]] = cir.const #cir.int<0> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_DESTROY]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_RESUME]]:
+// CIR-FLAT-NEXT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[USER_AWAITER]])
+// CIR-FLAT: %[[DEST_NORMAL:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_NORMAL]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR]]
+// CIR-FLAT: ^[[DTOR]]:
+// CIR-FLAT-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[LOCAL]])
+// CIR-FLAT-NEXT: cir.br ^[[DTOR_EXIT:bb[0-9]+]]
+// CIR-FLAT: ^[[DTOR_EXIT]]:
+// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.switch.flat %[[DEST]] : !s32i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[DTOR_TO_DESTROY:bb[0-9]+]],
+// CIR-FLAT-NEXT: 1: ^[[DTOR_TO_BODY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[DTOR_TO_DESTROY]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_DTOR:bb[0-9]+]]
+// CIR-FLAT: ^[[DTOR_TO_BODY]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_DTOR]]
+// CIR-FLAT: ^[[AFTER_DTOR]]:
+// CIR-FLAT-NEXT: cir.br ^[[BODY_END:bb[0-9]+]]
+// CIR-FLAT: ^[[BODY_END]]:
+// CIR-FLAT-NEXT: cir.br ^[[STEP:bb[0-9]+]]
+// CIR-FLAT: ^[[STEP]]:
+// CIR-FLAT-NEXT: %[[I_VAL:.*]] = cir.load {{.*}}%[[I]]
+// CIR-FLAT-NEXT: %[[INC:.*]] = cir.inc nsw %[[I_VAL]]
+// CIR-FLAT-NEXT: cir.store {{.*}}%[[INC]], %[[I]]
+// CIR-FLAT-NEXT: cir.br ^[[COND]]
+// CIR-FLAT: ^[[LOOP_EXIT]]:
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// Classic codegen: after ~HasDtor, only slot 0 continues to for.inc; the
+// destroy case (slot 2) goes through the loop cleanup to %coro.cleanup.
+
+// OGCG: define {{.*}} void @_Z23await_in_loop_with_dtori(
+// OGCG: %[[LOCAL:.*]] = alloca %struct.HasDtor
+// OGCG: await.cleanup:
+// OGCG-NEXT: br label %[[AWAIT_CLEANUP:cleanup[0-9]+]]
+// OGCG: [[AWAIT_CLEANUP]]:
+// OGCG-NEXT: %[[SLOT1:.*]] = phi i32 [ 0, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT1]], label %[[DTOR:cleanup[0-9]+]] [
+// OGCG: [[DTOR]]:
+// OGCG-NEXT: %[[SLOT2:.*]] = phi i32 [ %[[SLOT1]], %[[AWAIT_CLEANUP]] ], [ 0, %{{.*}} ]
+// OGCG-NEXT: call void @_ZN7HasDtorD1Ev({{.*}} %[[LOCAL]])
+// OGCG: switch i32 %[[SLOT2]], label %[[LOOP_CLEANUP:cleanup[0-9]+]] [
+// OGCG-NEXT: i32 0, label %[[DTOR_CONT:cleanup.cont[0-9]+]]
+// OGCG-NEXT: ]
+// OGCG: [[DTOR_CONT]]:
+// OGCG-NEXT: br label %for.inc
+// OGCG: [[LOOP_CLEANUP]]:
+// OGCG-NEXT: %[[SLOT3:.*]] = phi i32 [ %[[SLOT2]], %[[DTOR]] ], [ 4, %for.cond.cleanup ]
+// OGCG: switch i32 %[[SLOT3]], label %unreachable [
+// OGCG-NEXT: i32 4, label %for.end
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: ]
+// OGCG: for.end:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+
+VoidTask await_in_if_with_dtor(bool b) {
+ if (b) {
+ HasDtor local;
+ co_await std::suspend_always();
+ }
+}
+
+// CIR: cir.func coroutine {{.*}} @_Z21await_in_if_with_dtorb
+// CIR: }, body : {
+// CIR: cir.if %{{.*}} {
+// CIR: cir.cleanup.scope {
+// CIR: cir.await(user, ready : {
+// CIR: },)
+// CIR: cir.yield
+// CIR: } cleanup normal {
+// CIR: cir.call @_ZN7HasDtorD1Ev
+// CIR: cir.yield
+// CIR: }
+// CIR: }
+// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv
+// CIR: cir.co_return
+// CIR: }, finalSuspend : {
+
+// FIXME: unlike await_in_if, the destroy path is wrong here: once the await
+// is inside a cir.cleanup.scope, the destroy block's cir.yield is consumed by
+// that scope. After ~HasDtor both slots continue to the end of the if and on
+// to return_void + final_suspend. Classic codegen (OGCG below) goes to
+// %coro.cleanup. Once fixed, DTOR_TO_DESTROY should reach coro.free instead
+// of AFTER_DTOR.
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z21await_in_if_with_dtorb(
+// CIR-FLAT: %[[CLEANUP_DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+// CIR-FLAT: cir.alloca "__coro_frame_addr"
+// CIR-FLAT: %[[B:.*]] = cir.alloca "b"
+// CIR-FLAT: %[[PROMISE:.*]] = cir.alloca "__promise"
+// CIR-FLAT: %[[COND:.*]] = cir.load {{.*}}%[[B]]
+// CIR-FLAT-NEXT: cir.brcond %[[COND]] ^[[THEN:bb[0-9]+]], ^[[IF_END:bb[0-9]+]]
+// CIR-FLAT: ^[[THEN]]:
+// CIR-FLAT-NEXT: %[[LOCAL:.*]] = cir.alloca "local"
+// CIR-FLAT-NEXT: %[[USER_AWAITER:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: %[[READY:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[USER_AWAITER]])
+// CIR-FLAT-NEXT: cir.brcond %[[READY]] ^[[USER_RESUME:bb[0-9]+]], ^[[USER_SUSPEND:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_SUSPEND]]:
+// CIR-FLAT: %[[RES:.*]] = cir.coro.intrinsic.suspend
+// CIR-FLAT: cir.switch.flat %[[RES]] : !s8i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[USER_RESUME]],
+// CIR-FLAT-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[USER_DESTROY]]:
+// CIR-FLAT-NEXT: %[[DEST_DESTROY:.*]] = cir.const #cir.int<0> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_DESTROY]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR:bb[0-9]+]]
+// CIR-FLAT: ^[[USER_RESUME]]:
+// CIR-FLAT-NEXT: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[USER_AWAITER]])
+// CIR-FLAT: %[[DEST_NORMAL:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_NORMAL]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR]]
+// CIR-FLAT: ^[[DTOR]]:
+// CIR-FLAT-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[LOCAL]])
+// CIR-FLAT-NEXT: cir.br ^[[DTOR_EXIT:bb[0-9]+]]
+// CIR-FLAT: ^[[DTOR_EXIT]]:
+// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.switch.flat %[[DEST]] : !s32i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^[[DTOR_TO_DESTROY:bb[0-9]+]],
+// CIR-FLAT-NEXT: 1: ^[[DTOR_TO_BODY:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[DTOR_TO_DESTROY]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_DTOR:bb[0-9]+]]
+// CIR-FLAT: ^[[DTOR_TO_BODY]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_DTOR]]
+// CIR-FLAT: ^[[AFTER_DTOR]]:
+// CIR-FLAT-NEXT: cir.br ^[[IF_END]]
+// CIR-FLAT: ^[[IF_END]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_IF:bb[0-9]+]]
+// CIR-FLAT: ^[[AFTER_IF]]:
+// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[PROMISE]])
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[PROMISE]])
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT: cir.return
+
+// OGCG: define {{.*}} void @_Z21await_in_if_with_dtorb(
+// OGCG: %[[LOCAL:.*]] = alloca %struct.HasDtor
+// OGCG: await.cleanup:
+// OGCG-NEXT: br label %[[AWAIT_CLEANUP:cleanup[0-9]+]]
+// OGCG: [[AWAIT_CLEANUP]]:
+// OGCG-NEXT: %[[SLOT1:.*]] = phi i32 [ 0, %await.ready ], [ 2, %await.cleanup ]
+// OGCG: switch i32 %[[SLOT1]], label %[[DTOR:cleanup[0-9]+]] [
+// OGCG: [[DTOR]]:
+// OGCG-NEXT: %[[SLOT2:.*]] = phi i32 [ %[[SLOT1]], %[[AWAIT_CLEANUP]] ], [ 0, %{{.*}} ]
+// OGCG-NEXT: call void @_ZN7HasDtorD1Ev({{.*}} %[[LOCAL]])
+// OGCG: switch i32 %[[SLOT2]], label %unreachable [
+// OGCG-NEXT: i32 0, label %[[DTOR_CONT:cleanup.cont[0-9]+]]
+// OGCG-NEXT: i32 2, label %coro.cleanup
+// OGCG-NEXT: ]
+// OGCG: [[DTOR_CONT]]:
+// OGCG-NEXT: br label %if.end
+// OGCG: if.end:
+// OGCG-NEXT: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(
+// OGCG-NEXT: br label %coro.final
+// OGCG: coro.cleanup:
+// OGCG: call ptr @llvm.coro.free(
+
folly::coro::Task<int __complex__> fetchData() noexcept {
int __complex__ a;
co_return a;
@@ -779,6 +1905,25 @@ folly::coro::Task<int __complex__> complex_co_await() noexcept {
// CIR: }, destroy : {
// CIR: }, exit : {
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z9fetchDatav()
+
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z16complex_co_awaitv()
+// CIR-FLAT: %[[COERCE_ADDR:.*]] = cir.alloca "coerce"
+// CIR-FLAT: %[[TASK_ADDR:.*]] = cir.alloca "ref.tmp1"
+// CIR-FLAT: %[[RESUME_VAL_ADDR:.*]] = cir.alloca "__coawait_resume_rval"
+// CIR-FLAT: cir.call @_Z9fetchDatav()
+// CIR-FLAT: cir.call @_ZN5folly4coro4TaskICiE11await_readyEv(%[[TASK_ADDR]])
+// CIR-FLAT: cir.coro.intrinsic.suspend
+// CIR-FLAT: %[[RESUME_VAL:.*]] = cir.try_call @_ZN5folly4coro4TaskICiE12await_resumeEv(%[[TASK_ADDR]]) ^[[RESUME_CONT:bb[0-9]+]],
+// CIR-FLAT: ^[[RESUME_CONT]]:
+// CIR-FLAT-NEXT: cir.store %[[RESUME_VAL]], %[[COERCE_ADDR]] : !u64i, !cir.ptr<!u64i>
+// CIR-FLAT-NEXT: %[[COERCE_CAST:.*]] = cir.cast bitcast %[[COERCE_ADDR]]
+// CIR-FLAT-NEXT: %[[RESUME_COMPLEX:.*]] = cir.load %[[COERCE_CAST]]
+// CIR-FLAT-NEXT: cir.store %[[RESUME_COMPLEX]], %[[RESUME_VAL_ADDR]]
+// CIR-FLAT: cir.load %[[RESUME_VAL_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.return
+
// OGCG: define dso_local void @_Z16complex_co_awaitv()
// OGCG: %[[RESUME_VAL_ADDR:.*]] = alloca { i32, i32 }, align 4
@@ -812,6 +1957,16 @@ void void_coroutine() {
// CIR: cir.return
// CIR: }
+// CIR-FLAT-LABEL: cir.func flattened_coroutine {{.*}} @_Z14void_coroutinev()
+// CIR-FLAT: cir.call @_ZNSt13suspend_never11await_readyEv
+// CIR-FLAT: cir.call @_ZNSt16coroutine_traitsIvJEE12promise_type11return_voidEv
+// CIR-FLAT: cir.call @_ZNSt16coroutine_traitsIvJEE12promise_type13final_suspendEv
+// CIR-FLAT: %[[IS_FINAL:.*]] = cir.const #true
+// CIR-FLAT: cir.coro.intrinsic.suspend(%{{.*}}, %[[IS_FINAL]])
+// CIR-FLAT: cir.coro.intrinsic.free
+// CIR-FLAT: cir.coro.intrinsic.end
+// CIR-FLAT-NEXT: cir.return
+
// OGCG: define {{.*}} void @_Z14void_coroutinev()
// OGCG: coro.ret:
// OGCG-NEXT: call void @llvm.coro.end
diff --git a/clang/test/CIR/Transforms/flatten-coroutine.cir b/clang/test/CIR/Transforms/flatten-coroutine.cir
new file mode 100644
index 00000000000000..f97d5109390874
--- /dev/null
+++ b/clang/test/CIR/Transforms/flatten-coroutine.cir
@@ -0,0 +1,226 @@
+// RUN: cir-opt %s -cir-flatten-cfg -o - | FileCheck %s \
+// RUN: --implicit-check-not=cir.coroutine --implicit-check-not=cir.await \
+// RUN: --implicit-check-not=cir.co_return --implicit-check-not=cir.coro.exit \
+// RUN: --implicit-check-not=cir.coro.suspend_point
+
+// FlattenCFG lowers a cir.coroutine as follows:
+// - initialSuspend, body and finalSuspend become the body of a cleanup scope
+// whose cleanup is the destroy region, followed by the exit region.
+// - Each cir.await becomes ready -> brcond(resume, suspend). The suspend
+// block calls cir.coro.intrinsic.save/suspend and switches on the result:
+// 0 resumes, 1 destroys (runs the destroy region), and the default (-1,
+// the coroutine stays suspended) jumps straight to the exit region.
+// - cir.co_return becomes a branch to the final suspend.
+
+module {
+ cir.func private @prelude()
+ cir.func private @await_suspend()
+ cir.func private @await_resume()
+ cir.func private @user_code()
+ cir.func private @return_void()
+ cir.func private @free_frame()
+ cir.func private @coro_end()
+
+ cir.func coroutine @minimal_coroutine(%ready : !cir.bool) {
+ cir.coroutine initialSuspend : {
+ cir.call @prelude() nothrow : () -> ()
+ cir.await(init, ready : {
+ cir.condition(%ready)
+ }, suspend : {
+ cir.call @await_suspend() nothrow : () -> ()
+ cir.coro.suspend_point
+ }, resume : {
+ cir.call @await_resume() nothrow : () -> ()
+ cir.yield
+ },)
+ cir.yield
+ }, body : {
+ cir.call @user_code() nothrow : () -> ()
+ cir.yield
+ }, finalSuspend : {
+ cir.await(final, ready : {
+ cir.condition(%ready)
+ }, suspend : {
+ cir.coro.suspend_point
+ }, resume : {
+ cir.yield
+ },)
+ cir.yield
+ }, destroy : {
+ cir.call @free_frame() nothrow : () -> ()
+ cir.yield
+ }, exit : {
+ cir.call @coro_end() nothrow : () -> ()
+ cir.return
+ }
+ cir.trap
+ }
+
+// CHECK-LABEL: cir.func flattened_coroutine @minimal_coroutine(%arg0: !cir.bool)
+// CHECK: %[[DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot
+
+// initialSuspend: the prelude, then the init await.
+// CHECK: cir.call @prelude()
+// CHECK-NEXT: cir.br ^[[INIT_READY:bb[0-9]+]]
+// CHECK: ^[[INIT_READY]]:
+// CHECK-NEXT: cir.brcond %arg0 ^[[INIT_RESUME:bb[0-9]+]], ^[[INIT_SUSPEND:bb[0-9]+]]
+// CHECK: ^[[INIT_SUSPEND]]:
+// CHECK-NEXT: %[[NULL:.*]] = cir.const #cir.ptr<null>
+// CHECK-NEXT: %[[SAVE:.*]] = cir.coro.intrinsic.save(%[[NULL]])
+// CHECK-NEXT: cir.call @await_suspend()
+// CHECK-NEXT: %[[NOT_FINAL:.*]] = cir.const #false
+// CHECK-NEXT: %[[RES:.*]] = cir.coro.intrinsic.suspend(%[[SAVE]], %[[NOT_FINAL]])
+// CHECK-NEXT: cir.switch.flat %[[RES]] : !s8i, ^[[INIT_SUSPENDED:bb[0-9]+]] [
+// CHECK-NEXT: 0: ^[[INIT_RESUME]],
+// CHECK-NEXT: 1: ^[[INIT_DESTROY:bb[0-9]+]]
+// CHECK-NEXT: ]
+
+// Destroy: record where we came from and run the destroy region.
+// CHECK: ^[[INIT_DESTROY]]:
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CHECK-NEXT: cir.store %[[ZERO]], %[[DEST_SLOT]]
+// CHECK-NEXT: cir.br ^[[DESTROY:bb[0-9]+]]
+
+// Suspended: leave through the exit region without running the destroy region.
+// CHECK: ^[[INIT_SUSPENDED]]:
+// CHECK-NEXT: cir.br ^[[EXIT:bb[0-9]+]]
+
+// Resume, then fall through into the body.
+// CHECK: ^[[INIT_RESUME]]:
+// CHECK-NEXT: cir.call @await_resume()
+// CHECK: cir.call @user_code()
+
+// The body falls off the end (cir.yield) into the final suspend.
+// CHECK: cir.brcond %arg0 ^[[FINAL_RESUME:bb[0-9]+]], ^[[FINAL_SUSPEND:bb[0-9]+]]
+// CHECK: ^[[FINAL_SUSPEND]]:
+// CHECK: %[[IS_FINAL:.*]] = cir.const #true
+// CHECK-NEXT: %[[FINAL_RES:.*]] = cir.coro.intrinsic.suspend(%{{.*}}, %[[IS_FINAL]])
+// CHECK-NEXT: cir.switch.flat %[[FINAL_RES]] : !s8i, ^[[FINAL_SUSPENDED:bb[0-9]+]] [
+// CHECK-NEXT: 0: ^[[FINAL_RESUME]],
+// CHECK-NEXT: 1: ^[[FINAL_DESTROY:bb[0-9]+]]
+// CHECK-NEXT: ]
+// CHECK: ^[[FINAL_DESTROY]]:
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CHECK-NEXT: cir.store %[[ONE]], %[[DEST_SLOT]]
+// CHECK-NEXT: cir.br ^[[DESTROY]]
+// CHECK: ^[[FINAL_SUSPENDED]]:
+// CHECK-NEXT: cir.br ^[[EXIT]]
+
+// Leaving the final suspend normally also runs the destroy region.
+// CHECK: ^[[FINAL_RESUME]]:
+// CHECK-NEXT: cir.br ^[[FINAL_DONE:bb[0-9]+]]
+// CHECK: ^[[FINAL_DONE]]:
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+// CHECK-NEXT: cir.store %[[TWO]], %[[DEST_SLOT]]
+// CHECK-NEXT: cir.br ^[[DESTROY]]
+
+// The destroy region is the cleanup. Every way into it continues to the exit
+// region.
+// CHECK: ^[[DESTROY]]:
+// CHECK-NEXT: cir.call @free_frame()
+// CHECK-NEXT: cir.br ^[[DISPATCH:bb[0-9]+]]
+// CHECK: ^[[DISPATCH]]:
+// CHECK-NEXT: %[[DEST:.*]] = cir.load %[[DEST_SLOT]]
+// CHECK-NEXT: cir.switch.flat %[[DEST]] : !s32i, ^[[UNREACHABLE:bb[0-9]+]] [
+// CHECK-NEXT: 0: ^[[FROM_INIT:bb[0-9]+]],
+// CHECK-NEXT: 1: ^[[FROM_FINAL:bb[0-9]+]],
+// CHECK-NEXT: 2: ^[[FROM_DONE:bb[0-9]+]]
+// CHECK-NEXT: ]
+// CHECK: ^[[FROM_INIT]]:
+// CHECK-NEXT: cir.br ^[[AFTER_CLEANUP:bb[0-9]+]]
+// CHECK: ^[[FROM_FINAL]]:
+// CHECK-NEXT: cir.br ^[[AFTER_CLEANUP]]
+// CHECK: ^[[FROM_DONE]]:
+// CHECK-NEXT: cir.br ^[[AFTER_CLEANUP]]
+// CHECK: ^[[UNREACHABLE]]:
+// CHECK-NEXT: cir.unreachable
+// CHECK: ^[[AFTER_CLEANUP]]:
+// CHECK-NEXT: cir.br ^[[EXIT]]
+
+// The exit region.
+// CHECK: ^[[EXIT]]:
+// CHECK-NEXT: cir.call @coro_end()
+// CHECK-NEXT: cir.return
+// CHECK: }
+
+ cir.func coroutine @co_return_in_if(%ready : !cir.bool, %cond : !cir.bool) {
+ cir.coroutine initialSuspend : {
+ cir.await(init, ready : {
+ cir.condition(%ready)
+ }, suspend : {
+ cir.coro.suspend_point
+ }, resume : {
+ cir.yield
+ },)
+ cir.yield
+ }, body : {
+ cir.await(user, ready : {
+ cir.condition(%ready)
+ }, suspend : {
+ cir.coro.suspend_point
+ }, resume : {
+ cir.yield
+ },)
+ cir.if %cond {
+ cir.call @return_void() nothrow : () -> ()
+ cir.co_return
+ }
+ cir.call @user_code() nothrow : () -> ()
+ cir.co_return
+ }, finalSuspend : {
+ cir.await(final, ready : {
+ cir.condition(%ready)
+ }, suspend : {
+ cir.coro.suspend_point
+ }, resume : {
+ cir.yield
+ },)
+ cir.yield
+ }, destroy : {
+ cir.call @free_frame() nothrow : () -> ()
+ cir.yield
+ }, exit : {
+ cir.return
+ }
+ cir.trap
+ }
+
+// Both cir.co_return ops, including the one nested in the cir.if, branch to
+// the same final suspend.
+
+// CHECK-LABEL: cir.func flattened_coroutine @co_return_in_if(%arg0: !cir.bool, %arg1: !cir.bool)
+// CHECK: %[[DEST_SLOT:.*]] = cir.alloca "__cleanup_dest_slot"
+
+// The user await. Its destroy case goes to the destroy region like the init
+// one.
+// CHECK: cir.coro.intrinsic.suspend
+// CHECK: cir.coro.intrinsic.suspend
+// CHECK-NEXT: cir.switch.flat %{{.*}} : !s8i, ^{{.*}} [
+// CHECK-NEXT: 0: ^[[USER_RESUME:bb[0-9]+]],
+// CHECK-NEXT: 1: ^[[USER_DESTROY:bb[0-9]+]]
+// CHECK-NEXT: ]
+// CHECK: ^[[USER_DESTROY]]:
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CHECK-NEXT: cir.store %[[ONE]], %[[DEST_SLOT]]
+// CHECK-NEXT: cir.br ^[[DESTROY:bb[0-9]+]]
+
+// CHECK: cir.brcond %arg1 ^[[THEN:bb[0-9]+]], ^[[ELSE:bb[0-9]+]]
+// CHECK: ^[[THEN]]:
+// CHECK-NEXT: cir.call @return_void()
+// CHECK-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CHECK: ^[[ELSE]]:
+// CHECK-NEXT: cir.call @user_code()
+// CHECK-NEXT: cir.br ^[[FINAL]]
+// CHECK: ^[[FINAL]]:
+// CHECK-NEXT: cir.br ^[[FINAL_READY:bb[0-9]+]]
+// CHECK: ^[[FINAL_READY]]:
+// CHECK-NEXT: cir.brcond %arg0 ^{{.*}}, ^[[FINAL_SUSPEND:bb[0-9]+]]
+// CHECK: ^[[FINAL_SUSPEND]]:
+// CHECK: %[[IS_FINAL:.*]] = cir.const #true
+// CHECK-NEXT: cir.coro.intrinsic.suspend(%{{.*}}, %[[IS_FINAL]])
+
+// CHECK: ^[[DESTROY]]:
+// CHECK-NEXT: cir.call @free_frame()
+// CHECK: cir.return
+// CHECK: }
+}
>From c48301a473aa7a365c153bee20bba40c39dd5063 Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Tue, 13 Oct 2026 23:01:13 -0500
Subject: [PATCH 05/10] Fix formatting
---
clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
index 4c9878a68e8c6e..0542d4266e05fb 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
@@ -566,7 +566,7 @@ CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) {
// Builds `exit`: coro.end(/*unwind*/ false) followed by the actual return
// to the caller.
-
+
assert(!cir::MissingFeatures::coroutineGroManager());
auto exitBuilder = [&]() {
cir::ConstantOp nullHandler =
>From fe67fcc98ecff3e763c5cd38d2525117a29559ff Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Wed, 14 Oct 2026 18:35:32 -0500
Subject: [PATCH 06/10] Remove obsolete coroutine suspend point documentation
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 4 ----
1 file changed, 4 deletions(-)
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 8c52638ca4cc91..12dce514b581a8 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -766,10 +766,6 @@ def CIR_AllocaOp : CIR_Op<"alloca", [
The `cleanup_dest_slot` attribute indicates that this was a temporary
alloca generated by the compiler to handle cleanup exit dispatching.
- The `coroutine_suspend_point` attribute indicates that this alloca is
- used to track the coroutine suspend destination. The use of this alloca
- identifies where control should continue when the coroutine suspends.
-
The result type is a pointer to the input's type.
Example:
>From d517e9cc7f1edbafee18ba6fabb5413ca552237b Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Wed, 14 Oct 2026 18:51:13 -0500
Subject: [PATCH 07/10] Address some review comments
---
clang/include/clang/CIR/Dialect/IR/CIROps.td | 6 +++---
clang/include/clang/CIR/MissingFeatures.h | 2 +-
clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 6 +++---
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 12dce514b581a8..ecdba7995d5e91 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -5291,9 +5291,9 @@ def CIR_CoroExitOp : CIR_Op<"coro.exit", [
]> {
let summary = "Branches to the enclosing cir.coroutine's exit region";
let description = [{
- Placeholder emitted while flattening `cir.await`.
- `cir.coroutine` flattening rewrites it into a `cir.br`
- to the `exit` block. It must not survive flattening.
+ Placeholder emitted while flattening `cir.await`. `cir.coroutine`flattening
+ rewrites it into a `cir.br` to the `exit` block.
+ It must not survive flattening.
}];
let assemblyFormat = "attr-dict";
let hasLLVMLowering = false;
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index e30e16679883b9..708ed80dd349f5 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -127,7 +127,7 @@ struct MissingFeatures {
// Coroutines
static bool coroAwaitFullExprCleanups() { return false; }
static bool coroOutsideFrameMD() { return false; }
- static bool coroutineGroManager() { return false; };
+ static bool coroutineGetRetObjManager() { return false; };
static bool coroutineUnwindRegion() { return false; };
// Various handling of deferred processing in CIRGenModule.
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index 12f0537ce0c38f..24cbf326b2002a 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -2126,7 +2126,7 @@ collectCoReturnsOps(mlir::Region ®ion,
}
// Flatten `cir.coroutine`. It runs once every nested op (awaits, scopes,
-// cleanups, ...) has been flattened. The initialSuspend, body and finalSuspend
+// cleanups, ...) has been flattened. This initialSuspend, body and finalSuspend
// regions become the body of a cleanup scope whose cleanup is the destroy
// region; the exit region follows the scope:
//
@@ -2248,7 +2248,7 @@ class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
for (auto &coroExit : coroExitsOps) {
// TODO(cir): once the GRO region is supported, this may need to branch
// there instead of to the exit region.
- assert(!cir::MissingFeatures::coroutineGroManager());
+ assert(!cir::MissingFeatures::coroutineGetRetObjManager());
rewriter.setInsertionPoint(coroExit);
rewriter.replaceOpWithNewOp<cir::BrOp>(coroExit, mlir::ValueRange{},
beforeExit);
@@ -2412,7 +2412,7 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
mlir::Block *retBlock = nullptr;
{
mlir::OpBuilder::InsertionGuard guard(rewriter);
- assert(!cir::MissingFeatures::coroutineGroManager());
+ assert(!cir::MissingFeatures::coroutineGetRetObjManager());
retBlock = rewriter.createBlock(remainingOpsBlock);
cir::CoroExitOp::create(rewriter, loc);
}
>From 7980657f05705e9c478b26cd461abe2afed215c5 Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Wed, 14 Oct 2026 18:55:48 -0500
Subject: [PATCH 08/10] Address some review comments
---
clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
index 0542d4266e05fb..bca22f8582d307 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp
@@ -567,7 +567,7 @@ CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) {
// Builds `exit`: coro.end(/*unwind*/ false) followed by the actual return
// to the caller.
- assert(!cir::MissingFeatures::coroutineGroManager());
+ assert(!cir::MissingFeatures::coroutineGetRetObjManager());
auto exitBuilder = [&]() {
cir::ConstantOp nullHandler =
builder.getNullPtr(builder.getVoidPtrTy(), openCurlyLoc);
>From 3b2004da9519891cb990f264ff12c40aa553bbce Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Wed, 14 Oct 2026 20:46:47 -0500
Subject: [PATCH 09/10] Address review comments
---
.../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 499 +++++++++++-------
1 file changed, 294 insertions(+), 205 deletions(-)
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index 24cbf326b2002a..47d4c8d82a1470 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -2125,27 +2125,27 @@ collectCoReturnsOps(mlir::Region ®ion,
[&](cir::CoReturnOp coReturnOp) { coReturnOps.push_back(coReturnOp); });
}
-// Flatten `cir.coroutine`. It runs once every nested op (awaits, scopes,
-// cleanups, ...) has been flattened. This initialSuspend, body and finalSuspend
-// regions become the body of a cleanup scope whose cleanup is the destroy
-// region; the exit region follows the scope:
+// Flatten `cir.coroutine`. This runs once every nested op (awaits, cir.scope,
+// cleanup scopes, ...) has been flattened. The initialSuspend, body and
+// finalSuspend regions become the body of a cir.cleanup.scope whose cleanup is
+// the destroy region; the exit region follows the cleanup scope:
//
// cir.cleanup.scope {
// initialSuspend ---> body ---> finalSuspend
// | ^
// +-------------+ cir.co_return
// } cleanup all {
-// destroy coro.free
+// destroy cir.coro.intrinsic.free
// }
-// exit coro.end + cir.return
+// exit cir.coro.intrinsic.end, cir.return
//
-// Leaving the scope body always runs the destroy region. In particular, the
-// cir.yield emitted for an await's destroy case leaves through it. A
-// suspended coroutine (cir.coro.exit) must keep its frame, so it branches
-// straight to the exit region.
+// Leaving the cleanup scope body always runs the destroy region. In
+// particular, the cir.yield emitted for an await's destroy case leaves through
+// it. A suspended coroutine (cir.coro.exit) must keep its frame, so it
+// branches straight to the exit region.
class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
void collectCoroExitsOps(
- cir::CoroutineOp &coroOp,
+ cir::CoroutineOp coroOp,
llvm::SmallVectorImpl<cir::CoroExitOp> &coroExitsOps) const {
// cir.coro.exit is produced by flattening a cir.await, which can only
// appear in the initialSuspend, body and finalSuspend regions.
@@ -2157,6 +2157,115 @@ class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
coroOp.getFinalSuspend().walk(collectCoroExit);
}
+ // Cleanup scope body: initialSuspend -> body -> finalSuspend. The three
+ // regions are moved, in this order, in front of the placeholder block the
+ // cleanup scope was created with, and the placeholder is erased once they
+ // are in place.
+ void inlineCleanUpScopeBody(mlir::PatternRewriter &rewriter,
+ cir::CoroutineOp coroOp,
+ cir::CleanupScopeOp scope) const {
+ mlir::Block *bodyPlaceholder = &scope.getBodyRegion().front();
+ // Take the entry blocks before the regions are moved.
+ mlir::Block *beforeBody = &coroOp.getBody().front();
+ mlir::Block *beforeFinal = &coroOp.getFinalSuspend().front();
+
+ inlineInitialSuspend(rewriter, coroOp, bodyPlaceholder, beforeBody);
+ inlineBody(rewriter, coroOp, bodyPlaceholder, beforeFinal);
+ inlineFinalSuspend(rewriter, coroOp, bodyPlaceholder);
+ rewriter.eraseBlock(bodyPlaceholder);
+ }
+
+ // initialSuspend ends in a cir.yield once the init await is done. Replace it
+ // with a branch to the body.
+ void inlineInitialSuspend(mlir::PatternRewriter &rewriter,
+ cir::CoroutineOp coroOp, mlir::Block *insertBefore,
+ mlir::Block *beforeBody) const {
+ mlir::Region &initialRegion = coroOp.getInitialSuspend();
+ auto initialYieldOp =
+ cast<cir::YieldOp>(initialRegion.back().getTerminator());
+ rewriter.setInsertionPoint(initialYieldOp);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(
+ initialYieldOp, initialYieldOp.getArgs(), beforeBody);
+ rewriter.inlineRegionBefore(initialRegion, insertBefore);
+ }
+
+ // The body either falls off the end (cir.yield) or ends in a
+ // cir.co_return. Both, like every other cir.co_return in the body, branch
+ // to the final suspend.
+ void inlineBody(mlir::PatternRewriter &rewriter, cir::CoroutineOp coroOp,
+ mlir::Block *insertBefore, mlir::Block *beforeFinal) const {
+ mlir::Region &bodyRegion = coroOp.getBody();
+
+ llvm::SmallVector<cir::CoReturnOp> coReturnsOps;
+ collectCoReturnsOps(bodyRegion, coReturnsOps);
+
+ mlir::Operation *bodyTerminator = bodyRegion.back().getTerminator();
+ if (auto bodyYield = dyn_cast<cir::YieldOp>(bodyTerminator)) {
+ rewriter.setInsertionPoint(bodyYield);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(bodyYield, mlir::ValueRange{},
+ beforeFinal);
+ }
+
+ for (auto &coReturn : coReturnsOps) {
+ rewriter.setInsertionPoint(coReturn);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(coReturn, mlir::ValueRange{},
+ beforeFinal);
+ }
+
+ rewriter.inlineRegionBefore(bodyRegion, insertBefore);
+ }
+
+ // finalSuspend keeps its cir.yield, which now ends the cleanup scope body:
+ // once the final await completes, the coroutine leaves the cleanup scope
+ // normally and runs the destroy region.
+ void inlineFinalSuspend(mlir::PatternRewriter &rewriter,
+ cir::CoroutineOp coroOp,
+ mlir::Block *insertBefore) const {
+ rewriter.inlineRegionBefore(coroOp.getFinalSuspend(), insertBefore);
+ }
+
+ // Cleanup scope cleanup region: the destroy region.
+ void inlineCleanUpScopeCleanup(mlir::PatternRewriter &rewriter,
+ cir::CoroutineOp coroOp,
+ cir::CleanupScopeOp scope) const {
+ mlir::Region &destroyRegion = coroOp.getDestroy();
+ mlir::Block *cleanupPlaceholder = &scope.getCleanupRegion().front();
+ rewriter.inlineRegionBefore(destroyRegion, cleanupPlaceholder);
+ rewriter.eraseBlock(cleanupPlaceholder);
+ }
+
+ // A suspended coroutine keeps its frame, so each cir.coro.exit branches
+ // straight to the exit region without running the destroy region.
+ void lowerCoroExits(mlir::PatternRewriter &rewriter,
+ llvm::ArrayRef<cir::CoroExitOp> coroExitsOps,
+ mlir::Block *beforeExit) const {
+ for (cir::CoroExitOp coroExit : coroExitsOps) {
+ // TODO(cir): once the GRO region is supported, this may need to branch
+ // there instead of to the exit region.
+ assert(!cir::MissingFeatures::coroutineGetRetObjManager());
+ rewriter.setInsertionPoint(coroExit);
+ rewriter.replaceOpWithNewOp<cir::BrOp>(coroExit, mlir::ValueRange{},
+ beforeExit);
+ }
+ }
+
+ // Exit region: inline it right after the cleanup scope, which falls through
+ // to it.
+ void inlineExitRegion(mlir::PatternRewriter &rewriter,
+ cir::CoroutineOp coroOp, cir::CleanupScopeOp scope,
+ mlir::Block *remainingOpsBlock) const {
+ mlir::Region &exitRegion = coroOp.getExit();
+ mlir::Block *beforeExit = &exitRegion.front();
+
+ rewriter.setInsertionPointAfter(scope);
+ mlir::Block *afterCleanup =
+ rewriter.splitBlock(remainingOpsBlock, rewriter.getInsertionPoint());
+ rewriter.inlineRegionBefore(exitRegion, afterCleanup);
+
+ rewriter.setInsertionPointAfter(scope);
+ cir::BrOp::create(rewriter, scope.getLoc(), mlir::ValueRange(), beforeExit);
+ }
+
public:
using OpRewritePattern<cir::CoroutineOp>::OpRewritePattern;
@@ -2186,6 +2295,14 @@ class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
// coroOp now lives in remainingOpsBlock. Build the cleanup scope in front
// of it.
+ //
+ // The coroutine becomes a cir.cleanup.scope so that its frame is freed
+ // (the destroy region, as its cleanup region) on every way out of the
+ // coroutine: completing the final suspend, an await's destroy case and,
+ // because the kind is All, an exception escaping the coroutine. The
+ // regular cleanup scope flattening then routes each of those exits
+ // through the destroy region. A suspended coroutine must keep its frame,
+ // so cir.coro.exit skips the cleanup scope (see lowerCoroExits).
rewriter.setInsertionPoint(coroOp);
auto emitYield = [](mlir::OpBuilder &b, mlir::Location l) {
@@ -2194,74 +2311,13 @@ class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
auto scope = cir::CleanupScopeOp::create(rewriter, loc, CleanupKind::All,
emitYield, emitYield);
- mlir::Block *bodyPlaceholder = &scope.getBodyRegion().front();
- // Scope body: initialSuspend -> body -> finalSuspend.
- mlir::Region &initialRegion = coroOp.getInitialSuspend();
- mlir::Block *afterInitial = &initialRegion.back();
- rewriter.inlineRegionBefore(initialRegion, bodyPlaceholder);
-
- mlir::Region &bodyRegion = coroOp.getBody();
- mlir::Block *beforeBody = &bodyRegion.front();
- mlir::Block *afterBody = &bodyRegion.back();
+ inlineCleanUpScopeBody(rewriter, coroOp, scope);
+ inlineCleanUpScopeCleanup(rewriter, coroOp, scope);
- rewriter.setInsertionPointToEnd(afterInitial);
- auto initialYieldOp = cast<cir::YieldOp>(afterInitial->getTerminator());
- rewriter.replaceOpWithNewOp<cir::BrOp>(
- initialYieldOp, initialYieldOp.getArgs(), beforeBody);
-
- llvm::SmallVector<cir::CoReturnOp> coReturnsOps;
- collectCoReturnsOps(bodyRegion, coReturnsOps);
-
- mlir::Region &finalRegion = coroOp.getFinalSuspend();
- mlir::Block *beforeFinal = &finalRegion.front();
-
- rewriter.inlineRegionBefore(bodyRegion, bodyPlaceholder);
- rewriter.setInsertionPointToEnd(afterBody);
-
- // The body either falls off the end (cir.yield) or ends in a
- // cir.co_return. Both, like every other cir.co_return in the body, branch
- // to the final suspend.
- mlir::Operation *bodyTerminator = afterBody->getTerminator();
- if (auto bodyYield = dyn_cast<cir::YieldOp>(bodyTerminator))
- rewriter.replaceOpWithNewOp<cir::BrOp>(bodyYield, mlir::ValueRange{},
- beforeFinal);
-
- for (auto &coReturn : coReturnsOps) {
- rewriter.setInsertionPoint(coReturn);
- rewriter.replaceOpWithNewOp<cir::BrOp>(coReturn, mlir::ValueRange{},
- beforeFinal);
- }
-
- rewriter.inlineRegionBefore(finalRegion, bodyPlaceholder);
- rewriter.eraseBlock(bodyPlaceholder);
-
- // Scope cleanup: the destroy region.
- mlir::Region &destroyRegion = coroOp.getDestroy();
- mlir::Block *cleanupPlaceholder = &scope.getCleanupRegion().front();
- rewriter.inlineRegionBefore(destroyRegion, cleanupPlaceholder);
- rewriter.eraseBlock(cleanupPlaceholder);
-
- mlir::Region &exitRegion = coroOp.getExit();
- mlir::Block *beforeExit = &exitRegion.front();
-
- for (auto &coroExit : coroExitsOps) {
- // TODO(cir): once the GRO region is supported, this may need to branch
- // there instead of to the exit region.
- assert(!cir::MissingFeatures::coroutineGetRetObjManager());
- rewriter.setInsertionPoint(coroExit);
- rewriter.replaceOpWithNewOp<cir::BrOp>(coroExit, mlir::ValueRange{},
- beforeExit);
- }
-
- // Exit region: inline it right after the scope, which falls through to it.
- rewriter.setInsertionPointAfter(scope);
- mlir::Block *afterCleanup =
- rewriter.splitBlock(remainingOpsBlock, rewriter.getInsertionPoint());
- rewriter.inlineRegionBefore(exitRegion, afterCleanup);
-
- rewriter.setInsertionPointAfter(scope);
- cir::BrOp::create(rewriter, loc, mlir::ValueRange(), beforeExit);
+ mlir::Block *beforeExit = &coroOp.getExit().front();
+ lowerCoroExits(rewriter, coroExitsOps, beforeExit);
+ inlineExitRegion(rewriter, coroOp, scope, remainingOpsBlock);
auto func = coroOp->getParentOfType<cir::FuncOp>();
// The coroutine regions have been flattened, so the original coroutine
@@ -2276,6 +2332,148 @@ class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
};
class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
+ // The ready region ends in a cir.condition that holds the result of
+ // `await_ready()`. Replace it with a cir.brcond: if the awaiter is already
+ // ready, skip the suspension and go straight to the resume region
+ // otherwise go to the suspend region.
+ void flattenReadyRegion(mlir::PatternRewriter &rewriter, cir::AwaitOp awaitOp,
+ mlir::Block *resumeEntry, mlir::Block *suspendEntry,
+ mlir::Block *remainingOpsBlock) const {
+ mlir::Region &readyRegion = awaitOp.getReady();
+ auto conditionOp =
+ cast<cir::ConditionOp>(readyRegion.back().getTerminator());
+ rewriter.setInsertionPoint(conditionOp);
+ rewriter.replaceOpWithNewOp<cir::BrCondOp>(
+ conditionOp, conditionOp.getCondition(), resumeEntry, suspendEntry);
+ rewriter.inlineRegionBefore(readyRegion, remainingOpsBlock);
+ }
+
+ // The suspend region runs `await_suspend()` and ends in a
+ // cir.coro.suspend_point. Emit cir.coro.intrinsic.save and
+ // cir.coro.intrinsic.suspend around it, and replace the suspend point with a
+ // cir.switch.flat on the suspend result, which decides whether the coroutine
+ // resumes, is destroyed or stays suspended.
+ void flattenSuspendRegion(mlir::PatternRewriter &rewriter,
+ cir::AwaitOp awaitOp, mlir::Block *resumeEntry,
+ mlir::Block *remainingOpsBlock) const {
+ mlir::Region &suspendRegion = awaitOp.getSuspend();
+ // TODO(cir): support an await_suspend() that returns bool (it can veto
+ // the suspension and resume right away) by flattening it into a brcond.
+ auto suspendPoint =
+ cast<cir::CoroSuspendPoint>(suspendRegion.back().getTerminator());
+ cir::CoroSuspendOp coroSuspendIntri =
+ emitCoroSuspend(rewriter, awaitOp, suspendPoint);
+
+ // Inline the suspend region before creating the destroy and suspended
+ // blocks, so they are placed right after it, in front of the continuation.
+ rewriter.inlineRegionBefore(suspendRegion, remainingOpsBlock);
+
+ mlir::Location loc = awaitOp.getLoc();
+ mlir::Block *cleanupBlock =
+ createDestroyBlock(rewriter, loc, remainingOpsBlock);
+ mlir::Block *retBlock =
+ createSuspendedBlock(rewriter, loc, remainingOpsBlock);
+
+ // 0 resumes, 1 destroys and the default (-1) stays suspended.
+ llvm::SmallVector<mlir::APInt> caseValues{mlir::APInt(8, 0),
+ mlir::APInt(8, 1)};
+ llvm::SmallVector<mlir::Block *> caseDestinations{resumeEntry,
+ cleanupBlock};
+ llvm::SmallVector<mlir::ValueRange> caseOperands{caseValues.size()};
+
+ // Dispatch to the appropriate coroutine path based on the result of
+ // `cir.coro.intrinsic.suspend`: resume, destroy, or suspend.
+ rewriter.setInsertionPoint(suspendPoint);
+ rewriter.replaceOpWithNewOp<cir::SwitchFlatOp>(
+ suspendPoint, coroSuspendIntri.getResult(), retBlock,
+ mlir::ValueRange{}, caseValues, caseDestinations, caseOperands);
+ }
+
+ // Emit cir.coro.intrinsic.save at the start of the suspend region and
+ // cir.coro.intrinsic.suspend right before its cir.coro.suspend_point.
+ cir::CoroSuspendOp emitCoroSuspend(mlir::PatternRewriter &rewriter,
+ cir::AwaitOp awaitOp,
+ cir::CoroSuspendPoint suspendPoint) const {
+ mlir::Location loc = awaitOp.getLoc();
+
+ // cir.coro.intrinsic.suspend takes the token produced by
+ // cir.coro.intrinsic.save as its first argument, so emit the save first.
+ rewriter.setInsertionPointToStart(&awaitOp.getSuspend().front());
+
+ // The save goes first in the suspend region, before await_suspend()
+ // runs, so the coroutine is already considered suspended (and can be
+ // resumed) from that point.
+ auto voidPtrTy = cir::PointerType::get(cir::VoidType::get(getContext()));
+ auto nullPtr = cir::ConstantOp::create(
+ rewriter, loc,
+ cir::ConstPtrAttr::get(voidPtrTy, rewriter.getI64IntegerAttr(0)));
+ auto coroSave = cir::CoroSaveOp::create(rewriter, loc, nullPtr);
+ rewriter.setInsertionPoint(suspendPoint);
+
+ // The second argument to `cir.coro.intrinsic.suspend` indicates whether
+ // this is the final suspend point. Coroutines suspended at a final suspend
+ // point are considered done (`cir.coro.intrinsic.done` returns true) and
+ // may only be destroyed; resuming them is undefined behavior.
+ bool isFinalSuspend = awaitOp.getKind() == cir::AwaitKind::Final;
+ auto isFinalCoroSuspend = cir::ConstantOp::create(
+ rewriter, loc, cir::BoolAttr::get(getContext(), isFinalSuspend));
+
+ // cir.coro.intrinsic.suspend returns:
+ // -1 : coroutine suspended
+ // 0 : coroutine resumed
+ // 1 : coroutine destroyed
+ return cir::CoroSuspendOp::create(
+ rewriter, loc,
+ mlir::ValueRange{coroSave.getResult(), isFinalCoroSuspend});
+ }
+
+ // Destroy case. Classic codegen uses EmitBranchThroughCleanup() so every
+ // enclosing cleanup runs before the frame is freed. Here the destroy block
+ // is a single cir.yield, meant to leave through the coroutine cleanup scope
+ // (cir.coro.intrinsic.free) and then reach the exit region.
+ //
+ // FIXME: This only works when the await is directly in the coroutine
+ // body. Any enclosing cir.cleanup.scope or loop treats the cir.yield as
+ // a normal fallthrough, so after running its cleanups the destroy path
+ // continues into the rest of the body (or the next loop iteration)
+ // instead of reaching cir.coro.intrinsic.free. This needs a dedicated
+ // terminator that createExitTerminator propagates through cleanup scopes
+ // (like cir.co_return) and that CIRCoroutineFlattening rewrites into a
+ // branch to the coroutine cleanup. See silly_task_with_dtor in
+ // clang/test/CIR/CodeGenCoroutines/coro-task.cpp.
+ mlir::Block *createDestroyBlock(mlir::PatternRewriter &rewriter,
+ mlir::Location loc,
+ mlir::Block *insertBefore) const {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ mlir::Block *cleanupBlock = rewriter.createBlock(insertBefore);
+ cir::YieldOp::create(rewriter, loc);
+ return cleanupBlock;
+ }
+
+ // Suspended case: cir.coro.exit, which CIRCoroutineFlattening turns into
+ // a branch to the exit region.
+ mlir::Block *createSuspendedBlock(mlir::PatternRewriter &rewriter,
+ mlir::Location loc,
+ mlir::Block *insertBefore) const {
+ mlir::OpBuilder::InsertionGuard guard(rewriter);
+ assert(!cir::MissingFeatures::coroutineGetRetObjManager());
+ mlir::Block *retBlock = rewriter.createBlock(insertBefore);
+ cir::CoroExitOp::create(rewriter, loc);
+ return retBlock;
+ }
+
+ void flattenResumeRegion(mlir::PatternRewriter &rewriter,
+ cir::AwaitOp awaitOp,
+ mlir::Block *remainingOpsBlock) const {
+ mlir::Region &resumeRegion = awaitOp.getResume();
+ auto resumeYield = cast<cir::YieldOp>(resumeRegion.back().getTerminator());
+ rewriter.setInsertionPoint(resumeYield);
+ // Once the coroutine resumes, continue with the operations that
+ // originally followed the await.
+ rewriter.replaceOpWithNewOp<cir::BrOp>(resumeYield, remainingOpsBlock);
+ rewriter.inlineRegionBefore(resumeRegion, remainingOpsBlock);
+ }
+
public:
using OpRewritePattern<cir::AwaitOp>::OpRewritePattern;
@@ -2287,8 +2485,8 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
// ready await_ready()
// true / \ false
// | v
- // | suspend coro.save, await_suspend(),
- // | / | \ coro.suspend
+ // | suspend await_suspend(), with
+ // | / | \ cir.coro.intrinsic.save/suspend
// | 0/ 1| \-1 (default)
// v v v v
// resume destroy suspended
@@ -2297,12 +2495,14 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
// continuation ops after the await
//
// `destroy` is meant to leave through the enclosing cleanups to the
- // coroutine's destroy region (see the FIXME below), and `suspended` goes
- // straight to its exit region (see CIRCoroutineFlattening).
+ // coroutine's destroy region (see the FIXME on createDestroyBlock), and
+ // `suspended` goes straight to its exit region (see CIRCoroutineFlattening).
mlir::LogicalResult
matchAndRewrite(cir::AwaitOp awaitOp,
mlir::PatternRewriter &rewriter) const override {
-
+ // Nested ops (cir.if, cir.scope, cir.try, cleanup scopes, ...) must be
+ // flattened first. Fail here so the rewriter processes them before
+ // flattening this op.
for (mlir::Region ®ion : awaitOp->getRegions())
if (hasNestedOpsToFlatten(region))
return mlir::failure();
@@ -2313,134 +2513,23 @@ class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
mlir::Block *remainingOpsBlock =
rewriter.splitBlock(awaitBlock, rewriter.getInsertionPoint());
- mlir::Location loc = awaitOp.getLoc();
+ // Take the entry blocks before the regions are inlined.
+ mlir::Block *readyEntry = &awaitOp.getReady().front();
+ mlir::Block *suspendEntry = &awaitOp.getSuspend().front();
+ mlir::Block *resumeEntry = &awaitOp.getResume().front();
- mlir::Region &readyRegion = awaitOp.getReady();
- mlir::Block &beforeReady = awaitOp.getReady().front();
- mlir::Region &suspendRegion = awaitOp.getSuspend();
- mlir::Region &resumeRegion = awaitOp.getResume();
- auto conditionOp =
- cast<cir::ConditionOp>(readyRegion.back().getTerminator());
- {
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- rewriter.setInsertionPoint(conditionOp);
- // The condition is the result of `await_ready()`. If true, execution
- // continues in the resume region otherwise, control transfers to the
- // suspend region.
- rewriter.replaceOpWithNewOp<cir::BrCondOp>(
- conditionOp, conditionOp.getCondition(), &resumeRegion.front(),
- &suspendRegion.front());
- }
- rewriter.inlineRegionBefore(readyRegion, remainingOpsBlock);
+ flattenReadyRegion(rewriter, awaitOp, resumeEntry, suspendEntry,
+ remainingOpsBlock);
- {
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- rewriter.setInsertionPointToEnd(awaitBlock);
- // After inlining the ready region, branch from the original await block
- // to the beginning of the inlined ready-region
- cir::BrOp::create(rewriter, loc, mlir::ValueRange(), &beforeReady);
- }
+ // After inlining the ready region, branch from the original await block
+ // to the beginning of the inlined ready region.
+ rewriter.setInsertionPointToEnd(awaitBlock);
+ cir::BrOp::create(rewriter, awaitOp.getLoc(), readyEntry);
- // TODO(cir): support an await_suspend() that returns bool (it can veto
- // the suspension and resume right away) by flattening it into a brcond.
- auto suspendPoint =
- cast<cir::CoroSuspendPoint>(suspendRegion.back().getTerminator());
- cir::CoroSuspendOp coroSuspendIntri = nullptr;
- {
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- rewriter.setInsertionPoint(&suspendRegion.front().front());
-
- // coro.save goes first in the suspend region, before await_suspend()
- // runs, so the coroutine is already considered suspended (and can be
- // resumed) from that point.
- auto voidPtrTy = cir::PointerType::get(cir::VoidType::get(getContext()));
- auto nullPtr = cir::ConstantOp::create(
- rewriter, loc,
- cir::ConstPtrAttr::get(voidPtrTy, rewriter.getI64IntegerAttr(0)));
- auto coroSave = cir::CoroSaveOp::create(rewriter, loc, nullPtr);
- rewriter.setInsertionPoint(suspendPoint);
-
- // The second argument to `llvm.coro.suspend` indicates whether this is
- // the final suspend point. Coroutines suspended at a final suspend point
- // are considered done (`llvm.coro.done` returns true) and may only be
- // destroyed; resuming them is undefined behavior.
- bool isFinalSuspend = awaitOp.getKind() == cir::AwaitKind::Final;
- auto isFinalCoroSuspend = cir::ConstantOp::create(
- rewriter, loc, cir::BoolAttr::get(getContext(), isFinalSuspend));
-
- // llvm.coro.suspend returns:
- // -1 : coroutine suspended
- // 0 : coroutine resumed
- // 1 : coroutine destroyed
- coroSuspendIntri = cir::CoroSuspendOp::create(
- rewriter, loc,
- mlir::ValueRange{coroSave.getResult(), isFinalCoroSuspend});
- }
- rewriter.inlineRegionBefore(suspendRegion, remainingOpsBlock);
-
- {
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- rewriter.setInsertionPoint(suspendPoint);
- llvm::SmallVector<mlir::APInt> caseValues{mlir::APInt(8, 0),
- mlir::APInt(8, 1)};
-
- llvm::SmallVector<mlir::ValueRange> caseOperands{caseValues.size()};
-
- // Destroy case. Classic codegen uses EmitBranchThroughCleanup() so every
- // enclosing cleanup runs before coro.free. Here the destroy block is a
- // single cir.yield, meant to leave through the coroutine cleanup scope
- // (coro.free) and then reach the exit region.
- //
- // FIXME: This only works when the await is directly in the coroutine
- // body. Any enclosing cir.cleanup.scope or loop treats the cir.yield as
- // a normal fallthrough, so after running its cleanups the destroy path
- // continues into the rest of the body (or the next loop iteration)
- // instead of reaching coro.free. This needs a dedicated terminator that
- // createExitTerminator propagates through cleanup scopes (like
- // cir.co_return) and that CIRCoroutineFlattening rewrites into a branch
- // to the coroutine cleanup. See silly_task_with_dtor in
- // clang/test/CIR/CodeGenCoroutines/coro-task.cpp.
- mlir::Block *cleanupBlock = nullptr;
- {
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- cleanupBlock = rewriter.createBlock(remainingOpsBlock);
- cir::YieldOp::create(rewriter, loc);
- }
-
- // Suspended case: cir.coro.exit, which CIRCoroutineFlattening turns into
- // a branch to the exit region.
- mlir::Block *retBlock = nullptr;
- {
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- assert(!cir::MissingFeatures::coroutineGetRetObjManager());
- retBlock = rewriter.createBlock(remainingOpsBlock);
- cir::CoroExitOp::create(rewriter, loc);
- }
- // 0 resumes, 1 destroys and the default (-1) stays suspended.
- llvm::SmallVector<mlir::Block *> caseDestinations{&resumeRegion.front(),
- cleanupBlock};
-
- // Dispatch to the appropriate coroutine path based on the result of
- // `llvm.coro.suspend`: resume, destroy, or suspend.
- auto coroSuspendSwitch = cir::SwitchFlatOp::create(
- rewriter, loc, coroSuspendIntri.getResult(), retBlock,
- mlir::ValueRange{}, caseValues, caseDestinations, caseOperands);
-
- rewriter.replaceOp(suspendPoint, coroSuspendSwitch);
- }
-
- auto resumeYield = cast<cir::YieldOp>(resumeRegion.back().getTerminator());
- {
- mlir::OpBuilder::InsertionGuard guard(rewriter);
- rewriter.setInsertionPoint(resumeYield);
- // Once the coroutine resumes, continue with the operations that
- // originally followed the await.
- rewriter.replaceOpWithNewOp<cir::BrOp>(resumeYield, remainingOpsBlock);
- }
- rewriter.inlineRegionBefore(resumeRegion, remainingOpsBlock);
+ flattenSuspendRegion(rewriter, awaitOp, resumeEntry, remainingOpsBlock);
+ flattenResumeRegion(rewriter, awaitOp, remainingOpsBlock);
rewriter.eraseOp(awaitOp);
-
return mlir::success();
}
};
>From 3aa1b1b696047e95872b7b25c3797dfcb2f61491 Mon Sep 17 00:00:00 2001
From: Andres Salamanca <andrealebarbaritos at gmail.com>
Date: Wed, 14 Oct 2026 23:28:18 -0500
Subject: [PATCH 10/10] Fix tests after #231046
---
.../CodeGenCoroutines/coro-fallthrough.cpp | 130 ++++++++++++------
.../test/CIR/CodeGenCoroutines/coro-task.cpp | 31 +++--
2 files changed, 113 insertions(+), 48 deletions(-)
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-fallthrough.cpp b/clang/test/CIR/CodeGenCoroutines/coro-fallthrough.cpp
index d912d5ba7351b1..157334ef2b2759 100644
--- a/clang/test/CIR/CodeGenCoroutines/coro-fallthrough.cpp
+++ b/clang/test/CIR/CodeGenCoroutines/coro-fallthrough.cpp
@@ -234,10 +234,13 @@ IntTask co_return_value(int v) {
// CIR: }, finalSuspend : {
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z15co_return_valuei(
-// CIR-FLAT: cir.call @_ZN7IntTask12promise_type12return_valueEi(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN7IntTask12promise_type12return_valueEi({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.call @dead()
+// CIR-FLAT-NEXT: cir.try_call @dead()
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN7IntTask12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z15co_return_valuei(
// OGCG: {{call|invoke}} void @_ZN7IntTask12promise_type12return_valueEi(
@@ -271,12 +274,15 @@ Task co_return_void() {
// CIR: }, finalSuspend : {
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z14co_return_voidv(
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT: cir.call @dead()
+// CIR-FLAT: cir.try_call @dead()
+// CIR-FLAT: cir.call @_ZN1SD1Ev(
// CIR-FLAT-NOT: @_ZN4Task12promise_type11return_voidEv
-// CIR-FLAT: }, finalSuspend : {
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z14co_return_voidv(
// OGCG: {{call|invoke}} void @_ZN4Task12promise_type11return_voidEv(
@@ -302,9 +308,11 @@ Task co_return_at_end_without_cleanup() {
// CIR-EH-NEXT: } catch all
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z32co_return_at_end_without_cleanupv(
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
-// CIR-FLAT-NEXT: }, finalSuspend : {
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT-NEXT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z32co_return_at_end_without_cleanupv(
// OGCG: await.ready:
@@ -336,10 +344,23 @@ Task co_return_in_block() {
// CIR-NEXT: }
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z18co_return_in_blockv(
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST]], %{{.*}}
+// CIR-FLAT-NEXT: cir.br ^[[DTOR:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.call @dead()
+// CIR-FLAT-NEXT: cir.try_call @dead()
+// CIR-FLAT: ^[[DTOR]]:
+// CIR-FLAT-NEXT: cir.call @_ZN1SD1Ev(
+// CIR-FLAT: cir.switch.flat %{{.*}} : !s32i, ^{{.*}} [
+// CIR-FLAT-NEXT: 0: ^{{.*}},
+// CIR-FLAT-NEXT: 1: ^[[FROM_RET:bb[0-9]+]]
+// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[FROM_RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z18co_return_in_blockv(
// OGCG: {{call|invoke}} void @_ZN4Task12promise_type11return_voidEv(
@@ -372,14 +393,18 @@ Task co_return_in_if_with_dead_code(bool b) {
// CIR-NEXT: }
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z30co_return_in_if_with_dead_codeb(
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET1:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET1]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.call @dead()
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT-NEXT: cir.try_call @dead()
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET2:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET2]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.call @dead()
+// CIR-FLAT-NEXT: cir.try_call @dead()
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z30co_return_in_if_with_dead_codeb(
// OGCG: if.then:
@@ -409,10 +434,13 @@ Task co_return_in_loop(bool b) {
// CIR-NEXT: }
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z17co_return_in_loopb(
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.call @dead()
+// CIR-FLAT-NEXT: cir.try_call @dead()
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z17co_return_in_loopb(
// OGCG: while.body:
@@ -443,10 +471,13 @@ Task co_return_in_switch(int i) {
// CIR-NEXT: }
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z19co_return_in_switchi(
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.call @dead()
+// CIR-FLAT-NEXT: cir.try_call @dead()
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z19co_return_in_switchi(
// OGCG: sw.bb:
@@ -474,10 +505,15 @@ Task dead_co_await() {
// CIR: }, finalSuspend : {
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z13dead_co_awaitv(
-// CIR-FLAT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.await(user, ready : {
+// CIR-FLAT-NEXT: cir.br ^[[DEAD_AWAIT:bb[0-9]+]]
+// CIR-FLAT-NEXT: ^[[DEAD_AWAIT]]:
+// CIR-FLAT-NEXT: cir.call @_ZNSt13suspend_never11await_readyEv(
+// CIR-FLAT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z13dead_co_awaitv(
// OGCG-NOT: suspend_never
@@ -512,13 +548,18 @@ Task label_after_co_return(bool b) {
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z21label_after_co_returnb(
// CIR-FLAT: cir.goto "l"
-// CIR-FLAT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET1:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET1]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
// CIR-FLAT-NEXT: cir.label "l"
-// CIR-FLAT-NEXT: cir.call @live()
-// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type11return_voidEv(
-// CIR-FLAT-NEXT: cir.co_return
-// CIR-FLAT-NEXT: }, finalSuspend : {
+// CIR-FLAT-NEXT: cir.try_call @live() ^[[LIVE:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[LIVE]]:
+// CIR-FLAT-NEXT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET2:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET2]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
+// CIR-FLAT-NEXT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z21label_after_co_returnb(
// OGCG: {{^}}l:
@@ -625,8 +666,13 @@ IntTask goto_at_end() {
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z11goto_at_endv(
// CIR-FLAT: cir.goto "l"
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.yield
-// CIR-FLAT-NEXT: }, finalSuspend : {
+// CIR-FLAT-NEXT: cir.br ^[[FALLTHROUGH:bb[0-9]+]]
+// CIR-FLAT-NEXT: ^[[FALLTHROUGH]]:
+// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.const #cir.int<2> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST]], %{{.*}}
+// CIR-FLAT-NEXT: cir.br ^[[CORO_CLEANUP:bb[0-9]+]]
+// CIR-FLAT-NEXT: ^[[CORO_CLEANUP]]:
+// CIR-FLAT-NEXT: {{.*}} = cir.coro.intrinsic.free
// OGCG-LABEL: define {{.*}} @_Z11goto_at_endv(
// OGCG: {{^}}l:
@@ -654,12 +700,15 @@ Task goto_after_co_return() {
// CIR-EH-NEXT: } catch all
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z20goto_after_co_returnv(
-// CIR-FLAT: cir.co_return
+// CIR-FLAT: cir.try_call @_ZN4Task12promise_type11return_voidEv({{.*}}) ^[[RET:bb[0-9]+]],
+// CIR-FLAT-NEXT: ^[[RET]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
// CIR-FLAT-NEXT: cir.goto "l"
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.yield
-// CIR-FLAT-NEXT: }, finalSuspend : {
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
+// CIR-FLAT-NEXT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN4Task12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z20goto_after_co_returnv(
// OGCG: await.ready:
@@ -690,8 +739,9 @@ IntTask unreachable_after_switch(int i) {
// CIR-FLAT-LABEL: cir.func {{.*}} @_Z24unreachable_after_switchi(
// CIR-FLAT: cir.unreachable
// CIR-FLAT-NEXT: ^bb{{[0-9]+}}: // no predecessors
-// CIR-FLAT-NEXT: cir.yield
-// CIR-FLAT-NEXT: }, finalSuspend : {
+// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
+// CIR-FLAT-NEXT: ^[[FINAL]]:
+// CIR-FLAT-NEXT: cir.call @_ZN7IntTask12promise_type13final_suspendEv(
// OGCG-LABEL: define {{.*}} @_Z24unreachable_after_switchi(
// OGCG: sw.bb:
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-task.cpp b/clang/test/CIR/CodeGenCoroutines/coro-task.cpp
index b5b97fd5af8d27..12f71afc5cf404 100644
--- a/clang/test/CIR/CodeGenCoroutines/coro-task.cpp
+++ b/clang/test/CIR/CodeGenCoroutines/coro-task.cpp
@@ -1308,31 +1308,46 @@ folly::coro::Task<int> co_return_with_dtor(int flag) {
// CIR-FLAT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[ONE]]) ^[[RET1:bb[0-9]+]],
// CIR-FLAT: ^[[RET1]]:
-// CIR-FLAT: %[[DEST_0:.*]] = cir.const #cir.int<0> : !s32i
-// CIR-FLAT: cir.store %[[DEST_0]], %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[DTOR:bb[0-9]+]]
+// CIR-FLAT-NEXT: %[[DEST_RET1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_RET1]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR:bb[0-9]+]]
// CIR-FLAT: ^[[IF_END]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_IF:bb[0-9]+]]
+// CIR-FLAT: ^[[AFTER_IF]]:
// CIR-FLAT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
// CIR-FLAT: cir.try_call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[TWO]]) ^[[RET2:bb[0-9]+]],
// CIR-FLAT: ^[[RET2]]:
-// CIR-FLAT: %[[DEST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR-FLAT: cir.store %[[DEST_1]], %[[CLEANUP_DEST_SLOT]]
-// CIR-FLAT: cir.br ^[[DTOR]]
+// CIR-FLAT-NEXT: %[[DEST_RET2:.*]] = cir.const #cir.int<2> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_RET2]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR]]
+
+// The normal fallthrough out of the cleanup scope gets slot 0. It is dead
+// here because both paths end in a cir.co_return.
+// CIR-FLAT: ^[[FALLTHROUGH:bb[0-9]+]]: // no predecessors
+// CIR-FLAT-NEXT: %[[DEST_FALLTHROUGH:.*]] = cir.const #cir.int<0> : !s32i
+// CIR-FLAT-NEXT: cir.store %[[DEST_FALLTHROUGH]], %[[CLEANUP_DEST_SLOT]]
+// CIR-FLAT-NEXT: cir.br ^[[DTOR]]
+
// CIR-FLAT: ^[[DTOR]]:
// CIR-FLAT-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[LOCAL]])
// CIR-FLAT-NEXT: cir.br ^[[DTOR_EXIT:bb[0-9]+]]
// CIR-FLAT: ^[[DTOR_EXIT]]:
// CIR-FLAT-NEXT: %[[DEST:.*]] = cir.load %[[CLEANUP_DEST_SLOT]]
// CIR-FLAT-NEXT: cir.switch.flat %[[DEST]] : !s32i, ^[[DEFAULT:bb[0-9]+]] [
-// CIR-FLAT-NEXT: 0: ^[[FROM_RET1:bb[0-9]+]],
-// CIR-FLAT-NEXT: 1: ^[[FROM_RET2:bb[0-9]+]]
+// CIR-FLAT-NEXT: 0: ^[[FROM_FALLTHROUGH:bb[0-9]+]],
+// CIR-FLAT-NEXT: 1: ^[[FROM_RET1:bb[0-9]+]],
+// CIR-FLAT-NEXT: 2: ^[[FROM_RET2:bb[0-9]+]]
// CIR-FLAT-NEXT: ]
+// CIR-FLAT: ^[[FROM_FALLTHROUGH]]:
+// CIR-FLAT-NEXT: cir.br ^[[AFTER_CLEANUP:bb[0-9]+]]
// CIR-FLAT: ^[[FROM_RET1]]:
// CIR-FLAT-NEXT: cir.br ^[[FINAL:bb[0-9]+]]
// CIR-FLAT: ^[[FROM_RET2]]:
// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
// CIR-FLAT: ^[[DEFAULT]]:
// CIR-FLAT-NEXT: cir.unreachable
+// CIR-FLAT: ^[[AFTER_CLEANUP]]:
+// CIR-FLAT-NEXT: cir.br ^[[FINAL]]
// CIR-FLAT: ^[[FINAL]]:
// CIR-FLAT-NEXT: cir.call @_ZN5folly4coro4TaskIiE12promise_type13final_suspendEv(%[[PROMISE]])
// CIR-FLAT: cir.coro.intrinsic.free
More information about the cfe-commits
mailing list