[llvm] [Coroutines] Model retcon returns explicitly and upgrade legacy IR (PR #227232)
Karim Alweheshy via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 09:55:40 PDT 2026
https://github.com/karim-alweheshy updated https://github.com/llvm/llvm-project/pull/227232
>From b1888bd922d28c32e33a8d6aabb15c62a10310cb Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Tue, 29 Sep 2026 11:06:43 +0200
Subject: [PATCH 1/8] [SCCP] Do not track returns from pre-split coroutines
---
llvm/lib/Analysis/ValueLatticeUtils.cpp | 8 +-
.../SCCP/ipsccp-presplit-coroutine.ll | 82 +++++++++++++++++++
2 files changed, 89 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/SCCP/ipsccp-presplit-coroutine.ll
diff --git a/llvm/lib/Analysis/ValueLatticeUtils.cpp b/llvm/lib/Analysis/ValueLatticeUtils.cpp
index acc19667d77649..ee24c987f6ff73 100644
--- a/llvm/lib/Analysis/ValueLatticeUtils.cpp
+++ b/llvm/lib/Analysis/ValueLatticeUtils.cpp
@@ -22,7 +22,13 @@ bool llvm::canTrackArgumentsInterprocedurally(Function *F) {
}
bool llvm::canTrackReturnsInterprocedurally(Function *F) {
- return F->hasExactDefinition() && !F->hasFnAttribute(Attribute::Naked);
+ // Before coroutine splitting, the returns from a coroutine may not be
+ // represented by ret instructions. In particular, returned-continuation
+ // coroutines end in coro.end followed by unreachable, but splitting later
+ // creates a concrete return value. Do not infer undef from the absence of
+ // a ret in the pre-split body.
+ return F->hasExactDefinition() && !F->hasFnAttribute(Attribute::Naked) &&
+ !F->hasFnAttribute(Attribute::PresplitCoroutine);
}
bool llvm::canTrackGlobalVariableInterprocedurally(GlobalVariable *GV) {
diff --git a/llvm/test/Transforms/SCCP/ipsccp-presplit-coroutine.ll b/llvm/test/Transforms/SCCP/ipsccp-presplit-coroutine.ll
new file mode 100644
index 00000000000000..8836cd5493f9c5
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-presplit-coroutine.ll
@@ -0,0 +1,82 @@
+; RUN: opt -passes=ipsccp -S %s | FileCheck %s
+; RUN: opt -passes='coro-early,ipsccp,cgscc(coro-split),verify' -S %s | FileCheck %s --check-prefix=SPLIT
+
+; A returned-continuation coroutine has no ret before splitting. Its
+; coro.end is replaced with a real pair of continuation and yield pointers
+; during splitting. IPSCCP must not replace the direct call's result with
+; undef just because the pre-split body ends in unreachable.
+
+declare token @llvm.coro.id.retcon.once(i32, i32, ptr, ptr, ptr, ptr)
+declare ptr @llvm.coro.begin(token, ptr)
+declare i1 @llvm.coro.suspend.retcon.i1(...)
+declare void @llvm.coro.end(ptr, i1, token)
+declare ptr @llvm.coro.prepare.retcon(ptr)
+declare ptr @malloc(i64)
+declare void @free(ptr)
+declare void @resume(ptr, i1)
+declare void @consume(ptr)
+
+define internal swiftcc { ptr, ptr } @accessor(ptr noalias %buffer,
+ ptr swiftself %object) #0 {
+entry:
+ %id = call token @llvm.coro.id.retcon.once(
+ i32 32, i32 8, ptr %buffer, ptr @resume, ptr @malloc, ptr @free)
+ %frame = call ptr @llvm.coro.begin(token %id, ptr null)
+ %field = getelementptr i8, ptr %object, i64 8
+ %suspended = call i1 (...) @llvm.coro.suspend.retcon.i1(ptr %field)
+ call void @llvm.coro.end(ptr %frame, i1 false, token none)
+ unreachable
+}
+
+define void @caller(ptr %target, ptr %buffer, ptr %object) {
+entry:
+ %prepared = call ptr @llvm.coro.prepare.retcon(ptr %target)
+ %is_direct = icmp eq ptr %prepared, @accessor
+ br i1 %is_direct, label %direct, label %indirect
+
+direct:
+ %direct_pair = call swiftcc { ptr, ptr } @accessor(
+ ptr noalias %buffer, ptr swiftself %object)
+ br label %join
+
+indirect:
+ %indirect_pair = call swiftcc { ptr, ptr } %prepared(
+ ptr noalias %buffer, ptr swiftself %object)
+ br label %join
+
+join:
+ %pair = phi { ptr, ptr } [ %direct_pair, %direct ],
+ [ %indirect_pair, %indirect ]
+ %continuation = extractvalue { ptr, ptr } %pair, 0
+ %field = extractvalue { ptr, ptr } %pair, 1
+ call void @consume(ptr %field)
+ call swiftcc void %continuation(ptr %buffer, i1 false)
+ ret void
+}
+
+; CHECK-LABEL: define void @caller(
+; CHECK: direct:
+; CHECK: %direct_pair = call swiftcc { ptr, ptr } @accessor(
+; CHECK: join:
+; CHECK: %pair = phi { ptr, ptr } [ %direct_pair, %direct ], [ %indirect_pair, %indirect ]
+
+; SPLIT-LABEL: define internal swiftcc { ptr, ptr } @accessor(
+; SPLIT: ret { ptr, ptr }
+; SPLIT-LABEL: define void @caller(
+; SPLIT: %pair = phi { ptr, ptr } [ %direct_pair, %direct ], [ %indirect_pair, %indirect ]
+
+define internal i32 @ordinary() {
+entry:
+ ret i32 7
+}
+
+define i32 @ordinary_caller() {
+entry:
+ %value = call i32 @ordinary()
+ ret i32 %value
+}
+
+; CHECK-LABEL: define i32 @ordinary_caller()
+; CHECK: ret i32 7
+
+attributes #0 = { noinline presplitcoroutine }
>From f7f4035f2de25461a39a2a993ac04d64ad0bd30c Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Wed, 30 Sep 2026 14:28:13 +0200
Subject: [PATCH 2/8] Split retcon coroutines before ThinLTO interprocedural
passes
---
.../llvm/Transforms/Coroutines/CoroSplit.h | 5 +++
llvm/lib/Analysis/ValueLatticeUtils.cpp | 8 +----
llvm/lib/Passes/PassBuilderPipelines.cpp | 9 +++++
llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 12 ++++++-
.../retcon-early-split-ipsccp.ll} | 33 +++----------------
5 files changed, 31 insertions(+), 36 deletions(-)
rename llvm/test/Transforms/{SCCP/ipsccp-presplit-coroutine.ll => Coroutines/retcon-early-split-ipsccp.ll} (66%)
diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
index 3c589370a04fc1..c4e30cf7593552 100644
--- a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
+++ b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
@@ -29,10 +29,13 @@ struct Shape;
} // namespace coro
struct CoroSplitPass : RequiredPassInfoMixin<CoroSplitPass> {
+ enum class Mode { All, RetconOnly };
+
using BaseABITy =
std::function<std::unique_ptr<coro::BaseABI>(Function &, coro::Shape &)>;
LLVM_ABI CoroSplitPass(bool OptimizeFrame = false);
+ LLVM_ABI CoroSplitPass(bool OptimizeFrame, Mode SplitMode);
LLVM_ABI CoroSplitPass(SmallVector<BaseABITy> GenCustomABIs,
bool OptimizeFrame = false);
@@ -55,6 +58,8 @@ struct CoroSplitPass : RequiredPassInfoMixin<CoroSplitPass> {
// Would be true if the Optimization level isn't O0.
bool OptimizeFrame;
+
+ Mode SplitMode = Mode::All;
};
} // end namespace llvm
diff --git a/llvm/lib/Analysis/ValueLatticeUtils.cpp b/llvm/lib/Analysis/ValueLatticeUtils.cpp
index ee24c987f6ff73..acc19667d77649 100644
--- a/llvm/lib/Analysis/ValueLatticeUtils.cpp
+++ b/llvm/lib/Analysis/ValueLatticeUtils.cpp
@@ -22,13 +22,7 @@ bool llvm::canTrackArgumentsInterprocedurally(Function *F) {
}
bool llvm::canTrackReturnsInterprocedurally(Function *F) {
- // Before coroutine splitting, the returns from a coroutine may not be
- // represented by ret instructions. In particular, returned-continuation
- // coroutines end in coro.end followed by unreachable, but splitting later
- // creates a concrete return value. Do not infer undef from the absence of
- // a ret in the pre-split body.
- return F->hasExactDefinition() && !F->hasFnAttribute(Attribute::Naked) &&
- !F->hasFnAttribute(Attribute::PresplitCoroutine);
+ return F->hasExactDefinition() && !F->hasFnAttribute(Attribute::Naked);
}
bool llvm::canTrackGlobalVariableInterprocedurally(GlobalVariable *GV) {
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index bd37fffc8e9914..f5a429182ba3e1 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -2006,6 +2006,15 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
return MPM;
}
+ // Make returned-continuation coroutine returns explicit before ICP and
+ // return-sensitive IPO inspect their ramp functions. Leave other coroutine
+ // ABIs on the existing late-split path.
+ CGSCCPassManager EarlyRetconCGPM;
+ EarlyRetconCGPM.addPass(CoroSplitPass(
+ Level != OptimizationLevel::O0, CoroSplitPass::Mode::RetconOnly));
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ std::move(EarlyRetconCGPM)));
+
if (!UseCtxProfile.empty()) {
MPM.addPass(
buildModuleInlinerPipeline(Level, ThinOrFullLTOPhase::ThinLTOPostLink));
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index db8f835a5cedf4..91fa685a3708e7 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -2279,6 +2279,11 @@ CoroSplitPass::CoroSplitPass(bool OptimizeFrame)
}),
OptimizeFrame(OptimizeFrame) {}
+CoroSplitPass::CoroSplitPass(bool OptimizeFrame, Mode SplitMode)
+ : CoroSplitPass(OptimizeFrame) {
+ this->SplitMode = SplitMode;
+}
+
CoroSplitPass::CoroSplitPass(
SmallVector<CoroSplitPass::BaseABITy> GenCustomABIs, bool OptimizeFrame)
: CreateAndInitABI([=](Function &F, coro::Shape &S) {
@@ -2327,7 +2332,8 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
// Check for uses of llvm.coro.prepare.retcon/async.
SmallVector<Function *, 2> PrepareFns;
addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
- addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
+ if (SplitMode == Mode::All)
+ addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
// Find coroutines for processing.
SmallVector<LazyCallGraph::Node *> Coroutines;
@@ -2354,6 +2360,10 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
if (!Shape.CoroBegin)
continue;
+ if (SplitMode == Mode::RetconOnly && Shape.ABI != coro::ABI::Retcon &&
+ Shape.ABI != coro::ABI::RetconOnce)
+ continue;
+
F.setSplittedCoroutine();
// Query BFI and populate SuspendFreqs right before splitting.
diff --git a/llvm/test/Transforms/SCCP/ipsccp-presplit-coroutine.ll b/llvm/test/Transforms/Coroutines/retcon-early-split-ipsccp.ll
similarity index 66%
rename from llvm/test/Transforms/SCCP/ipsccp-presplit-coroutine.ll
rename to llvm/test/Transforms/Coroutines/retcon-early-split-ipsccp.ll
index 8836cd5493f9c5..65e502fb7cbb1c 100644
--- a/llvm/test/Transforms/SCCP/ipsccp-presplit-coroutine.ll
+++ b/llvm/test/Transforms/Coroutines/retcon-early-split-ipsccp.ll
@@ -1,10 +1,7 @@
-; RUN: opt -passes=ipsccp -S %s | FileCheck %s
-; RUN: opt -passes='coro-early,ipsccp,cgscc(coro-split),verify' -S %s | FileCheck %s --check-prefix=SPLIT
+; RUN: opt -verify-each -passes='thinlto<O2>' -S %s | FileCheck %s
-; A returned-continuation coroutine has no ret before splitting. Its
-; coro.end is replaced with a real pair of continuation and yield pointers
-; during splitting. IPSCCP must not replace the direct call's result with
-; undef just because the pre-split body ends in unreachable.
+; An ICP-style direct/indirect call join must retain the direct return pair.
+; The ThinLTO pipeline must materialize retcon ramp returns before IPSCCP.
declare token @llvm.coro.id.retcon.once(i32, i32, ptr, ptr, ptr, ptr)
declare ptr @llvm.coro.begin(token, ptr)
@@ -54,29 +51,9 @@ join:
ret void
}
+; CHECK-LABEL: define internal swiftcc { ptr, ptr } @accessor(
+; CHECK: ret { ptr, ptr }
; CHECK-LABEL: define void @caller(
-; CHECK: direct:
-; CHECK: %direct_pair = call swiftcc { ptr, ptr } @accessor(
-; CHECK: join:
; CHECK: %pair = phi { ptr, ptr } [ %direct_pair, %direct ], [ %indirect_pair, %indirect ]
-; SPLIT-LABEL: define internal swiftcc { ptr, ptr } @accessor(
-; SPLIT: ret { ptr, ptr }
-; SPLIT-LABEL: define void @caller(
-; SPLIT: %pair = phi { ptr, ptr } [ %direct_pair, %direct ], [ %indirect_pair, %indirect ]
-
-define internal i32 @ordinary() {
-entry:
- ret i32 7
-}
-
-define i32 @ordinary_caller() {
-entry:
- %value = call i32 @ordinary()
- ret i32 %value
-}
-
-; CHECK-LABEL: define i32 @ordinary_caller()
-; CHECK: ret i32 7
-
attributes #0 = { noinline presplitcoroutine }
>From 92ab995c2ba934906525b72f61dd3e5eb2e85c77 Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Wed, 30 Sep 2026 14:33:55 +0200
Subject: [PATCH 3/8] Skip unrelated coroutines before early split cleanup
---
llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 91fa685a3708e7..66e8c1164f6d85 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -2337,9 +2337,18 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
// Find coroutines for processing.
SmallVector<LazyCallGraph::Node *> Coroutines;
- for (LazyCallGraph::Node &N : C)
- if (N.getFunction().isPresplitCoroutine())
- Coroutines.push_back(&N);
+ for (LazyCallGraph::Node &N : C) {
+ Function &F = N.getFunction();
+ if (!F.isPresplitCoroutine())
+ continue;
+ if (SplitMode == Mode::RetconOnly &&
+ !any_of(instructions(F), [](Instruction &I) {
+ auto *Begin = dyn_cast<CoroBeginInst>(&I);
+ return Begin && isa<AnyCoroIdRetconInst>(Begin->getId());
+ }))
+ continue;
+ Coroutines.push_back(&N);
+ }
if (Coroutines.empty() && PrepareFns.empty())
return PreservedAnalyses::all();
>From 27382e36fe4c08cabcda200104398de713fb064a Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Wed, 30 Sep 2026 14:43:27 +0200
Subject: [PATCH 4/8] Format early retcon pass scheduling
---
llvm/lib/Passes/PassBuilderPipelines.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index f5a429182ba3e1..a8999b8c1fcc4c 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -2010,10 +2010,10 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
// return-sensitive IPO inspect their ramp functions. Leave other coroutine
// ABIs on the existing late-split path.
CGSCCPassManager EarlyRetconCGPM;
- EarlyRetconCGPM.addPass(CoroSplitPass(
- Level != OptimizationLevel::O0, CoroSplitPass::Mode::RetconOnly));
- MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
- std::move(EarlyRetconCGPM)));
+ EarlyRetconCGPM.addPass(CoroSplitPass(Level != OptimizationLevel::O0,
+ CoroSplitPass::Mode::RetconOnly));
+ MPM.addPass(
+ createModuleToPostOrderCGSCCPassAdaptor(std::move(EarlyRetconCGPM)));
if (!UseCtxProfile.empty()) {
MPM.addPass(
>From 17756f7f74bbfc22094a5e2ad6021698c560f3c0 Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Wed, 30 Sep 2026 17:18:35 +0200
Subject: [PATCH 5/8] Represent retcon coroutine returns explicitly and upgrade
legacy IR
---
llvm/docs/Coroutines.md | 37 +++++++++---
llvm/include/llvm/IR/Intrinsics.td | 4 +-
.../llvm/Transforms/Coroutines/CoroInstr.h | 12 +++-
.../llvm/Transforms/Coroutines/CoroSplit.h | 4 --
llvm/lib/IR/AutoUpgrade.cpp | 58 +++++++++++++++++++
llvm/lib/Passes/PassBuilderPipelines.cpp | 9 ---
llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 27 ++-------
llvm/lib/Transforms/Coroutines/Coroutines.cpp | 7 +++
llvm/lib/Transforms/Coroutines/SpillUtils.cpp | 7 +++
.../coro-retcon-alloca-opaque-ptr.ll | 2 +-
.../Coroutines/coro-retcon-alloca.ll | 4 +-
.../Coroutines/coro-retcon-once-value2.ll | 3 +
.../Coroutines/coro-retcon-remat.ll | 1 +
.../Coroutines/coro-retcon-resume-values2.ll | 1 +
.../Coroutines/coro-retcon-unreachable.ll | 3 +-
.../test/Transforms/Coroutines/coro-retcon.ll | 9 ++-
.../Coroutines/retcon-explicit-return-slot.ll | 58 +++++++++++++++++++
...ipsccp.ll => retcon-return-slot-ipsccp.ll} | 28 ++++++---
18 files changed, 211 insertions(+), 63 deletions(-)
create mode 100644 llvm/test/Transforms/Coroutines/retcon-explicit-return-slot.ll
rename llvm/test/Transforms/Coroutines/{retcon-early-split-ipsccp.ll => retcon-return-slot-ipsccp.ll} (63%)
diff --git a/llvm/docs/Coroutines.md b/llvm/docs/Coroutines.md
index fa18a71ad66ddf..b1b0066038b2b0 100644
--- a/llvm/docs/Coroutines.md
+++ b/llvm/docs/Coroutines.md
@@ -1329,7 +1329,8 @@ A frontend should emit function attribute `presplitcoroutine` for the coroutine.
declare token @llvm.coro.id.retcon(i32 <size>, i32 <align>, ptr <buffer>,
ptr <continuation prototype>,
- ptr <alloc>, ptr <dealloc>)
+ ptr <alloc>, ptr <dealloc>,
+ ptr <return.slot>)
```
##### Overview:
@@ -1372,6 +1373,17 @@ or throwing an exception. It must take an integer and return a pointer.
The sixth argument must be a reference to a global function that will
be used to deallocate memory. It must take a pointer and return `void`.
+The seventh argument points to an alloca of the coroutine function's return
+type in that function. A non-unwind `llvm.coro.end` is followed by a load
+from this alloca and an explicit return. This makes the return visible to
+interprocedural optimizations before the coroutine is split. The alloca
+is not stored in the coroutine frame; `coro-split` replaces the load with
+the ABI-specific return value.
+
+Bitcode and assembly using the older six-argument form, with an implicit
+return and `unreachable` after a non-unwind `coro.end`, are upgraded when
+read by LLVM.
+
##### Semantics:
A frontend should emit function attribute `presplitcoroutine` for the coroutine.
@@ -1381,7 +1393,8 @@ A frontend should emit function attribute `presplitcoroutine` for the coroutine.
declare token @llvm.coro.id.retcon.once(i32 <size>, i32 <align>, ptr <buffer>,
ptr <prototype>,
- ptr <alloc>, ptr <dealloc>)
+ ptr <alloc>, ptr <dealloc>,
+ ptr <return.slot>)
```
##### Overview:
@@ -1391,7 +1404,7 @@ unique-suspend returned-continuation coroutine.
##### Arguments:
-As for `llvm.core.id.retcon`, except that the return type of the
+As for `llvm.coro.id.retcon`, except that the return type of the
continuation prototype must represent the normal return type of the continuation
(instead of matching the coroutine's return type).
@@ -1439,10 +1452,13 @@ The purposes of `coro.end` are:
optimizations from erasing stores to frame before returning.
In returned-continuation lowering, `llvm.coro.end` fully destroys the
-coroutine frame. If the second argument is `false`, it also returns from
-the coroutine with a null continuation pointer, and the next instruction
-will be unreachable. If the second argument is `true`, it falls through
-so that the following logic can resume unwinding. In a yield-once
+coroutine frame. If the second argument is `false`, it falls through to
+a load from the return slot supplied to `llvm.coro.id.retcon` or
+`llvm.coro.id.retcon.once`, followed by an explicit return. `coro-split`
+replaces that return with the ABI-specific value, including a null
+continuation on completion for the multiple-suspend ABI. If the second
+argument is `true`, it falls through so that the following logic can
+resume unwinding. In a yield-once
coroutine, reaching a non-unwind `llvm.coro.end` without having first
reached a `llvm.coro.suspend.retcon` has undefined behavior.
@@ -1537,9 +1553,11 @@ The number of arguments must match the return type of the continuation function:
```llvm
define {ptr, ptr} @g(ptr %buffer, ptr %ptr, i8 %val) presplitcoroutine {
entry:
+ %return.slot = alloca {ptr, ptr}
%id = call token @llvm.coro.id.retcon.once(i32 8, i32 8, ptr %buffer,
ptr @prototype,
- ptr @allocate, ptr @deallocate)
+ ptr @allocate, ptr @deallocate,
+ ptr %return.slot)
%hdl = call ptr @llvm.coro.begin(token %id, ptr null)
...
@@ -1547,7 +1565,8 @@ entry:
cleanup:
%tok = call token (...) @llvm.coro.end.results(i8 %val)
call void @llvm.coro.end(ptr %hdl, i1 0, token %tok)
- unreachable
+ %result = load {ptr, ptr}, ptr %return.slot
+ ret {ptr, ptr} %result
...
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 52be391132b7e8..6858361a0ce428 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1977,11 +1977,11 @@ def int_coro_id : DefaultAttrsIntrinsic<[llvm_token_ty],
NoCapture<ArgIndex<2>>]>;
def int_coro_id_retcon : Intrinsic<[llvm_token_ty],
[llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty,
- llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
+ llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
[]>;
def int_coro_id_retcon_once : Intrinsic<[llvm_token_ty],
[llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty,
- llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
+ llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
[]>;
def int_coro_alloc : Intrinsic<[llvm_i1_ty], [llvm_token_ty], [IntrNoMem]>;
def int_coro_id_async : Intrinsic<[llvm_token_ty],
diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
index 71632a46dcde17..d1f1b1ee3c734c 100644
--- a/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
+++ b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
@@ -236,7 +236,15 @@ class CoroIdInst : public AnyCoroIdInst {
/// This represents either the llvm.coro.id.retcon or
/// llvm.coro.id.retcon.once instruction.
class AnyCoroIdRetconInst : public AnyCoroIdInst {
- enum { SizeArg, AlignArg, StorageArg, PrototypeArg, AllocArg, DeallocArg };
+ enum {
+ SizeArg,
+ AlignArg,
+ StorageArg,
+ PrototypeArg,
+ AllocArg,
+ DeallocArg,
+ ReturnSlotArg
+ };
public:
LLVM_ABI void checkWellFormed() const;
@@ -251,6 +259,8 @@ class AnyCoroIdRetconInst : public AnyCoroIdInst {
Value *getStorage() const { return getArgOperand(StorageArg); }
+ Value *getReturnSlot() const { return getArgOperand(ReturnSlotArg); }
+
/// Return the prototype for the continuation function. The type,
/// attributes, and calling convention of the continuation function(s)
/// are taken from this declaration.
diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
index c4e30cf7593552..29a2dbb903658e 100644
--- a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
+++ b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
@@ -29,13 +29,10 @@ struct Shape;
} // namespace coro
struct CoroSplitPass : RequiredPassInfoMixin<CoroSplitPass> {
- enum class Mode { All, RetconOnly };
-
using BaseABITy =
std::function<std::unique_ptr<coro::BaseABI>(Function &, coro::Shape &)>;
LLVM_ABI CoroSplitPass(bool OptimizeFrame = false);
- LLVM_ABI CoroSplitPass(bool OptimizeFrame, Mode SplitMode);
LLVM_ABI CoroSplitPass(SmallVector<BaseABITy> GenCustomABIs,
bool OptimizeFrame = false);
@@ -59,7 +56,6 @@ struct CoroSplitPass : RequiredPassInfoMixin<CoroSplitPass> {
// Would be true if the Optimization level isn't O0.
bool OptimizeFrame;
- Mode SplitMode = Mode::All;
};
} // end namespace llvm
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index c4e4df7ace8b1b..cd14074ac7d609 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1803,6 +1803,16 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
break;
}
case 'c': {
+ if ((Name == "coro.id.retcon" || Name == "coro.id.retcon.once") &&
+ F->arg_size() == 6) {
+ Intrinsic::ID ID = Name == "coro.id.retcon"
+ ? Intrinsic::coro_id_retcon
+ : Intrinsic::coro_id_retcon_once;
+ rename(F);
+ NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
+ return true;
+ }
+
if (F->arg_size() == 1) {
if (Name.consume_front("convert.")) {
if (convertIntrinsicValidType(Name, F->getFunctionType())) {
@@ -5866,6 +5876,54 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
DefaultCase();
return;
}
+ case Intrinsic::coro_id_retcon:
+ case Intrinsic::coro_id_retcon_once: {
+ // Older frontends emitted an implicit return at coro.end. Keep their
+ // bitcode usable by making that return explicit before any IPO passes.
+ Function *Coro = CI->getFunction();
+ // An older optimizer could have inferred these from the unreachable that
+ // used to follow coro.end. They are not valid for the explicit return.
+ Coro->removeFnAttr(Attribute::NoReturn);
+ Coro->removeRetAttr(Attribute::NoAlias);
+ Coro->removeRetAttr(Attribute::NonNull);
+ for (User *U : Coro->users()) {
+ auto *Call = dyn_cast<CallBase>(U);
+ if (!Call || Call->getCalledOperand() != Coro)
+ continue;
+ Call->removeFnAttr(Attribute::NoReturn);
+ Call->removeRetAttr(Attribute::NoAlias);
+ Call->removeRetAttr(Attribute::NonNull);
+ }
+ IRBuilder<> EntryBuilder(&*Coro->getEntryBlock().getFirstInsertionPt());
+ AllocaInst *ReturnSlot =
+ EntryBuilder.CreateAlloca(Coro->getReturnType(), nullptr, "coro.ret");
+
+ SmallVector<Value *, 7> Args(CI->args());
+ Args.push_back(ReturnSlot);
+ NewCall = Builder.CreateCall(NewFn, Args);
+ NewCall->setAttributes(CI->getAttributes());
+ NewCall->copyMetadata(*CI);
+ NewCall->setDebugLoc(CI->getDebugLoc());
+
+ for (BasicBlock &BB : *Coro) {
+ auto *Unreachable = dyn_cast<UnreachableInst>(BB.getTerminator());
+ if (!Unreachable)
+ continue;
+ auto *End = dyn_cast_or_null<CallBase>(Unreachable->getPrevNode());
+ if (!End || End->getIntrinsicID() != Intrinsic::coro_end)
+ continue;
+ auto *Unwind = dyn_cast<ConstantInt>(End->getArgOperand(1));
+ if (!Unwind || !Unwind->isZero())
+ continue;
+
+ IRBuilder<> ReturnBuilder(Unreachable);
+ Value *Result =
+ ReturnBuilder.CreateLoad(Coro->getReturnType(), ReturnSlot);
+ ReturnBuilder.CreateRet(Result);
+ Unreachable->eraseFromParent();
+ }
+ break;
+ }
case Intrinsic::arm_neon_vst1:
case Intrinsic::arm_neon_vst2:
case Intrinsic::arm_neon_vst3:
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index a8999b8c1fcc4c..bd37fffc8e9914 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -2006,15 +2006,6 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
return MPM;
}
- // Make returned-continuation coroutine returns explicit before ICP and
- // return-sensitive IPO inspect their ramp functions. Leave other coroutine
- // ABIs on the existing late-split path.
- CGSCCPassManager EarlyRetconCGPM;
- EarlyRetconCGPM.addPass(CoroSplitPass(Level != OptimizationLevel::O0,
- CoroSplitPass::Mode::RetconOnly));
- MPM.addPass(
- createModuleToPostOrderCGSCCPassAdaptor(std::move(EarlyRetconCGPM)));
-
if (!UseCtxProfile.empty()) {
MPM.addPass(
buildModuleInlinerPipeline(Level, ThinOrFullLTOPhase::ThinLTOPostLink));
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 66e8c1164f6d85..db8f835a5cedf4 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -2279,11 +2279,6 @@ CoroSplitPass::CoroSplitPass(bool OptimizeFrame)
}),
OptimizeFrame(OptimizeFrame) {}
-CoroSplitPass::CoroSplitPass(bool OptimizeFrame, Mode SplitMode)
- : CoroSplitPass(OptimizeFrame) {
- this->SplitMode = SplitMode;
-}
-
CoroSplitPass::CoroSplitPass(
SmallVector<CoroSplitPass::BaseABITy> GenCustomABIs, bool OptimizeFrame)
: CreateAndInitABI([=](Function &F, coro::Shape &S) {
@@ -2332,23 +2327,13 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
// Check for uses of llvm.coro.prepare.retcon/async.
SmallVector<Function *, 2> PrepareFns;
addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
- if (SplitMode == Mode::All)
- addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
+ addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
// Find coroutines for processing.
SmallVector<LazyCallGraph::Node *> Coroutines;
- for (LazyCallGraph::Node &N : C) {
- Function &F = N.getFunction();
- if (!F.isPresplitCoroutine())
- continue;
- if (SplitMode == Mode::RetconOnly &&
- !any_of(instructions(F), [](Instruction &I) {
- auto *Begin = dyn_cast<CoroBeginInst>(&I);
- return Begin && isa<AnyCoroIdRetconInst>(Begin->getId());
- }))
- continue;
- Coroutines.push_back(&N);
- }
+ for (LazyCallGraph::Node &N : C)
+ if (N.getFunction().isPresplitCoroutine())
+ Coroutines.push_back(&N);
if (Coroutines.empty() && PrepareFns.empty())
return PreservedAnalyses::all();
@@ -2369,10 +2354,6 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
if (!Shape.CoroBegin)
continue;
- if (SplitMode == Mode::RetconOnly && Shape.ABI != coro::ABI::Retcon &&
- Shape.ABI != coro::ABI::RetconOnce)
- continue;
-
F.setSplittedCoroutine();
// Query BFI and populate SuspendFreqs right before splitting.
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index 2ecf66d18b5d33..53e17442e5ccba 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -635,6 +635,13 @@ void AnyCoroIdRetconInst::checkWellFormed() const {
checkWFRetconPrototype(this, getArgOperand(PrototypeArg));
checkWFAlloc(this, getArgOperand(AllocArg));
checkWFDealloc(this, getArgOperand(DeallocArg));
+ auto *ReturnSlot = dyn_cast<AllocaInst>(getReturnSlot());
+ if (!ReturnSlot || ReturnSlot->getFunction() != getFunction() ||
+ ReturnSlot->getAllocatedType() != getFunction()->getReturnType())
+ fail(this,
+ "return slot of coro.id.retcon.* must be a return-typed "
+ "alloca in the coroutine function",
+ getReturnSlot());
}
static void checkAsyncFuncPointer(const Instruction *I, Value *V) {
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index 05abccf0f9a972..5d59a73abd17cb 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -434,6 +434,13 @@ static void collectFrameAlloca(AllocaInst *AI, const coro::Shape &Shape,
if (AI == Shape.SwitchLowering.PromiseAlloca)
return;
+ // The return slot expresses a retcon coroutine's pre-split return. It is
+ // not coroutine-frame state: CoroSplit replaces the post-coro.end load with
+ // the concrete ABI return before execution.
+ if ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) &&
+ AI == Shape.getRetconCoroId()->getReturnSlot())
+ return;
+
// The __coro_gro alloca should outlive the promise, make sure we
// keep it outside the frame.
if (AI->hasMetadata(LLVMContext::MD_coro_outside_frame))
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll b/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll
index 94ed43ee43f2e1..359c83cbb7360b 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll
@@ -46,7 +46,7 @@ define {ptr, i32} @g(ptr %buffer, i32 %n) {
; CHECK-NEXT: store i32 [[N:%.*]], ptr [[BUFFER:%.*]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = alloca i8, i64 [[TMP0]], align 8
-; CHECK-NEXT: tail call void @use(ptr nonnull [[TMP1]])
+; CHECK-NEXT: call void @use(ptr nonnull [[TMP1]])
; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr, i32 } { ptr @g.resume.0, i32 poison }, i32 [[N]], 1
; CHECK-NEXT: ret { ptr, i32 } [[TMP2]]
;
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
index 6caa571cf1bbc5..f507bb17e9522f 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
@@ -46,7 +46,7 @@ define {ptr, i32} @g(ptr %buffer, i32 %n) {
; CHECK-NEXT: store i32 [[N:%.*]], ptr [[BUFFER:%.*]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = alloca i8, i64 [[TMP0]], align 8
-; CHECK-NEXT: tail call void @use(ptr nonnull [[TMP1]])
+; CHECK-NEXT: call void @use(ptr nonnull [[TMP1]])
; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr, i32 } { ptr @g.resume.0, i32 poison }, i32 [[N]], 1
; CHECK-NEXT: ret { ptr, i32 } [[TMP2]]
;
@@ -186,7 +186,7 @@ define {ptr, i32} @k(ptr %buffer, i32 %n, i1 %cond) {
; CHECK: alloca_block:
; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[SIZE]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
-; CHECK-NEXT: tail call void @use(ptr nonnull [[TMP2]])
+; CHECK-NEXT: call void @use(ptr nonnull [[TMP2]])
; CHECK-NEXT: br label [[CORO_RETURN]]
;
entry:
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
index bf95b2a74e6de7..8952ade6890a57 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
@@ -84,6 +84,7 @@ declare fastcc void @deallocate(ptr %ptr)
declare void @print(i32)
; CHECK-LABEL: @f(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CORO_RET:%.*]] = alloca { ptr, ptr }, align 8
; CHECK-NEXT: [[TMP0:%.*]] = call ptr @allocate(i32 12)
; CHECK-NEXT: store ptr [[TMP0]], ptr [[BUFFER:%.*]], align 8
; CHECK-NEXT: [[TEMP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
@@ -111,6 +112,7 @@ declare void @print(i32)
;
; CHECK-LABEL: @g(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CORO_RET:%.*]] = alloca { ptr, ptr }, align 8
; CHECK-NEXT: [[TMP0:%.*]] = call ptr @allocate(i32 13)
; CHECK-NEXT: store ptr [[TMP0]], ptr [[BUFFER:%.*]], align 8
; CHECK-NEXT: [[TEMP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
@@ -142,6 +144,7 @@ declare void @print(i32)
;
; CHECK-LABEL: @h(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CORO_RET:%.*]] = alloca { ptr, ptr }, align 8
; CHECK-NEXT: [[TMP0:%.*]] = call ptr @allocate(i32 12)
; CHECK-NEXT: store ptr [[TMP0]], ptr [[BUFFER:%.*]], align 8
; CHECK-NEXT: [[TEMP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-remat.ll b/llvm/test/Transforms/Coroutines/coro-retcon-remat.ll
index ed680adb48eefc..95b16f23b8d7ee 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-remat.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-remat.ll
@@ -11,6 +11,7 @@ define { ptr, i32 } @f(ptr %buffer, i32 %n) {
; CHECK-LABEL: define { ptr, i32 } @f(
; CHECK-SAME: ptr [[BUFFER:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CORO_RET:%.*]] = alloca { ptr, i32 }, align 8
; CHECK-NEXT: store i32 [[N]], ptr [[BUFFER]], align 4
; CHECK-NEXT: call void @print(i32 [[N]])
; CHECK-NEXT: [[INC1:%.*]] = add i32 [[N]], 1
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
index b1dfbd1b6d4f69..9ae31a1676fbde 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
@@ -35,6 +35,7 @@ declare void @print(i32)
; CHECK-LABEL: @f(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CORO_RET:%.*]] = alloca ptr, align 8
; CHECK-NEXT: [[TMP0:%.*]] = call ptr @allocate(i32 20)
; CHECK-NEXT: store ptr [[TMP0]], ptr [[BUFFER:%.*]], align 8
; CHECK-NEXT: store i32 [[N:%.*]], ptr [[TMP0]], align 4
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll b/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll
index 6e4a287e53b0ac..9bbfbf12289bce 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll
@@ -10,7 +10,8 @@ target datalayout = "E-p:64:64"
define hidden swiftcc { ptr, ptr } @no_suspends(ptr %buffer, i64 %arg) #1 {
; CHECK-LABEL: @no_suspends(
; CHECK-NEXT: AllocaSpillBB:
-; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id.retcon.once(i32 32, i32 8, ptr [[BUFFER:%.*]], ptr @prototype, ptr @malloc, ptr @free)
+; CHECK-NEXT: [[CORO_RET:%.*]] = alloca { ptr, ptr }, align 8
+; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id.retcon.once(i32 32, i32 8, ptr [[BUFFER:%.*]], ptr @prototype, ptr @malloc, ptr @free, ptr [[CORO_RET]])
; CHECK-NEXT: call void @print(i64 [[ARG:%.*]])
; CHECK-NEXT: call void @llvm.trap()
; CHECK-NEXT: unreachable
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon.ll b/llvm/test/Transforms/Coroutines/coro-retcon.ll
index cd2488adc74be9..149584ad57ebe5 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon.ll
@@ -12,6 +12,7 @@ define ptr @f(ptr %buffer, i32 %n) {
;
; CORO-LABEL: @f(
; CORO-NEXT: entry:
+; CORO-NEXT: [[CORO_RET:%.*]] = alloca ptr, align 8
; CORO-NEXT: store i32 [[N:%.*]], ptr [[N_VAL_SPILL_ADDR:%.*]], align 4
; CORO-NEXT: call void @print(i32 [[N]])
; CORO-NEXT: ret ptr @f.resume.0
@@ -43,9 +44,9 @@ define i32 @main() {
; CHECK-NEXT: entry:
; CHECK-NEXT: tail call void @print(i32 4)
; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
-; CHECK-NEXT: tail call void @print(i32 5), !noalias [[META0]]
-; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]])
-; CHECK-NEXT: tail call void @print(i32 6), !noalias [[META3]]
+; CHECK-NEXT: tail call void @print(i32 5), !noalias [[META0]], !inline_history [[META3:![0-9]+]]
+; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata [[META4:![0-9]+]])
+; CHECK-NEXT: tail call void @print(i32 6), !noalias [[META4]], !inline_history [[META3]]
; CHECK-NEXT: ret i32 0
;
; CORO-LABEL: @main(
@@ -81,6 +82,7 @@ define hidden { ptr, ptr } @g(ptr %buffer, ptr %ptr) {
;
; CORO-LABEL: @g(
; CORO-NEXT: entry:
+; CORO-NEXT: [[CORO_RET:%.*]] = alloca { ptr, ptr }, align 8
; CORO-NEXT: [[TMP0:%.*]] = call ptr @allocate(i32 8)
; CORO-NEXT: store ptr [[TMP0]], ptr [[BUFFER:%.*]], align 8
; CORO-NEXT: store ptr [[PTR:%.*]], ptr [[TMP0]], align 8
@@ -119,6 +121,7 @@ define ptr @nosuspend(ptr %buffer, i32 %n) {
;
; CORO-LABEL: @nosuspend(
; CORO-NEXT: entry:
+; CORO-NEXT: [[CORO_RET:%.*]] = alloca ptr, align 8
; CORO-NEXT: [[A:%.*]] = alloca i32, align 4
; CORO-NEXT: store i32 [[N:%.*]], ptr [[A]], align 4
; CORO-NEXT: call void @use_var_ptr(ptr [[A]])
diff --git a/llvm/test/Transforms/Coroutines/retcon-explicit-return-slot.ll b/llvm/test/Transforms/Coroutines/retcon-explicit-return-slot.ll
new file mode 100644
index 00000000000000..d3ec4d93432ff7
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/retcon-explicit-return-slot.ll
@@ -0,0 +1,58 @@
+; RUN: opt -verify-each -passes='ipsccp' -S %s | FileCheck %s --check-prefix=PRE
+; RUN: opt -verify-each -passes='coro-early,ipsccp,coro-split' -S %s | FileCheck %s --check-prefix=SPLIT
+
+; Both returned-continuation ABIs have an explicit pre-split return. The
+; return slot is not part of the coroutine frame, even though it escapes via
+; coro.id.retcon, and splitting replaces the placeholder load.
+
+declare token @llvm.coro.id.retcon(i32, i32, ptr, ptr, ptr, ptr, ptr)
+declare token @llvm.coro.id.retcon.once(i32, i32, ptr, ptr, ptr, ptr, ptr)
+declare ptr @llvm.coro.begin(token, ptr)
+declare i1 @llvm.coro.suspend.retcon.i1(...)
+declare void @llvm.coro.end(ptr, i1, token)
+declare ptr @malloc(i64)
+declare void @free(ptr)
+declare ptr @normal.prototype(ptr, i1)
+declare void @once.prototype(ptr, i1)
+
+define ptr @normal(ptr %buffer) presplitcoroutine {
+entry:
+ %return.slot = alloca ptr
+ %id = call token @llvm.coro.id.retcon(i32 8, i32 8, ptr %buffer,
+ ptr @normal.prototype, ptr @malloc, ptr @free, ptr %return.slot)
+ %frame = call ptr @llvm.coro.begin(token %id, ptr null)
+ %suspended = call i1 (...) @llvm.coro.suspend.retcon.i1()
+ call void @llvm.coro.end(ptr %frame, i1 false, token none)
+ %result = load ptr, ptr %return.slot
+ ret ptr %result
+}
+
+define ptr @once(ptr %buffer) presplitcoroutine {
+entry:
+ %return.slot = alloca ptr
+ %id = call token @llvm.coro.id.retcon.once(i32 8, i32 8, ptr %buffer,
+ ptr @once.prototype, ptr @malloc, ptr @free, ptr %return.slot)
+ %frame = call ptr @llvm.coro.begin(token %id, ptr null)
+ %suspended = call i1 (...) @llvm.coro.suspend.retcon.i1()
+ call void @llvm.coro.end(ptr %frame, i1 false, token none)
+ %result = load ptr, ptr %return.slot
+ ret ptr %result
+}
+
+; PRE-LABEL: define ptr @normal(
+; PRE: %return.slot = alloca ptr
+; PRE: call token @llvm.coro.id.retcon({{.*}}ptr %return.slot)
+; PRE: load ptr, ptr %return.slot
+; PRE: ret ptr
+; PRE-LABEL: define ptr @once(
+; PRE: %return.slot = alloca ptr
+; PRE: call token @llvm.coro.id.retcon.once({{.*}}ptr %return.slot)
+; PRE: load ptr, ptr %return.slot
+; PRE: ret ptr
+
+; SPLIT-LABEL: define ptr @normal(
+; SPLIT: phi ptr [ @normal.resume.0
+; SPLIT: ret ptr
+; SPLIT-LABEL: define ptr @once(
+; SPLIT: phi ptr [ @once.resume.0
+; SPLIT: ret ptr
diff --git a/llvm/test/Transforms/Coroutines/retcon-early-split-ipsccp.ll b/llvm/test/Transforms/Coroutines/retcon-return-slot-ipsccp.ll
similarity index 63%
rename from llvm/test/Transforms/Coroutines/retcon-early-split-ipsccp.ll
rename to llvm/test/Transforms/Coroutines/retcon-return-slot-ipsccp.ll
index 65e502fb7cbb1c..8aba641660bd02 100644
--- a/llvm/test/Transforms/Coroutines/retcon-early-split-ipsccp.ll
+++ b/llvm/test/Transforms/Coroutines/retcon-return-slot-ipsccp.ll
@@ -1,7 +1,8 @@
-; RUN: opt -verify-each -passes='thinlto<O2>' -S %s | FileCheck %s
+; RUN: opt -verify-each -passes='ipsccp' -S %s | FileCheck %s --check-prefix=PRE
+; RUN: opt -verify-each -passes='thinlto<O2>' -S %s | FileCheck %s --check-prefix=POST
; An ICP-style direct/indirect call join must retain the direct return pair.
-; The ThinLTO pipeline must materialize retcon ramp returns before IPSCCP.
+; Older six-operand Swift-style IR is upgraded to an explicit return slot.
declare token @llvm.coro.id.retcon.once(i32, i32, ptr, ptr, ptr, ptr)
declare ptr @llvm.coro.begin(token, ptr)
@@ -33,7 +34,7 @@ entry:
direct:
%direct_pair = call swiftcc { ptr, ptr } @accessor(
- ptr noalias %buffer, ptr swiftself %object)
+ ptr noalias %buffer, ptr swiftself %object) #1
br label %join
indirect:
@@ -51,9 +52,20 @@ join:
ret void
}
-; CHECK-LABEL: define internal swiftcc { ptr, ptr } @accessor(
-; CHECK: ret { ptr, ptr }
-; CHECK-LABEL: define void @caller(
-; CHECK: %pair = phi { ptr, ptr } [ %direct_pair, %direct ], [ %indirect_pair, %indirect ]
+; PRE-LABEL: define internal swiftcc { ptr, ptr } @accessor(
+; PRE: %coro.ret = alloca { ptr, ptr }
+; PRE: call token @llvm.coro.id.retcon.once({{.*}}ptr %coro.ret)
+; PRE: load { ptr, ptr }, ptr %coro.ret
+; PRE: ret { ptr, ptr }
+; PRE-LABEL: define void @caller(
+; PRE: %direct_pair = call swiftcc { ptr, ptr } @accessor(ptr noalias %buffer, ptr swiftself %object){{$}}
+; PRE: %pair = phi { ptr, ptr } [ %direct_pair, %direct ], [ %indirect_pair, %indirect ]
+; PRE-NOT: noreturn
+
+; POST-LABEL: define internal swiftcc { ptr, ptr } @accessor(
+; POST: ret { ptr, ptr }
+; POST-LABEL: define void @caller(
+; POST: %pair = phi { ptr, ptr } [ %direct_pair, %direct ], [ %indirect_pair, %indirect ]
-attributes #0 = { noinline presplitcoroutine }
+attributes #0 = { noinline noreturn presplitcoroutine }
+attributes #1 = { noreturn }
>From f29130538fcc07687ac8137a1c735adb45b9cdfc Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Wed, 30 Sep 2026 17:36:09 +0200
Subject: [PATCH 6/8] Format coroutine pass declaration
---
llvm/include/llvm/Transforms/Coroutines/CoroSplit.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
index 29a2dbb903658e..3c589370a04fc1 100644
--- a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
+++ b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
@@ -55,7 +55,6 @@ struct CoroSplitPass : RequiredPassInfoMixin<CoroSplitPass> {
// Would be true if the Optimization level isn't O0.
bool OptimizeFrame;
-
};
} // end namespace llvm
>From 1e40a34de763c345f8bcd7c60c643ee7f4f7501c Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Wed, 30 Sep 2026 18:04:00 +0200
Subject: [PATCH 7/8] Handle void retcon functions during return-slot upgrade
---
llvm/docs/Coroutines.md | 2 ++
llvm/lib/IR/AutoUpgrade.cpp | 21 +++++++++++++------
llvm/lib/Transforms/Coroutines/Coroutines.cpp | 6 ++++++
.../test/Transforms/FunctionAttrs/noreturn.ll | 2 ++
4 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/llvm/docs/Coroutines.md b/llvm/docs/Coroutines.md
index b1b0066038b2b0..63e54bc36c61ca 100644
--- a/llvm/docs/Coroutines.md
+++ b/llvm/docs/Coroutines.md
@@ -1379,6 +1379,8 @@ from this alloca and an explicit return. This makes the return visible to
interprocedural optimizations before the coroutine is split. The alloca
is not stored in the coroutine frame; `coro-split` replaces the load with
the ABI-specific return value.
+For a void-returning function, the seventh argument is null and a non-unwind
+`coro.end` is followed by `ret void` instead.
Bitcode and assembly using the older six-argument form, with an implicit
return and `unreachable` after a non-unwind `coro.end`, are upgraded when
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index cd14074ac7d609..c4703e14248de6 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5894,9 +5894,16 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
Call->removeRetAttr(Attribute::NoAlias);
Call->removeRetAttr(Attribute::NonNull);
}
- IRBuilder<> EntryBuilder(&*Coro->getEntryBlock().getFirstInsertionPt());
- AllocaInst *ReturnSlot =
- EntryBuilder.CreateAlloca(Coro->getReturnType(), nullptr, "coro.ret");
+ Value *ReturnSlot;
+ if (Coro->getReturnType()->isVoidTy()) {
+ // Legacy IR can use retcon in a void function. There is no result to
+ // store in that case, but it still needs the new operand.
+ ReturnSlot = ConstantPointerNull::get(Builder.getPtrTy());
+ } else {
+ IRBuilder<> EntryBuilder(&*Coro->getEntryBlock().getFirstInsertionPt());
+ ReturnSlot =
+ EntryBuilder.CreateAlloca(Coro->getReturnType(), nullptr, "coro.ret");
+ }
SmallVector<Value *, 7> Args(CI->args());
Args.push_back(ReturnSlot);
@@ -5917,9 +5924,11 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
continue;
IRBuilder<> ReturnBuilder(Unreachable);
- Value *Result =
- ReturnBuilder.CreateLoad(Coro->getReturnType(), ReturnSlot);
- ReturnBuilder.CreateRet(Result);
+ if (Coro->getReturnType()->isVoidTy())
+ ReturnBuilder.CreateRetVoid();
+ else
+ ReturnBuilder.CreateRet(
+ ReturnBuilder.CreateLoad(Coro->getReturnType(), ReturnSlot));
Unreachable->eraseFromParent();
}
break;
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index 53e17442e5ccba..1611069e0dab54 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -635,6 +635,12 @@ void AnyCoroIdRetconInst::checkWellFormed() const {
checkWFRetconPrototype(this, getArgOperand(PrototypeArg));
checkWFAlloc(this, getArgOperand(AllocArg));
checkWFDealloc(this, getArgOperand(DeallocArg));
+ if (getFunction()->getReturnType()->isVoidTy()) {
+ if (!isa<ConstantPointerNull>(getReturnSlot()))
+ fail(this, "return slot of a void retcon coroutine must be null",
+ getReturnSlot());
+ return;
+ }
auto *ReturnSlot = dyn_cast<AllocaInst>(getReturnSlot());
if (!ReturnSlot || ReturnSlot->getFunction() != getFunction() ||
ReturnSlot->getAllocatedType() != getFunction()->getReturnType())
diff --git a/llvm/test/Transforms/FunctionAttrs/noreturn.ll b/llvm/test/Transforms/FunctionAttrs/noreturn.ll
index ae2ccb02733f38..70ae42f49af162 100644
--- a/llvm/test/Transforms/FunctionAttrs/noreturn.ll
+++ b/llvm/test/Transforms/FunctionAttrs/noreturn.ll
@@ -79,6 +79,8 @@ define void @unreachable() {
; CHECK-NOT: Function Attrs: {{.*}}noreturn
; CHECK: @coro
+; CHECK: call token @llvm.coro.id.retcon.once({{.*}}ptr null)
+; CHECK: ret void
define void @coro() presplitcoroutine {
call token @llvm.coro.id.retcon.once(i32 0, i32 0, ptr null, ptr @coro, ptr null, ptr null)
call void (ptr, i1, ...) @llvm.coro.end(ptr null, i1 false)
>From fa6f32d7fd71009b1c580db960d518373ef9b388 Mon Sep 17 00:00:00 2001
From: Karim Alweheshy <karim.alweheshy at reddit.com>
Date: Wed, 30 Sep 2026 18:54:56 +0200
Subject: [PATCH 8/8] Recognize legacy coro.end during retcon return upgrade
---
llvm/lib/IR/AutoUpgrade.cpp | 10 ++++++-
.../auto_upgrade_coro_retcon_legacy_end.ll | 28 +++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Assembler/auto_upgrade_coro_retcon_legacy_end.ll
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index c4703e14248de6..1d59812b398e60 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5917,7 +5917,15 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
if (!Unreachable)
continue;
auto *End = dyn_cast_or_null<CallBase>(Unreachable->getPrevNode());
- if (!End || End->getIntrinsicID() != Intrinsic::coro_end)
+ if (!End)
+ continue;
+ Function *EndFn = End->getCalledFunction();
+ // The legacy i1-returning coro.end emitted by older frontends may be
+ // upgraded later in the same module read. Recognize it before its own
+ // auto-upgrade has run as well as after it has been renamed to .old.
+ if (End->getIntrinsicID() != Intrinsic::coro_end &&
+ (!EndFn || (EndFn->getName() != "llvm.coro.end" &&
+ !EndFn->getName().starts_with("llvm.coro.end.old"))))
continue;
auto *Unwind = dyn_cast<ConstantInt>(End->getArgOperand(1));
if (!Unwind || !Unwind->isZero())
diff --git a/llvm/test/Assembler/auto_upgrade_coro_retcon_legacy_end.ll b/llvm/test/Assembler/auto_upgrade_coro_retcon_legacy_end.ll
new file mode 100644
index 00000000000000..3e893847d6221e
--- /dev/null
+++ b/llvm/test/Assembler/auto_upgrade_coro_retcon_legacy_end.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S %s | FileCheck %s
+
+; The old coro.end declaration may already have been renamed to .old by its
+; own auto-upgrade when the retcon intrinsic is upgraded. The return must
+; become explicit regardless of the order in which declarations are visited.
+
+declare token @llvm.coro.id.retcon.once(i32, i32, ptr, ptr, ptr, ptr)
+declare ptr @llvm.coro.begin(token, ptr)
+declare void @llvm.coro.end.old(ptr, i1, token)
+declare void @resume(ptr, i1)
+declare ptr @malloc(i64)
+declare void @free(ptr)
+
+define ptr @legacy_end(ptr %buffer) presplitcoroutine {
+entry:
+ %id = call token @llvm.coro.id.retcon.once(i32 8, i32 8, ptr %buffer,
+ ptr @resume, ptr @malloc, ptr @free)
+ %frame = call ptr @llvm.coro.begin(token %id, ptr null)
+ call void @llvm.coro.end.old(ptr %frame, i1 false, token none)
+ unreachable
+}
+
+; CHECK-LABEL: define ptr @legacy_end(
+; CHECK: %coro.ret = alloca ptr
+; CHECK: call token @llvm.coro.id.retcon.once({{.*}}ptr %coro.ret)
+; CHECK: call void @llvm.coro.end.old(ptr %frame, i1 false, token none)
+; CHECK: load ptr, ptr %coro.ret
+; CHECK: ret ptr
More information about the llvm-commits
mailing list