[llvm] [coro][CoroSplit] Use `llvm.lifetime.end` to compute putting objects on the frame vs the stack (PR #90265)
Alan Zhao via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 30 17:08:03 PDT 2024
https://github.com/alanzhao1 updated https://github.com/llvm/llvm-project/pull/90265
>From b57f3018355838e8465de47c69d1705a8b5d15d8 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Fri, 26 Apr 2024 11:05:24 -0700
Subject: [PATCH 1/5] [nfc] Precommit a test for using lifetime.end to compute
coroutine frame objects
---
.../Coroutines/coro-lifetime-end.ll | 152 ++++++++++++++++++
1 file changed, 152 insertions(+)
create mode 100644 llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
diff --git a/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll b/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
new file mode 100644
index 00000000000000..de861317436d6a
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
+
+declare ptr @malloc(i64)
+
+%i8.array = type { [100 x i8] }
+declare void @consume.i8.array(ptr)
+
+ at testbool = external local_unnamed_addr global i8, align 1
+
+; testval does not contain an explicit lifetime end. We must assume that it may
+; live across suspension.
+define void @HasNoLifetimeEnd() presplitcoroutine {
+; CHECK-LABEL: define void @HasNoLifetimeEnd() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TESTVAL:%.*]] = alloca [[I8_ARRAY:%.*]], align 8
+; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr @HasNoLifetimeEnd.resumers)
+; CHECK-NEXT: [[ALLOC:%.*]] = call ptr @malloc(i64 16)
+; CHECK-NEXT: [[VFRAME:%.*]] = call noalias nonnull ptr @llvm.coro.begin(token [[ID]], ptr [[ALLOC]])
+; CHECK-NEXT: store ptr @HasNoLifetimeEnd.resume, ptr [[VFRAME]], align 8
+; CHECK-NEXT: [[DESTROY_ADDR:%.*]] = getelementptr inbounds [[HASNOLIFETIMEEND_FRAME:%.*]], ptr [[VFRAME]], i32 0, i32 1
+; CHECK-NEXT: store ptr @HasNoLifetimeEnd.destroy, ptr [[DESTROY_ADDR]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 100, ptr [[TESTVAL]])
+; CHECK-NEXT: call void @consume.i8.array(ptr [[TESTVAL]])
+; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds [[HASNOLIFETIMEEND_FRAME]], ptr [[VFRAME]], i32 0, i32 2
+; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR1]], align 1
+; CHECK-NEXT: ret void
+;
+entry:
+ %testval = alloca %i8.array
+ %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
+ %alloc = call ptr @malloc(i64 16) #3
+ %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
+
+ call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+ call void @consume.i8.array(ptr %testval)
+
+ %save = call token @llvm.coro.save(ptr null)
+ %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+ switch i8 %suspend, label %exit [
+ i8 0, label %await.ready
+ i8 1, label %exit
+ ]
+await.ready:
+ br label %exit
+exit:
+ call i1 @llvm.coro.end(ptr null, i1 false, token none)
+ ret void
+}
+
+define void @LifetimeEndAfterCoroEnd() presplitcoroutine {
+; CHECK-LABEL: define void @LifetimeEndAfterCoroEnd() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TESTVAL:%.*]] = alloca [[I8_ARRAY:%.*]], align 8
+; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr @LifetimeEndAfterCoroEnd.resumers)
+; CHECK-NEXT: [[ALLOC:%.*]] = call ptr @malloc(i64 16)
+; CHECK-NEXT: [[VFRAME:%.*]] = call noalias nonnull ptr @llvm.coro.begin(token [[ID]], ptr [[ALLOC]])
+; CHECK-NEXT: store ptr @LifetimeEndAfterCoroEnd.resume, ptr [[VFRAME]], align 8
+; CHECK-NEXT: [[DESTROY_ADDR:%.*]] = getelementptr inbounds [[LIFETIMEENDAFTERCOROEND_FRAME:%.*]], ptr [[VFRAME]], i32 0, i32 1
+; CHECK-NEXT: store ptr @LifetimeEndAfterCoroEnd.destroy, ptr [[DESTROY_ADDR]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 100, ptr [[TESTVAL]])
+; CHECK-NEXT: call void @consume.i8.array(ptr [[TESTVAL]])
+; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds [[LIFETIMEENDAFTERCOROEND_FRAME]], ptr [[VFRAME]], i32 0, i32 2
+; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR1]], align 1
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 100, ptr [[TESTVAL]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %testval = alloca %i8.array
+ %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
+ %alloc = call ptr @malloc(i64 16) #3
+ %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
+
+ call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+ call void @consume.i8.array(ptr %testval)
+
+ %save = call token @llvm.coro.save(ptr null)
+ %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+ switch i8 %suspend, label %exit [
+ i8 0, label %await.ready
+ i8 1, label %exit
+ ]
+await.ready:
+ br label %exit
+exit:
+ call i1 @llvm.coro.end(ptr null, i1 false, token none)
+ call void @llvm.lifetime.end.p0(i64 100, ptr %testval)
+ ret void
+}
+
+define void @BranchWithoutLifetimeEnd() presplitcoroutine {
+; CHECK-LABEL: define void @BranchWithoutLifetimeEnd() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TESTVAL:%.*]] = alloca [[I8_ARRAY:%.*]], align 8
+; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr @BranchWithoutLifetimeEnd.resumers)
+; CHECK-NEXT: [[ALLOC:%.*]] = call ptr @malloc(i64 16)
+; CHECK-NEXT: [[VFRAME:%.*]] = call noalias nonnull ptr @llvm.coro.begin(token [[ID]], ptr [[ALLOC]])
+; CHECK-NEXT: store ptr @BranchWithoutLifetimeEnd.resume, ptr [[VFRAME]], align 8
+; CHECK-NEXT: [[DESTROY_ADDR:%.*]] = getelementptr inbounds [[BRANCHWITHOUTLIFETIMEEND_FRAME:%.*]], ptr [[VFRAME]], i32 0, i32 1
+; CHECK-NEXT: store ptr @BranchWithoutLifetimeEnd.destroy, ptr [[DESTROY_ADDR]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 100, ptr [[TESTVAL]])
+; CHECK-NEXT: call void @consume.i8.array(ptr [[TESTVAL]])
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr @testbool, align 1
+; CHECK-NEXT: [[TOBOOL:%.*]] = trunc nuw i8 [[TMP0]] to i1
+; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[COROSAVE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 100, ptr [[TESTVAL]])
+; CHECK-NEXT: br label [[COROSAVE]]
+; CHECK: CoroSave:
+; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds [[BRANCHWITHOUTLIFETIMEEND_FRAME]], ptr [[VFRAME]], i32 0, i32 2
+; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR1]], align 1
+; CHECK-NEXT: ret void
+;
+entry:
+ %testval = alloca %i8.array
+ %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
+ %alloc = call ptr @malloc(i64 16) #3
+ %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
+
+ call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+ call void @consume.i8.array(ptr %testval)
+
+ %0 = load i8, ptr @testbool, align 1
+ %tobool = trunc nuw i8 %0 to i1
+ br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+ call void @llvm.lifetime.end.p0(i64 100, ptr %testval)
+ br label %if.end
+
+if.end:
+ %save = call token @llvm.coro.save(ptr null)
+ %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+ switch i8 %suspend, label %exit [
+ i8 0, label %await.ready
+ i8 1, label %exit
+ ]
+await.ready:
+ br label %exit
+exit:
+ call i1 @llvm.coro.end(ptr null, i1 false, token none)
+ ret void
+}
+
+
+declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr)
+declare ptr @llvm.coro.begin(token, ptr writeonly) #3
+declare ptr @llvm.coro.frame() #5
+declare i8 @llvm.coro.suspend(token, i1) #3
+declare i1 @llvm.coro.end(ptr, i1, token) #3
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
>From e31657cb587b15b6621142c09931e1750d46b5da Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Fri, 26 Apr 2024 11:33:52 -0700
Subject: [PATCH 2/5] [coro] Use `llvm.lifetime.end` to compute putting objects
on the frame vs the stack
The current logic for using lifetime intrinsics to determine whether a
coroutine alloca should live on the coroutine frame or stack doesn't
consider `llvm.lifetime.end`. As a result, some allocas are incorrectly
placed on the stack even though their lifetimes may outlive the stack.
For example, SimplifyCFG may generate code that drops the corresponding
`llvm.lifetime.end` of an `llvm.lifetime.start`, and that code is
incorrectly handled by the existing logic.
To fix this, new logic is introduced where if an alloca's address is
escaped, and there is a path from an `llvm.lifetime.start` to a
coroutine suspend point (e.g. `llvm.coro.suspend`) without an
`llvm.lifetime.end`, then we know the object lives beyond the suspension
point and therefore must go on the coroutine frame.
Fixes #86580
---
llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 35 ++++++++++++++++---
.../Coroutines/coro-lifetime-end.ll | 30 ++++++----------
2 files changed, 40 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 08a4522e3fac6d..81d25f9e617a90 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -19,6 +19,7 @@
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/PtrUseVisitor.h"
#include "llvm/Analysis/StackLifetime.h"
#include "llvm/Config/llvm-config.h"
@@ -1441,9 +1442,11 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
using Base = PtrUseVisitor<AllocaUseVisitor>;
AllocaUseVisitor(const DataLayout &DL, const DominatorTree &DT,
const CoroBeginInst &CB, const SuspendCrossingInfo &Checker,
- bool ShouldUseLifetimeStartInfo)
+ bool ShouldUseLifetimeStartInfo,
+ const coro::Shape &CoroShape)
: PtrUseVisitor(DL), DT(DT), CoroBegin(CB), Checker(Checker),
- ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {}
+ ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo),
+ CoroShape(CoroShape) {}
void visit(Instruction &I) {
Users.insert(&I);
@@ -1550,6 +1553,8 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
}
void visitIntrinsicInst(IntrinsicInst &II) {
+ if (II.getIntrinsicID() == Intrinsic::lifetime_end)
+ LifetimeEndBBs.insert(II.getParent());
// When we found the lifetime markers refers to a
// subrange of the original alloca, ignore the lifetime
// markers to avoid misleading the analysis.
@@ -1594,8 +1599,10 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
DenseMap<Instruction *, std::optional<APInt>> AliasOffetMap{};
SmallPtrSet<Instruction *, 4> Users{};
SmallPtrSet<IntrinsicInst *, 2> LifetimeStarts{};
+ SmallPtrSet<BasicBlock *, 2> LifetimeEndBBs;
bool MayWriteBeforeCoroBegin{false};
bool ShouldUseLifetimeStartInfo{true};
+ const coro::Shape &CoroShape;
mutable std::optional<bool> ShouldLiveOnFrame{};
@@ -1614,6 +1621,24 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
// suspend point between lifetime markers. This should also cover the
// case of a single lifetime.start intrinsic in a loop with suspend point.
if (PI.isEscaped()) {
+ // If there is no explicit lifetime.end, then assume the address can
+ // cross suspension points.
+ if (LifetimeEndBBs.empty())
+ return true;
+
+ // If there is a path from a lifetime.start to a suspend without a
+ // corresponding lifetime.end, then the alloca's lifetime persists
+ // beyond that suspension point and the alloca must go on the frame.
+ for (AnyCoroSuspendInst *SuspendInst : CoroShape.CoroSuspends) {
+ SmallVector<BasicBlock *> LifetimeStartsBB;
+ for (IntrinsicInst *II : LifetimeStarts)
+ LifetimeStartsBB.push_back(II->getParent());
+ if (isPotentiallyReachableFromMany(LifetimeStartsBB,
+ SuspendInst->getParent(),
+ &LifetimeEndBBs, &DT))
+ return true;
+ }
+
for (auto *A : LifetimeStarts) {
for (auto *B : LifetimeStarts) {
if (Checker.hasPathOrLoopCrossingSuspendPoint(A->getParent(),
@@ -2830,9 +2855,9 @@ static void collectFrameAlloca(AllocaInst *AI, coro::Shape &Shape,
bool ShouldUseLifetimeStartInfo =
(Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon &&
Shape.ABI != coro::ABI::RetconOnce);
- AllocaUseVisitor Visitor{AI->getModule()->getDataLayout(), DT,
- *Shape.CoroBegin, Checker,
- ShouldUseLifetimeStartInfo};
+ AllocaUseVisitor Visitor{
+ AI->getModule()->getDataLayout(), DT, *Shape.CoroBegin, Checker,
+ ShouldUseLifetimeStartInfo, Shape};
Visitor.visitPtr(*AI);
if (!Visitor.getShouldLiveOnFrame())
return;
diff --git a/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll b/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
index de861317436d6a..9a26a722a1e0db 100644
--- a/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
+++ b/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
@@ -13,17 +13,16 @@ declare void @consume.i8.array(ptr)
define void @HasNoLifetimeEnd() presplitcoroutine {
; CHECK-LABEL: define void @HasNoLifetimeEnd() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TESTVAL:%.*]] = alloca [[I8_ARRAY:%.*]], align 8
; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr @HasNoLifetimeEnd.resumers)
; CHECK-NEXT: [[ALLOC:%.*]] = call ptr @malloc(i64 16)
; CHECK-NEXT: [[VFRAME:%.*]] = call noalias nonnull ptr @llvm.coro.begin(token [[ID]], ptr [[ALLOC]])
; CHECK-NEXT: store ptr @HasNoLifetimeEnd.resume, ptr [[VFRAME]], align 8
; CHECK-NEXT: [[DESTROY_ADDR:%.*]] = getelementptr inbounds [[HASNOLIFETIMEEND_FRAME:%.*]], ptr [[VFRAME]], i32 0, i32 1
; CHECK-NEXT: store ptr @HasNoLifetimeEnd.destroy, ptr [[DESTROY_ADDR]], align 8
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 100, ptr [[TESTVAL]])
-; CHECK-NEXT: call void @consume.i8.array(ptr [[TESTVAL]])
; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds [[HASNOLIFETIMEEND_FRAME]], ptr [[VFRAME]], i32 0, i32 2
-; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR1]], align 1
+; CHECK-NEXT: call void @consume.i8.array(ptr [[INDEX_ADDR1]])
+; CHECK-NEXT: [[INDEX_ADDR2:%.*]] = getelementptr inbounds [[HASNOLIFETIMEEND_FRAME]], ptr [[VFRAME]], i32 0, i32 3
+; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR2]], align 1
; CHECK-NEXT: ret void
;
entry:
@@ -51,18 +50,16 @@ exit:
define void @LifetimeEndAfterCoroEnd() presplitcoroutine {
; CHECK-LABEL: define void @LifetimeEndAfterCoroEnd() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TESTVAL:%.*]] = alloca [[I8_ARRAY:%.*]], align 8
; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr @LifetimeEndAfterCoroEnd.resumers)
; CHECK-NEXT: [[ALLOC:%.*]] = call ptr @malloc(i64 16)
; CHECK-NEXT: [[VFRAME:%.*]] = call noalias nonnull ptr @llvm.coro.begin(token [[ID]], ptr [[ALLOC]])
; CHECK-NEXT: store ptr @LifetimeEndAfterCoroEnd.resume, ptr [[VFRAME]], align 8
; CHECK-NEXT: [[DESTROY_ADDR:%.*]] = getelementptr inbounds [[LIFETIMEENDAFTERCOROEND_FRAME:%.*]], ptr [[VFRAME]], i32 0, i32 1
; CHECK-NEXT: store ptr @LifetimeEndAfterCoroEnd.destroy, ptr [[DESTROY_ADDR]], align 8
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 100, ptr [[TESTVAL]])
-; CHECK-NEXT: call void @consume.i8.array(ptr [[TESTVAL]])
; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds [[LIFETIMEENDAFTERCOROEND_FRAME]], ptr [[VFRAME]], i32 0, i32 2
-; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR1]], align 1
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 100, ptr [[TESTVAL]])
+; CHECK-NEXT: call void @consume.i8.array(ptr [[INDEX_ADDR1]])
+; CHECK-NEXT: [[INDEX_ADDR2:%.*]] = getelementptr inbounds [[LIFETIMEENDAFTERCOROEND_FRAME]], ptr [[VFRAME]], i32 0, i32 3
+; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR2]], align 1
; CHECK-NEXT: ret void
;
entry:
@@ -91,23 +88,16 @@ exit:
define void @BranchWithoutLifetimeEnd() presplitcoroutine {
; CHECK-LABEL: define void @BranchWithoutLifetimeEnd() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TESTVAL:%.*]] = alloca [[I8_ARRAY:%.*]], align 8
; CHECK-NEXT: [[ID:%.*]] = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr @BranchWithoutLifetimeEnd.resumers)
; CHECK-NEXT: [[ALLOC:%.*]] = call ptr @malloc(i64 16)
; CHECK-NEXT: [[VFRAME:%.*]] = call noalias nonnull ptr @llvm.coro.begin(token [[ID]], ptr [[ALLOC]])
; CHECK-NEXT: store ptr @BranchWithoutLifetimeEnd.resume, ptr [[VFRAME]], align 8
; CHECK-NEXT: [[DESTROY_ADDR:%.*]] = getelementptr inbounds [[BRANCHWITHOUTLIFETIMEEND_FRAME:%.*]], ptr [[VFRAME]], i32 0, i32 1
; CHECK-NEXT: store ptr @BranchWithoutLifetimeEnd.destroy, ptr [[DESTROY_ADDR]], align 8
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 100, ptr [[TESTVAL]])
+; CHECK-NEXT: [[TESTVAL:%.*]] = getelementptr inbounds [[BRANCHWITHOUTLIFETIMEEND_FRAME]], ptr [[VFRAME]], i32 0, i32 2
; CHECK-NEXT: call void @consume.i8.array(ptr [[TESTVAL]])
; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr @testbool, align 1
-; CHECK-NEXT: [[TOBOOL:%.*]] = trunc nuw i8 [[TMP0]] to i1
-; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[COROSAVE:%.*]]
-; CHECK: if.then:
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 100, ptr [[TESTVAL]])
-; CHECK-NEXT: br label [[COROSAVE]]
-; CHECK: CoroSave:
-; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds [[BRANCHWITHOUTLIFETIMEEND_FRAME]], ptr [[VFRAME]], i32 0, i32 2
+; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds [[BRANCHWITHOUTLIFETIMEEND_FRAME]], ptr [[VFRAME]], i32 0, i32 3
; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR1]], align 1
; CHECK-NEXT: ret void
;
@@ -132,8 +122,8 @@ if.end:
%save = call token @llvm.coro.save(ptr null)
%suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
switch i8 %suspend, label %exit [
- i8 0, label %await.ready
- i8 1, label %exit
+ i8 0, label %await.ready
+ i8 1, label %exit
]
await.ready:
br label %exit
>From baaf930e08ea8850f8f9eac2e3bfef4d6d881d7d Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Mon, 29 Apr 2024 15:41:13 -0700
Subject: [PATCH 3/5] remove Shape.CoroBegin parameter and class member
---
llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 28 +++++++++-----------
1 file changed, 13 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 81d25f9e617a90..d5144632bef33f 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1441,19 +1441,19 @@ namespace {
struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
using Base = PtrUseVisitor<AllocaUseVisitor>;
AllocaUseVisitor(const DataLayout &DL, const DominatorTree &DT,
- const CoroBeginInst &CB, const SuspendCrossingInfo &Checker,
- bool ShouldUseLifetimeStartInfo,
- const coro::Shape &CoroShape)
- : PtrUseVisitor(DL), DT(DT), CoroBegin(CB), Checker(Checker),
- ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo),
- CoroShape(CoroShape) {}
+ const coro::Shape &CoroShape,
+ const SuspendCrossingInfo &Checker,
+ bool ShouldUseLifetimeStartInfo)
+ : PtrUseVisitor(DL), DT(DT), CoroShape(CoroShape), Checker(Checker),
+ ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {}
void visit(Instruction &I) {
Users.insert(&I);
Base::visit(I);
// If the pointer is escaped prior to CoroBegin, we have to assume it would
// be written into before CoroBegin as well.
- if (PI.isEscaped() && !DT.dominates(&CoroBegin, PI.getEscapingInst())) {
+ if (PI.isEscaped() &&
+ !DT.dominates(CoroShape.CoroBegin, PI.getEscapingInst())) {
MayWriteBeforeCoroBegin = true;
}
}
@@ -1591,7 +1591,7 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
private:
const DominatorTree &DT;
- const CoroBeginInst &CoroBegin;
+ const coro::Shape &CoroShape;
const SuspendCrossingInfo &Checker;
// All alias to the original AllocaInst, created before CoroBegin and used
// after CoroBegin. Each entry contains the instruction and the offset in the
@@ -1602,7 +1602,6 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
SmallPtrSet<BasicBlock *, 2> LifetimeEndBBs;
bool MayWriteBeforeCoroBegin{false};
bool ShouldUseLifetimeStartInfo{true};
- const coro::Shape &CoroShape;
mutable std::optional<bool> ShouldLiveOnFrame{};
@@ -1671,13 +1670,13 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
}
void handleMayWrite(const Instruction &I) {
- if (!DT.dominates(&CoroBegin, &I))
+ if (!DT.dominates(CoroShape.CoroBegin, &I))
MayWriteBeforeCoroBegin = true;
}
bool usedAfterCoroBegin(Instruction &I) {
for (auto &U : I.uses())
- if (DT.dominates(&CoroBegin, U))
+ if (DT.dominates(CoroShape.CoroBegin, U))
return true;
return false;
}
@@ -1686,7 +1685,7 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
// We track all aliases created prior to CoroBegin but used after.
// These aliases may need to be recreated after CoroBegin if the alloca
// need to live on the frame.
- if (DT.dominates(&CoroBegin, &I) || !usedAfterCoroBegin(I))
+ if (DT.dominates(CoroShape.CoroBegin, &I) || !usedAfterCoroBegin(I))
return;
if (!IsOffsetKnown) {
@@ -2855,9 +2854,8 @@ static void collectFrameAlloca(AllocaInst *AI, coro::Shape &Shape,
bool ShouldUseLifetimeStartInfo =
(Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon &&
Shape.ABI != coro::ABI::RetconOnce);
- AllocaUseVisitor Visitor{
- AI->getModule()->getDataLayout(), DT, *Shape.CoroBegin, Checker,
- ShouldUseLifetimeStartInfo, Shape};
+ AllocaUseVisitor Visitor{AI->getModule()->getDataLayout(), DT, Shape, Checker,
+ ShouldUseLifetimeStartInfo};
Visitor.visitPtr(*AI);
if (!Visitor.getShouldLiveOnFrame())
return;
>From a5aff0d0a9e3c5f37b62e7336f43eee415c8f739 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Mon, 29 Apr 2024 16:19:36 -0700
Subject: [PATCH 4/5] move lifetime end logic to before PI.isEscaped to replace
existing logic with users
---
llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 40 +++++++++-----------
1 file changed, 18 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index d5144632bef33f..93a1e2e02c35a2 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1611,33 +1611,29 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
// every basic block that uses the pointer to see if they cross suspension
// points. The uses cover both direct uses as well as indirect uses.
if (ShouldUseLifetimeStartInfo && !LifetimeStarts.empty()) {
- for (auto *I : Users)
- for (auto *S : LifetimeStarts)
- if (Checker.isDefinitionAcrossSuspend(*S, I))
- return true;
+ // If there is no explicit lifetime.end, then assume the address can
+ // cross suspension points.
+ if (LifetimeEndBBs.empty())
+ return true;
+
+ // If there is a path from a lifetime.start to a suspend without a
+ // corresponding lifetime.end, then the alloca's lifetime persists
+ // beyond that suspension point and the alloca must go on the frame.
+ for (AnyCoroSuspendInst *SuspendInst : CoroShape.CoroSuspends) {
+ SmallVector<BasicBlock *> LifetimeStartsBB;
+ for (IntrinsicInst *II : LifetimeStarts)
+ LifetimeStartsBB.push_back(II->getParent());
+ if (isPotentiallyReachableFromMany(LifetimeStartsBB,
+ SuspendInst->getParent(),
+ &LifetimeEndBBs, &DT))
+ return true;
+ }
+
// Addresses are guaranteed to be identical after every lifetime.start so
// we cannot use the local stack if the address escaped and there is a
// suspend point between lifetime markers. This should also cover the
// case of a single lifetime.start intrinsic in a loop with suspend point.
if (PI.isEscaped()) {
- // If there is no explicit lifetime.end, then assume the address can
- // cross suspension points.
- if (LifetimeEndBBs.empty())
- return true;
-
- // If there is a path from a lifetime.start to a suspend without a
- // corresponding lifetime.end, then the alloca's lifetime persists
- // beyond that suspension point and the alloca must go on the frame.
- for (AnyCoroSuspendInst *SuspendInst : CoroShape.CoroSuspends) {
- SmallVector<BasicBlock *> LifetimeStartsBB;
- for (IntrinsicInst *II : LifetimeStarts)
- LifetimeStartsBB.push_back(II->getParent());
- if (isPotentiallyReachableFromMany(LifetimeStartsBB,
- SuspendInst->getParent(),
- &LifetimeEndBBs, &DT))
- return true;
- }
-
for (auto *A : LifetimeStarts) {
for (auto *B : LifetimeStarts) {
if (Checker.hasPathOrLoopCrossingSuspendPoint(A->getParent(),
>From ab6ed87c84eaf3f0575585b445c40747e528d9c5 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Tue, 30 Apr 2024 17:07:46 -0700
Subject: [PATCH 5/5] implement and use isManyPotentiallyReachableFromMany(...)
---
llvm/include/llvm/Analysis/CFG.h | 16 ++++++++++
llvm/lib/Analysis/CFG.cpp | 32 +++++++++++++++-----
llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 23 +++++++-------
3 files changed, 53 insertions(+), 18 deletions(-)
diff --git a/llvm/include/llvm/Analysis/CFG.h b/llvm/include/llvm/Analysis/CFG.h
index 86b01c13274fec..6ac9ef4e4a6490 100644
--- a/llvm/include/llvm/Analysis/CFG.h
+++ b/llvm/include/llvm/Analysis/CFG.h
@@ -96,6 +96,22 @@ bool isPotentiallyReachableFromMany(
const SmallPtrSetImpl<BasicBlock *> *ExclusionSet,
const DominatorTree *DT = nullptr, const LoopInfo *LI = nullptr);
+/// Determine whether there is at least one path from a block in
+/// 'Worklist' to a block in 'StopSet' without passing through any
+/// blocks in 'ExclusionSet', returning true if uncertain.
+///
+/// Determine whether there is a path from at least one block in Worklist to
+/// at least one block in StopSet within a single function without passing
+/// through any of the blocks in 'ExclusionSet'. Returns false only if we can
+/// prove that once any block 'Worklist' has been reached then no blocks in
+/// 'StopSet' can be executed.
+/// Conservatively returns true.
+bool isManyPotentiallyReachableFromMany(
+ SmallVectorImpl<BasicBlock *> &Worklist,
+ const SmallPtrSetImpl<const BasicBlock *> &StopSet,
+ const SmallPtrSetImpl<BasicBlock *> *ExclusionSet,
+ const DominatorTree *DT = nullptr, const LoopInfo *LI = nullptr);
+
/// Return true if the control flow in \p RPOTraversal is irreducible.
///
/// This is a generic implementation to detect CFG irreducibility based on loop
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index 8528aa9f77e022..556326406e7ad2 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -134,10 +134,21 @@ bool llvm::isPotentiallyReachableFromMany(
SmallVectorImpl<BasicBlock *> &Worklist, const BasicBlock *StopBB,
const SmallPtrSetImpl<BasicBlock *> *ExclusionSet, const DominatorTree *DT,
const LoopInfo *LI) {
- // When the stop block is unreachable, it's dominated from everywhere,
+ return isManyPotentiallyReachableFromMany(
+ Worklist, llvm::SmallPtrSet<const BasicBlock *, 1>{StopBB}, ExclusionSet,
+ DT, LI);
+}
+
+bool llvm::isManyPotentiallyReachableFromMany(
+ SmallVectorImpl<BasicBlock *> &Worklist,
+ const SmallPtrSetImpl<const BasicBlock *> &StopSet,
+ const SmallPtrSetImpl<BasicBlock *> *ExclusionSet, const DominatorTree *DT,
+ const LoopInfo *LI) {
+ // When a stop block is unreachable, it's dominated from everywhere,
// regardless of whether there's a path between the two blocks.
- if (DT && !DT->isReachableFromEntry(StopBB))
- DT = nullptr;
+ llvm::DenseMap<const BasicBlock *, bool> StopBBReachable;
+ for (auto *BB : StopSet)
+ StopBBReachable[BB] = DT && DT->isReachableFromEntry(BB);
// We can't skip directly from a block that dominates the stop block if the
// exclusion block is potentially in between.
@@ -155,7 +166,9 @@ bool llvm::isPotentiallyReachableFromMany(
}
}
- const Loop *StopLoop = LI ? getOutermostLoop(LI, StopBB) : nullptr;
+ llvm::DenseMap<const BasicBlock *, const Loop *> StopLoops;
+ for (auto *StopBB : StopSet)
+ StopLoops[StopBB] = LI ? getOutermostLoop(LI, StopBB) : nullptr;
unsigned Limit = DefaultMaxBBsToExplore;
SmallPtrSet<const BasicBlock*, 32> Visited;
@@ -163,11 +176,13 @@ bool llvm::isPotentiallyReachableFromMany(
BasicBlock *BB = Worklist.pop_back_val();
if (!Visited.insert(BB).second)
continue;
- if (BB == StopBB)
+ if (StopSet.contains(BB))
return true;
if (ExclusionSet && ExclusionSet->count(BB))
continue;
- if (DT && DT->dominates(BB, StopBB))
+ if (DT && llvm::any_of(StopSet, [&](const BasicBlock *StopBB) {
+ return StopBBReachable[BB] && DT->dominates(BB, StopBB);
+ }))
return true;
const Loop *Outer = nullptr;
@@ -179,7 +194,10 @@ bool llvm::isPotentiallyReachableFromMany(
// excluded block. Clear Outer so we process BB's successors.
if (LoopsWithHoles.count(Outer))
Outer = nullptr;
- if (StopLoop && Outer == StopLoop)
+ if (llvm::any_of(StopSet, [&](const BasicBlock *StopBB) {
+ const Loop *StopLoop = StopLoops[StopBB];
+ return StopLoop && StopLoop == Outer;
+ }))
return true;
}
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 93a1e2e02c35a2..7c05ed1552b514 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1445,7 +1445,10 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
const SuspendCrossingInfo &Checker,
bool ShouldUseLifetimeStartInfo)
: PtrUseVisitor(DL), DT(DT), CoroShape(CoroShape), Checker(Checker),
- ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {}
+ ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {
+ for (AnyCoroSuspendInst *SuspendInst : CoroShape.CoroSuspends)
+ CoroSuspendBBs.insert(SuspendInst->getParent());
+ }
void visit(Instruction &I) {
Users.insert(&I);
@@ -1562,6 +1565,7 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
!Offset.isZero())
return Base::visitIntrinsicInst(II);
LifetimeStarts.insert(&II);
+ LifetimeStartBBs.push_back(II.getParent());
}
void visitCallBase(CallBase &CB) {
@@ -1599,7 +1603,9 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
DenseMap<Instruction *, std::optional<APInt>> AliasOffetMap{};
SmallPtrSet<Instruction *, 4> Users{};
SmallPtrSet<IntrinsicInst *, 2> LifetimeStarts{};
- SmallPtrSet<BasicBlock *, 2> LifetimeEndBBs;
+ SmallVector<BasicBlock *> LifetimeStartBBs{};
+ SmallPtrSet<BasicBlock *, 2> LifetimeEndBBs{};
+ SmallPtrSet<const BasicBlock *, 2> CoroSuspendBBs{};
bool MayWriteBeforeCoroBegin{false};
bool ShouldUseLifetimeStartInfo{true};
@@ -1619,15 +1625,10 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
// If there is a path from a lifetime.start to a suspend without a
// corresponding lifetime.end, then the alloca's lifetime persists
// beyond that suspension point and the alloca must go on the frame.
- for (AnyCoroSuspendInst *SuspendInst : CoroShape.CoroSuspends) {
- SmallVector<BasicBlock *> LifetimeStartsBB;
- for (IntrinsicInst *II : LifetimeStarts)
- LifetimeStartsBB.push_back(II->getParent());
- if (isPotentiallyReachableFromMany(LifetimeStartsBB,
- SuspendInst->getParent(),
- &LifetimeEndBBs, &DT))
- return true;
- }
+ llvm::SmallVector<BasicBlock *> Worklist(LifetimeStartBBs);
+ if (isManyPotentiallyReachableFromMany(Worklist, CoroSuspendBBs,
+ &LifetimeEndBBs, &DT))
+ return true;
// Addresses are guaranteed to be identical after every lifetime.start so
// we cannot use the local stack if the address escaped and there is a
More information about the llvm-commits
mailing list