[llvm] 094190c - [llvm][CallBrPrepare] add llvm.callbr.landingpad intrinsic
Nick Desaulniers via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 16 18:03:58 PST 2023
Author: Nick Desaulniers
Date: 2023-02-16T17:58:33-08:00
New Revision: 094190c2f52900d9d5d26dba9522e70030cc00a1
URL: https://github.com/llvm/llvm-project/commit/094190c2f52900d9d5d26dba9522e70030cc00a1
DIFF: https://github.com/llvm/llvm-project/commit/094190c2f52900d9d5d26dba9522e70030cc00a1.diff
LOG: [llvm][CallBrPrepare] add llvm.callbr.landingpad intrinsic
Insert a new intrinsic call after splitting critical edges, and verify
it. Later commits will update the SSA values to use this new value along
indirect branches rather than the callbr's value, and have SelectionDAG
consume this new value.
Part 2b of
https://discourse.llvm.org/t/rfc-syncing-asm-goto-with-outputs-with-gcc/65453/8.
Reviewed By: efriedma, jyknight
Differential Revision: https://reviews.llvm.org/D139883
Added:
Modified:
llvm/include/llvm/IR/Intrinsics.td
llvm/lib/CodeGen/CallBrPrepare.cpp
llvm/lib/IR/Verifier.cpp
llvm/test/CodeGen/AArch64/callbr-prepare.ll
llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll
llvm/test/Verifier/callbr.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index e0fd727607ce6..6090dee6f769e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -620,6 +620,11 @@ def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i
def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>;
def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>;
+// This intrinsic is intentionally undocumented and users shouldn't call it;
+// it's produced then quickly consumed during codegen.
+def int_callbr_landingpad : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>],
+ [IntrNoMerge]>;
+
//===------------------- Standard C Library Intrinsics --------------------===//
//
diff --git a/llvm/lib/CodeGen/CallBrPrepare.cpp b/llvm/lib/CodeGen/CallBrPrepare.cpp
index b4eb0b1028840..39192d0ad37c6 100644
--- a/llvm/lib/CodeGen/CallBrPrepare.cpp
+++ b/llvm/lib/CodeGen/CallBrPrepare.cpp
@@ -32,13 +32,17 @@
//===----------------------------------------------------------------------===//
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -104,6 +108,23 @@ bool CallBrPrepare::SplitCriticalEdges(ArrayRef<CallBrInst *> CBRs,
return Changed;
}
+static bool InsertIntrinsicCalls(ArrayRef<CallBrInst *> CBRs) {
+ bool Changed = false;
+ SmallPtrSet<const BasicBlock *, 4> Visited;
+ IRBuilder<> Builder(CBRs[0]->getContext());
+ for (CallBrInst *CBR : CBRs) {
+ for (BasicBlock *IndDest : CBR->getIndirectDests()) {
+ if (!Visited.insert(IndDest).second)
+ continue;
+ Builder.SetInsertPoint(&*IndDest->begin());
+ Builder.CreateIntrinsic(CBR->getType(), Intrinsic::callbr_landingpad,
+ {CBR});
+ Changed = true;
+ }
+ }
+ return Changed;
+}
+
bool CallBrPrepare::runOnFunction(Function &Fn) {
bool Changed = false;
SmallVector<CallBrInst *, 2> CBRs = FindCallBrs(Fn);
@@ -130,5 +151,8 @@ bool CallBrPrepare::runOnFunction(Function &Fn) {
if (SplitCriticalEdges(CBRs, *DT))
Changed = true;
+ if (InsertIntrinsicCalls(CBRs))
+ Changed = true;
+
return Changed;
}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index c1bb196bd49a0..40968a262811c 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5824,6 +5824,35 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"isdata argument to llvm.aarch64.prefetch must be 0 or 1", Call);
break;
}
+ case Intrinsic::callbr_landingpad: {
+ const auto *CBR = dyn_cast<CallBrInst>(Call.getOperand(0));
+ Check(CBR, "intrinstic requires callbr operand", &Call);
+ if (!CBR)
+ break;
+
+ const BasicBlock *LandingPadBB = Call.getParent();
+ const BasicBlock *PredBB = LandingPadBB->getUniquePredecessor();
+ if (!PredBB) {
+ CheckFailed("Intrinsic in block must have 1 unique predecessor", &Call);
+ break;
+ }
+ if (!isa<CallBrInst>(PredBB->getTerminator())) {
+ CheckFailed("Intrinsic must have corresponding callbr in predecessor",
+ &Call);
+ break;
+ }
+ Check(llvm::any_of(CBR->getIndirectDests(),
+ [LandingPadBB](const BasicBlock *IndDest) {
+ return IndDest == LandingPadBB;
+ }),
+ "Intrinsic's corresponding callbr must have intrinsic's parent basic "
+ "block in indirect destination list",
+ &Call);
+ const Instruction &First = *LandingPadBB->begin();
+ Check(&First == &Call, "No other instructions may proceed intrinsic",
+ &Call);
+ break;
+ }
};
// Verify that there aren't any unmediated control transfers between funclets.
diff --git a/llvm/test/CodeGen/AArch64/callbr-prepare.ll b/llvm/test/CodeGen/AArch64/callbr-prepare.ll
index 383e12bb24ac4..b1faf186aeff4 100644
--- a/llvm/test/CodeGen/AArch64/callbr-prepare.ll
+++ b/llvm/test/CodeGen/AArch64/callbr-prepare.ll
@@ -7,11 +7,13 @@ define i32 @test0() {
; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"()
; CHECK-NEXT: to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge]
; CHECK: entry.indirect_crit_edge:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT]])
; CHECK-NEXT: br label [[INDIRECT:%.*]]
; CHECK: direct:
; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"()
; CHECK-NEXT: to label [[DIRECT2:%.*]] [label %direct.indirect_crit_edge]
; CHECK: direct.indirect_crit_edge:
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT2]])
; CHECK-NEXT: br label [[INDIRECT]]
; CHECK: direct2:
; CHECK-NEXT: ret i32 0
@@ -67,6 +69,7 @@ define i32 @dont_split1() {
; CHECK: x:
; CHECK-NEXT: ret i32 42
; CHECK: y:
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
; CHECK-NEXT: ret i32 [[TMP0]]
;
entry:
@@ -138,12 +141,13 @@ define i32 @split_me0() {
; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i"()
; CHECK-NEXT: to label [[X:%.*]] [label %entry.y_crit_edge]
; CHECK: entry.y_crit_edge:
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
; CHECK-NEXT: br label [[Y:%.*]]
; CHECK: x:
; CHECK-NEXT: br label [[Y]]
; CHECK: y:
-; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ]
-; CHECK-NEXT: ret i32 [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ]
+; CHECK-NEXT: ret i32 [[TMP2]]
;
entry:
%0 = callbr i32 asm "", "=r,!i"()
@@ -168,12 +172,13 @@ define i32 @split_me1(i1 %z) {
; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"()
; CHECK-NEXT: to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge]
; CHECK: w.v_crit_edge:
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
; CHECK-NEXT: br label [[V]]
; CHECK: x:
; CHECK-NEXT: ret i32 42
; CHECK: v:
-; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ]
-; CHECK-NEXT: ret i32 [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT: ret i32 [[TMP2]]
;
entry:
br i1 %z, label %w, label %v
@@ -200,12 +205,13 @@ define i32 @split_me2(i1 %z) {
; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"()
; CHECK-NEXT: to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge]
; CHECK: w.v_crit_edge:
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
; CHECK-NEXT: br label [[V]]
; CHECK: x:
; CHECK-NEXT: ret i32 42
; CHECK: v:
-; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ]
-; CHECK-NEXT: ret i32 [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ]
+; CHECK-NEXT: ret i32 [[TMP2]]
;
entry:
br i1 %z, label %w, label %v
diff --git a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll
index ec30d6b2347c7..edca02d95d875 100644
--- a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll
+++ b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s
+; RUN: opt < %s -S -passes="simplifycfg<sink-common-insts>" \
+; RUN: -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s
define void @callbr_duplicate_dest() {
; CHECK-LABEL: @callbr_duplicate_dest(
@@ -57,3 +58,49 @@ bb2:
bb3:
ret void
}
+
+; Validate that callbr landingpad intrinsics do not get merged (via the
+; IntrNoMerge attribute).
+define i32 @callbr_landingpad_nomerge() {
+; CHECK-LABEL: @callbr_landingpad_nomerge(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"()
+; CHECK-NEXT: to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge]
+; CHECK: entry.indirect_crit_edge:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT]])
+; CHECK-NEXT: br label [[COMMON_RET:%.*]]
+; CHECK: direct:
+; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"()
+; CHECK-NEXT: to label [[COMMON_RET]] [label %direct.indirect_crit_edge]
+; CHECK: direct.indirect_crit_edge:
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT2]])
+; CHECK-NEXT: br label [[COMMON_RET]]
+; CHECK: common.ret:
+; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[DIRECT]] ], [ [[TMP0]], [[ENTRY_INDIRECT_CRIT_EDGE:%.*]] ], [ [[TMP1]], [[DIRECT_INDIRECT_CRIT_EDGE:%.*]] ]
+; CHECK-NEXT: ret i32 [[COMMON_RET_OP]]
+;
+entry:
+ %out = callbr i32 asm "# $0", "=r,!i"()
+ to label %direct [label %entry.indirect_crit_edge]
+
+entry.indirect_crit_edge:
+ %0 = call i32 @llvm.callbr.landingpad.i32(i32 %out)
+ br label %indirect
+
+direct:
+ %out2 = callbr i32 asm "# $0", "=r,!i"()
+ to label %direct2 [label %direct.indirect_crit_edge]
+
+direct.indirect_crit_edge:
+ %1 = call i32 @llvm.callbr.landingpad.i32(i32 %out2)
+ br label %indirect
+
+direct2:
+ ret i32 0
+
+indirect:
+ %out3 = phi i32 [ %0, %entry.indirect_crit_edge ], [ %1, %direct.indirect_crit_edge ]
+ ret i32 %out3
+}
+
+declare i32 @llvm.callbr.landingpad.i32(i32)
diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll
index a77b6fee037ab..9b819c5fed48b 100644
--- a/llvm/test/Verifier/callbr.ll
+++ b/llvm/test/Verifier/callbr.ll
@@ -68,3 +68,55 @@ normal:
abnormal:
ret i32 %ret
}
+
+;; Tests of the callbr.landingpad intrinsic function.
+declare i32 @llvm.callbr.landingpad.i64(i64)
+define void @callbrpad_bad_type() {
+entry:
+; CHECK: Intrinsic has incorrect argument type!
+; CHECK-NEXT: ptr @llvm.callbr.landingpad.i64
+ %foo = call i32 @llvm.callbr.landingpad.i64(i64 42)
+ ret void
+}
+
+declare i32 @llvm.callbr.landingpad.i32(i32)
+define i32 @callbrpad_multi_preds() {
+entry:
+ %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect]
+direct:
+ br label %indirect
+indirect:
+; CHECK-NEXT: Intrinsic in block must have 1 unique predecessor
+; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+ %out = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+ ret i32 %out
+}
+
+define void @callbrpad_wrong_callbr() {
+entry:
+ %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect]
+direct:
+; CHECK-NEXT: Intrinsic's corresponding callbr must have intrinsic's parent basic block in indirect destination list
+; CHECK-NEXT: %x = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+ %x = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+ ret void
+indirect:
+ ret void
+}
+
+declare i32 @foo(i32)
+define i32 @test_callbr_landingpad_not_first_inst() {
+entry:
+ %0 = callbr i32 asm "", "=r,!i"()
+ to label %asm.fallthrough [label %landingpad]
+
+asm.fallthrough:
+ ret i32 42
+
+landingpad:
+ %foo = call i32 @foo(i32 42)
+; CHECK-NEXT: No other instructions may proceed intrinsic
+; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
+ %out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
+ ret i32 %out
+}
More information about the llvm-commits
mailing list