[llvm] [IR] Add CallBr intrinsics support (PR #133907)
Robert Imschweiler via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 2 03:04:42 PDT 2025
https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/133907
>From 345da0bb18e151f71b570becf49d2e750102f76b Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Tue, 1 Apr 2025 08:03:16 -0500
Subject: [PATCH 1/4] [IR] Add CallBr intrinsics support
This commit adds support for using intrinsics with callbr.
The uses of this will most of the time look like this example:
```llvm
callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
kill:
unreachable
cont:
...
```
---
llvm/include/llvm/Analysis/RegionInfoImpl.h | 12 ++++
.../llvm/Transforms/Utils/BasicBlockUtils.h | 6 +-
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 59 +++++++++++++++-
.../SelectionDAG/SelectionDAGBuilder.cpp | 45 +++++++++---
llvm/lib/IR/Verifier.cpp | 29 ++++++--
llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 11 ++-
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 4 +-
llvm/test/CodeGen/AMDGPU/callbr.ll | 70 +++++++++++++++++++
8 files changed, 207 insertions(+), 29 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/callbr.ll
diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h
index eb99d8bc6fb23..759e9c47bebb8 100644
--- a/llvm/include/llvm/Analysis/RegionInfoImpl.h
+++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h
@@ -553,6 +553,18 @@ bool RegionInfoBase<Tr>::isRegion(BlockT *entry, BlockT *exit) const {
using DST = typename DomFrontierT::DomSetType;
+ // TODO? post domination frontier?
+ if constexpr (std::is_same_v<BlockT, BasicBlock>) {
+ if (DomTreeNodeT *PDTNode = PDT->getNode(exit); PDTNode) {
+ for (DomTreeNodeT *PredNode : *PDTNode) {
+ for (BasicBlock *Pred : predecessors(PredNode->getBlock())) {
+ if (isa<CallBrInst>(Pred->getTerminator()))
+ return false;
+ }
+ }
+ }
+ }
+
DST *entrySuccs = &DF->find(entry)->second;
// Exit is the header of a loop that contains the entry. In this case,
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index 6faff3d1fd8e3..59143d235eb93 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -606,9 +606,9 @@ bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI,
// successors
void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder);
-// Check whether the function only has simple terminator:
-// br/brcond/unreachable/ret
-bool hasOnlySimpleTerminator(const Function &F);
+// Check whether the function only has blocks with simple terminators:
+// br/brcond/unreachable/ret (or callbr if AllowCallBr)
+bool hasOnlySimpleTerminator(const Function &F, bool AllowCallBr = true);
// Returns true if these basic blocks belong to a presplit coroutine and the
// edge corresponds to the 'default' case in the switch statement in the
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index f8afb42bf5535..47c825d67c3ea 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3009,8 +3009,63 @@ bool IRTranslator::translateInvoke(const User &U,
bool IRTranslator::translateCallBr(const User &U,
MachineIRBuilder &MIRBuilder) {
- // FIXME: Implement this.
- return false;
+ const CallBrInst &I = cast<CallBrInst>(U);
+ MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
+
+ // TODO: operand bundles (see SelDAG implementation of callbr)?
+ assert(!I.hasOperandBundles() &&
+ "Cannot lower callbrs with operand bundles yet");
+
+ if (I.isInlineAsm()) {
+ // FIXME: inline asm not yet supported
+ if (!translateInlineAsm(I, MIRBuilder))
+ return false;
+ } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
+ switch (I.getIntrinsicID()) {
+ default:
+ report_fatal_error("Unsupported intrinsic for callbr");
+ case Intrinsic::amdgcn_kill:
+ if (I.getNumIndirectDests() != 1)
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
+ CallInst *CI =
+ CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
+ SmallVector<Value *, 1>(I.args()));
+ bool Success = translateCall(*CI, MIRBuilder);
+ CI->deleteValue();
+ if (!Success)
+ return false;
+ break;
+ }
+ } else {
+ report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ }
+
+ // Retrieve successors.
+ SmallPtrSet<BasicBlock *, 8> Dests;
+ Dests.insert(I.getDefaultDest());
+ MachineBasicBlock *Return = &getMBB(*I.getDefaultDest());
+
+ // Update successor info.
+ addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
+ // TODO: For most of the cases where there is an intrinsic callbr, we're
+ // having exactly one indirect target, which will be unreachable. As soon as
+ // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
+ // or add something similar for intrinsic indirect branches.
+ if (I.isInlineAsm()) {
+ for (BasicBlock *Dest : I.getIndirectDests()) {
+ MachineBasicBlock *Target = &getMBB(*Dest);
+ Target->setIsInlineAsmBrIndirectTarget();
+ Target->setMachineBlockAddressTaken();
+ Target->setLabelMustBeEmitted();
+ // Don't add duplicate machine successors.
+ if (Dests.insert(Dest).second)
+ addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+ }
+ }
+ CallBrMBB->normalizeSuccProbs();
+
+ return true;
}
bool IRTranslator::translateLandingPad(const User &U,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 6db2a5ffbfb84..8d4f7c7e337a3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3385,8 +3385,26 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
{LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
"Cannot lower callbrs with arbitrary operand bundles yet!");
- assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
- visitInlineAsm(I);
+ if (I.isInlineAsm()) {
+ visitInlineAsm(I);
+ } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
+ switch (I.getIntrinsicID()) {
+ default:
+ report_fatal_error("Unsupported intrinsic for callbr");
+ case Intrinsic::amdgcn_kill:
+ if (I.getNumIndirectDests() != 1)
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
+ CallInst *CI =
+ CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
+ SmallVector<Value *, 1>(I.args()));
+ visitCall(*CI);
+ CI->deleteValue();
+ break;
+ }
+ } else {
+ report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ }
CopyToExportRegsIfNeeded(&I);
// Retrieve successors.
@@ -3396,15 +3414,20 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
// Update successor info.
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
- for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
- BasicBlock *Dest = I.getIndirectDest(i);
- MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
- Target->setIsInlineAsmBrIndirectTarget();
- Target->setMachineBlockAddressTaken();
- Target->setLabelMustBeEmitted();
- // Don't add duplicate machine successors.
- if (Dests.insert(Dest).second)
- addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+ // TODO: For most of the cases where there is an intrinsic callbr, we're
+ // having exactly one indirect target, which will be unreachable. As soon as
+ // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
+ // or add something similar for intrinsic indirect branches.
+ if (I.isInlineAsm()) {
+ for (BasicBlock *Dest : I.getIndirectDests()) {
+ MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
+ Target->setIsInlineAsmBrIndirectTarget();
+ Target->setMachineBlockAddressTaken();
+ Target->setLabelMustBeEmitted();
+ // Don't add duplicate machine successors.
+ if (Dests.insert(Dest).second)
+ addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+ }
}
CallBrMBB->normalizeSuccProbs();
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index ed86a10c3a25f..5f2d0fcea4dcd 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3249,11 +3249,30 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) {
}
void Verifier::visitCallBrInst(CallBrInst &CBI) {
- Check(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", &CBI);
- const InlineAsm *IA = cast<InlineAsm>(CBI.getCalledOperand());
- Check(!IA->canThrow(), "Unwinding from Callbr is not allowed");
+ if (!CBI.isInlineAsm()) {
+ switch (CBI.getIntrinsicID()) {
+ case Intrinsic::amdgcn_kill: {
+ Check(CBI.getNumIndirectDests() == 1,
+ "Callbr amdgcn_kill only supports one indirect dest");
+ bool Unreachable = isa<UnreachableInst>(CBI.getIndirectDest(0)->begin());
+ CallInst *Call = dyn_cast<CallInst>(CBI.getIndirectDest(0)->begin());
+ Check(Unreachable ||
+ (Call && Call->getIntrinsicID() == Intrinsic::amdgcn_unreachable),
+ "Callbr amdgcn_kill indirect dest needs to be unreachable");
+ visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI);
+ break;
+ }
+ default:
+ CheckFailed(
+ "Callbr currently only supports asm-goto and selected intrinsics");
+ }
+ visitIntrinsicCall(CBI.getIntrinsicID(), CBI);
+ } else {
+ const InlineAsm *IA = cast<InlineAsm>(CBI.getCalledOperand());
+ Check(!IA->canThrow(), "Unwinding from Callbr is not allowed");
- verifyInlineAsmCall(CBI);
+ verifyInlineAsmCall(CBI);
+ }
visitTerminator(CBI);
}
@@ -5211,7 +5230,7 @@ void Verifier::visitInstruction(Instruction &I) {
(CBI && &CBI->getCalledOperandUse() == &I.getOperandUse(i)) ||
IsAttachedCallOperand(F, CBI, i)),
"Cannot take the address of an intrinsic!", &I);
- Check(!F->isIntrinsic() || isa<CallInst>(I) ||
+ Check(!F->isIntrinsic() || isa<CallInst>(I) || isa<CallBrInst>(I) ||
F->getIntrinsicID() == Intrinsic::donothing ||
F->getIntrinsicID() == Intrinsic::seh_try_begin ||
F->getIntrinsicID() == Intrinsic::seh_try_end ||
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index d1054b9b045ca..bdd8b5fbb3212 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -486,11 +486,10 @@ void StructurizeCFG::analyzeLoops(RegionNode *N) {
} else {
// Test for successors as back edge
BasicBlock *BB = N->getNodeAs<BasicBlock>();
- BranchInst *Term = cast<BranchInst>(BB->getTerminator());
-
- for (BasicBlock *Succ : Term->successors())
- if (Visited.count(Succ))
- Loops[Succ] = BB;
+ if (BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator()); Term)
+ for (BasicBlock *Succ : Term->successors())
+ if (Visited.count(Succ))
+ Loops[Succ] = BB;
}
}
@@ -522,7 +521,7 @@ void StructurizeCFG::gatherPredicates(RegionNode *N) {
for (BasicBlock *P : predecessors(BB)) {
// Ignore it if it's a branch from outside into our region entry
- if (!ParentRegion->contains(P))
+ if (!ParentRegion->contains(P) || !dyn_cast<BranchInst>(P->getTerminator()))
continue;
Region *R = RI->getRegionFor(P);
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index ce5bf0c7207c7..3090f65fac627 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1907,11 +1907,11 @@ void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) {
PBI->swapSuccessors();
}
-bool llvm::hasOnlySimpleTerminator(const Function &F) {
+bool llvm::hasOnlySimpleTerminator(const Function &F, bool AllowCallBr) {
for (auto &BB : F) {
auto *Term = BB.getTerminator();
if (!(isa<ReturnInst>(Term) || isa<UnreachableInst>(Term) ||
- isa<BranchInst>(Term)))
+ isa<BranchInst>(Term) || (AllowCallBr && isa<CallBrInst>(Term))))
return false;
}
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll
new file mode 100644
index 0000000000000..e2e84dca96cbf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/callbr.ll
@@ -0,0 +1,70 @@
+; RUN: rm -rf %t && split-file %s %t
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-seldag.s < %t/with-callbr.ll
+; RUN: FileCheck --check-prefix=SELDAG %s < %t/with-callbr-seldag.s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-gisel.s -global-isel < %t/with-callbr.ll
+; RUN: FileCheck --check-prefix=GISEL %s < %t/with-callbr-gisel.s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-seldag.s < %t/without-callbr.ll
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-gisel.s -global-isel < %t/without-callbr.ll
+; RUN: diff %t/with-callbr-seldag.s %t/without-callbr-seldag.s
+; RUN: diff %t/with-callbr-gisel.s %t/without-callbr-gisel.s
+
+;--- with-callbr.ll
+
+; SELDAG-LABEL: test_kill:
+; SELDAG-NEXT: ; %bb.0:
+; SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: flat_load_dword v0, v[0:1]
+; SELDAG-NEXT: v_and_b32_e32 v1, 1, v4
+; SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; SELDAG-NEXT: s_mov_b64 s[4:5], exec
+; SELDAG-NEXT: s_andn2_b64 s[6:7], exec, vcc
+; SELDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; SELDAG-NEXT: s_cbranch_scc0 .LBB0_2
+; SELDAG-NEXT: ; %bb.1:
+; SELDAG-NEXT: s_and_b64 exec, exec, s[4:5]
+; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: flat_store_dword v[2:3], v0
+; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: s_setpc_b64 s[30:31]
+; SELDAG-NEXT: .LBB0_2:
+; SELDAG-NEXT: s_mov_b64 exec, 0
+; SELDAG-NEXT: s_endpgm
+
+; GISEL-LABEL: test_kill:
+; GISEL-NEXT: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: flat_load_dword v0, v[0:1]
+; GISEL-NEXT: v_and_b32_e32 v1, 1, v4
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: s_andn2_b64 s[6:7], exec, vcc
+; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GISEL-NEXT: s_cbranch_scc0 .LBB0_2
+; GISEL-NEXT: ; %bb.1:
+; GISEL-NEXT: s_and_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT: flat_store_dword v[2:3], v0
+; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB0_2:
+; GISEL-NEXT: s_mov_b64 exec, 0
+; GISEL-NEXT: s_endpgm
+
+define void @test_kill(ptr %src, ptr %dst, i1 %c) {
+ %a = load i32, ptr %src, align 4
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+kill:
+ unreachable
+cont:
+ store i32 %a, ptr %dst, align 4
+ ret void
+}
+
+;--- without-callbr.ll
+
+define void @test_kill(ptr %src, ptr %dst, i1 %c) {
+ %a = load i32, ptr %src, align 4
+ call void @llvm.amdgcn.kill(i1 %c)
+ store i32 %a, ptr %dst, align 4
+ ret void
+}
>From 4305f2cf33e4aeaba2008641ef68d4c38f60d00b Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Tue, 1 Apr 2025 08:07:44 -0500
Subject: [PATCH 2/4] fix formatting
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++--
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 9 +++++----
llvm/lib/IR/Verifier.cpp | 4 ++--
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 47c825d67c3ea..0f698375ad6cf 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3050,8 +3050,9 @@ bool IRTranslator::translateCallBr(const User &U,
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
// TODO: For most of the cases where there is an intrinsic callbr, we're
// having exactly one indirect target, which will be unreachable. As soon as
- // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
- // or add something similar for intrinsic indirect branches.
+ // this changes, we might need to enhance
+ // Target->setIsInlineAsmBrIndirectTarget or add something similar for
+ // intrinsic indirect branches.
if (I.isInlineAsm()) {
for (BasicBlock *Dest : I.getIndirectDests()) {
MachineBasicBlock *Target = &getMBB(*Dest);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 8d4f7c7e337a3..c9501128cd593 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3393,8 +3393,8 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
CallInst *CI =
CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
SmallVector<Value *, 1>(I.args()));
@@ -3416,8 +3416,9 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
// TODO: For most of the cases where there is an intrinsic callbr, we're
// having exactly one indirect target, which will be unreachable. As soon as
- // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
- // or add something similar for intrinsic indirect branches.
+ // this changes, we might need to enhance
+ // Target->setIsInlineAsmBrIndirectTarget or add something similar for
+ // intrinsic indirect branches.
if (I.isInlineAsm()) {
for (BasicBlock *Dest : I.getIndirectDests()) {
MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 5f2d0fcea4dcd..fbf6e087177c6 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3256,8 +3256,8 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) {
"Callbr amdgcn_kill only supports one indirect dest");
bool Unreachable = isa<UnreachableInst>(CBI.getIndirectDest(0)->begin());
CallInst *Call = dyn_cast<CallInst>(CBI.getIndirectDest(0)->begin());
- Check(Unreachable ||
- (Call && Call->getIntrinsicID() == Intrinsic::amdgcn_unreachable),
+ Check(Unreachable || (Call && Call->getIntrinsicID() ==
+ Intrinsic::amdgcn_unreachable),
"Callbr amdgcn_kill indirect dest needs to be unreachable");
visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI);
break;
>From db57ce26f1780f285db04481f22c56d8262d60ba Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 2 Apr 2025 04:26:22 -0500
Subject: [PATCH 3/4] implement feedback
---
llvm/include/llvm/CodeGen/Analysis.h | 2 +-
.../llvm/CodeGen/GlobalISel/IRTranslator.h | 18 ++---
llvm/include/llvm/CodeGen/SelectionDAG.h | 6 +-
llvm/include/llvm/CodeGen/TargetLowering.h | 6 +-
llvm/include/llvm/IR/DiagnosticInfo.h | 4 +-
llvm/include/llvm/IR/IntrinsicInst.h | 8 ++-
llvm/lib/CodeGen/Analysis.cpp | 2 +-
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 34 ++++-----
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 71 +++++++++----------
.../SelectionDAG/SelectionDAGBuilder.h | 61 ++++++++--------
.../SelectionDAG/StatepointLowering.cpp | 2 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 2 +-
llvm/lib/IR/DiagnosticInfo.cpp | 2 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 4 +-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +-
llvm/lib/Target/ARM/ARMISelLowering.cpp | 2 +-
llvm/lib/Target/ARM/ARMISelLowering.h | 3 +-
.../Target/Hexagon/HexagonISelLowering.cpp | 2 +-
llvm/lib/Target/Hexagon/HexagonISelLowering.h | 2 +-
.../LoongArch/LoongArchISelLowering.cpp | 2 +-
.../Target/LoongArch/LoongArchISelLowering.h | 2 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 7 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 2 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 7 +-
llvm/lib/Target/PowerPC/PPCISelLowering.h | 9 ++-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 2 +-
.../WebAssembly/WebAssemblyISelLowering.cpp | 2 +-
.../WebAssembly/WebAssemblyISelLowering.h | 2 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
llvm/lib/Target/X86/X86ISelLowering.h | 2 +-
36 files changed, 142 insertions(+), 150 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index 362cc30bbd06a..a4a604dd2c608 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -150,7 +150,7 @@ bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I,
/// Returns true if the parent of \p CI returns CI's first argument after
/// calling \p CI.
-bool funcReturnsFirstArgOfCall(const CallInst &CI);
+bool funcReturnsFirstArgOfCall(const CallBase &CI);
DenseMap<const MachineBasicBlock *, int>
getEHScopeMembership(const MachineFunction &MF);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 6fd05c8fddd5f..d3f731cf02be9 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -235,26 +235,26 @@ class IRTranslator : public MachineFunctionPass {
bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
/// Translate an LLVM string intrinsic (memcpy, memset, ...).
- bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
+ bool translateMemFunc(const CallBase &CI, MachineIRBuilder &MIRBuilder,
unsigned Opcode);
/// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
- bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
+ bool translateTrap(const CallBase &U, MachineIRBuilder &MIRBuilder,
unsigned Opcode);
// Translate @llvm.vector.interleave2 and
// @llvm.vector.deinterleave2 intrinsics for fixed-width vector
// types into vector shuffles.
- bool translateVectorInterleave2Intrinsic(const CallInst &CI,
+ bool translateVectorInterleave2Intrinsic(const CallBase &CI,
MachineIRBuilder &MIRBuilder);
- bool translateVectorDeinterleave2Intrinsic(const CallInst &CI,
+ bool translateVectorDeinterleave2Intrinsic(const CallBase &CI,
MachineIRBuilder &MIRBuilder);
void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
- bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
+ bool translateOverflowIntrinsic(const CallBase &CI, unsigned Op,
MachineIRBuilder &MIRBuilder);
- bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
+ bool translateFixedPointIntrinsic(unsigned Op, const CallBase &CI,
MachineIRBuilder &MIRBuilder);
/// Helper function for translateSimpleIntrinsic.
@@ -265,13 +265,13 @@ class IRTranslator : public MachineFunctionPass {
/// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
/// \return true if the translation succeeded.
- bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
+ bool translateSimpleIntrinsic(const CallBase &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI,
MachineIRBuilder &MIRBuilder);
- bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
+ bool translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
/// Returns the single livein physical register Arg was lowered to, if
@@ -588,7 +588,7 @@ class IRTranslator : public MachineFunctionPass {
return false;
}
- bool translateConvergenceControlIntrinsic(const CallInst &CI,
+ bool translateConvergenceControlIntrinsic(const CallBase &CI,
Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 15a2370e5d8b8..c690b4106900d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1205,7 +1205,7 @@ class SelectionDAG {
* the tail call optimization decision. */
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, const CallInst *CI,
+ bool AlwaysInline, const CallBase *CI,
std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
@@ -1217,7 +1217,7 @@ class SelectionDAG {
* the tail call optimization decision. */
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- const CallInst *CI, std::optional<bool> OverrideTailCall,
+ const CallBase *CI, std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1225,7 +1225,7 @@ class SelectionDAG {
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, const CallInst *CI,
+ bool AlwaysInline, const CallBase *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes());
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 58ac87206b9a6..fd7a5b1442d3f 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1238,7 +1238,7 @@ class TargetLoweringBase {
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and store the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
- virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
+ virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
MachineFunction &,
unsigned /*Intrinsic*/) const {
return false;
@@ -5149,9 +5149,9 @@ class TargetLowering : public TargetLoweringBase {
const AsmOperandInfo &OpInfo,
SelectionDAG &DAG) const;
- // Targets may override this function to collect operands from the CallInst
+ // Targets may override this function to collect operands from the CallBase
// and for example, lower them into the SelectionDAG operands.
- virtual void CollectTargetIntrinsicOperands(const CallInst &I,
+ virtual void CollectTargetIntrinsicOperands(const CallBase &I,
SmallVectorImpl<SDValue> &Ops,
SelectionDAG &DAG) const;
diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h
index 779c88993b71c..9b6998397b944 100644
--- a/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -38,7 +38,7 @@ namespace llvm {
class DiagnosticPrinter;
class DIFile;
class DISubprogram;
-class CallInst;
+class CallBase;
class Function;
class Instruction;
class InstructionCost;
@@ -1151,7 +1151,7 @@ class DiagnosticInfoSrcMgr : public DiagnosticInfo {
}
};
-void diagnoseDontCall(const CallInst &CI);
+void diagnoseDontCall(const CallBase &CI);
class DiagnosticInfoDontCall : public DiagnosticInfo {
StringRef CalleeName;
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 93750d6e3845e..7a760781d2b96 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -134,8 +134,14 @@ class IntrinsicInst : public CallInst {
return CF->isIntrinsic();
return false;
}
+ static bool classof(const CallBase *I) {
+ if (const Function *CF = I->getCalledFunction())
+ return CF->isIntrinsic();
+ return false;
+ }
static bool classof(const Value *V) {
- return isa<CallInst>(V) && classof(cast<CallInst>(V));
+ return (isa<CallInst>(V) && classof(cast<CallInst>(V))) ||
+ (isa<CallBase>(V) && classof(cast<CallBase>(V)));
}
};
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index e7b9417de8c9f..f950bb756fb2b 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -712,7 +712,7 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
return true;
}
-bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
+bool llvm::funcReturnsFirstArgOfCall(const CallBase &CI) {
const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
bool ReturnsFirstArg = false;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 0f698375ad6cf..9323966d9ada6 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1694,7 +1694,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
return true;
}
-bool IRTranslator::translateMemFunc(const CallInst &CI,
+bool IRTranslator::translateMemFunc(const CallBase &CI,
MachineIRBuilder &MIRBuilder,
unsigned Opcode) {
const Value *SrcPtr = CI.getArgOperand(1);
@@ -1785,7 +1785,7 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
return true;
}
-bool IRTranslator::translateTrap(const CallInst &CI,
+bool IRTranslator::translateTrap(const CallBase &CI,
MachineIRBuilder &MIRBuilder,
unsigned Opcode) {
StringRef TrapFuncName =
@@ -1812,7 +1812,7 @@ bool IRTranslator::translateTrap(const CallInst &CI,
}
bool IRTranslator::translateVectorInterleave2Intrinsic(
- const CallInst &CI, MachineIRBuilder &MIRBuilder) {
+ const CallBase &CI, MachineIRBuilder &MIRBuilder) {
assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
"This function can only be called on the interleave2 intrinsic!");
// Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
@@ -1828,7 +1828,7 @@ bool IRTranslator::translateVectorInterleave2Intrinsic(
}
bool IRTranslator::translateVectorDeinterleave2Intrinsic(
- const CallInst &CI, MachineIRBuilder &MIRBuilder) {
+ const CallBase &CI, MachineIRBuilder &MIRBuilder) {
assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
"This function can only be called on the deinterleave2 intrinsic!");
// Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
@@ -1868,7 +1868,7 @@ void IRTranslator::getStackGuard(Register DstReg,
MIB.setMemRefs({MemRef});
}
-bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
+bool IRTranslator::translateOverflowIntrinsic(const CallBase &CI, unsigned Op,
MachineIRBuilder &MIRBuilder) {
ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
MIRBuilder.buildInstr(
@@ -1878,7 +1878,7 @@ bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
return true;
}
-bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
+bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallBase &CI,
MachineIRBuilder &MIRBuilder) {
Register Dst = getOrCreateVReg(CI);
Register Src0 = getOrCreateVReg(*CI.getOperand(0));
@@ -2023,7 +2023,7 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return Intrinsic::not_intrinsic;
}
-bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
+bool IRTranslator::translateSimpleIntrinsic(const CallBase &CI,
Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
@@ -2145,7 +2145,7 @@ static unsigned getConvOpcode(Intrinsic::ID ID) {
}
bool IRTranslator::translateConvergenceControlIntrinsic(
- const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
+ const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
MachineInstrBuilder MIB = MIRBuilder.buildInstr(getConvOpcode(ID));
Register OutputReg = getOrCreateConvergenceTokenVReg(CI);
MIB.addDef(OutputReg);
@@ -2161,7 +2161,7 @@ bool IRTranslator::translateConvergenceControlIntrinsic(
return true;
}
-bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
+bool IRTranslator::translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
if (ORE->enabled()) {
@@ -2756,7 +2756,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
if (containsBF16Type(U))
return false;
- const CallInst &CI = cast<CallInst>(U);
+ const CallBase &CI = cast<CallBase>(U);
const Function *F = CI.getCalledFunction();
// FIXME: support Windows dllimport function calls and calls through
@@ -3023,22 +3023,14 @@ bool IRTranslator::translateCallBr(const User &U,
} else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
switch (I.getIntrinsicID()) {
default:
- report_fatal_error("Unsupported intrinsic for callbr");
+ return false;
case Intrinsic::amdgcn_kill:
- if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
- CallInst *CI =
- CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
- SmallVector<Value *, 1>(I.args()));
- bool Success = translateCall(*CI, MIRBuilder);
- CI->deleteValue();
- if (!Success)
+ if (!translateCall(I, MIRBuilder))
return false;
break;
}
} else {
- report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ return false;
}
// Retrieve successors.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 7ce4eebf685e1..7b75e43e0b08e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8518,7 +8518,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
SDValue SelectionDAG::getMemcpy(
SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
- Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
+ Align Alignment, bool isVol, bool AlwaysInline, const CallBase *CI,
std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
BatchAAResults *BatchAA) {
@@ -8644,7 +8644,7 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, const CallInst *CI,
+ bool isVol, const CallBase *CI,
std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
@@ -8762,7 +8762,7 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
bool isVol, bool AlwaysInline,
- const CallInst *CI,
+ const CallBase *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo) {
// Check to see if we should lower the memset to stores first.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c9501128cd593..ebca27d79406e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3392,14 +3392,7 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
default:
report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
- if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
- CallInst *CI =
- CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
- SmallVector<Value *, 1>(I.args()));
- visitCall(*CI);
- CI->deleteValue();
+ visitCallBase(I);
break;
}
} else {
@@ -4758,7 +4751,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
DAG.setRoot(StoreNode);
}
-void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
+void SelectionDAGBuilder::visitMaskedStore(const CallBase &I,
bool IsCompressing) {
SDLoc sdl = getCurSDLoc();
@@ -4889,7 +4882,7 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
return true;
}
-void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
+void SelectionDAGBuilder::visitMaskedScatter(const CallBase &I) {
SDLoc sdl = getCurSDLoc();
// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
@@ -4934,7 +4927,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
setValue(&I, Scatter);
}
-void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
+void SelectionDAGBuilder::visitMaskedLoad(const CallBase &I, bool IsExpanding) {
SDLoc sdl = getCurSDLoc();
auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
@@ -5003,7 +4996,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
setValue(&I, Res);
}
-void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
+void SelectionDAGBuilder::visitMaskedGather(const CallBase &I) {
SDLoc sdl = getCurSDLoc();
// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
@@ -5233,7 +5226,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
/// node.
-void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
+void SelectionDAGBuilder::visitTargetIntrinsic(const CallBase &I,
unsigned Intrinsic) {
// Ignore the callsite's attributes. A specific call site may be marked with
// readnone, but the lowering code will expect the chain based on the
@@ -6309,7 +6302,7 @@ bool SelectionDAGBuilder::visitEntryValueDbgValue(
}
/// Lower the call to the specified intrinsic function.
-void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
+void SelectionDAGBuilder::visitConvergenceControl(const CallBase &I,
unsigned Intrinsic) {
SDLoc sdl = getCurSDLoc();
switch (Intrinsic) {
@@ -6329,7 +6322,7 @@ void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
}
}
-void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
+void SelectionDAGBuilder::visitVectorHistogram(const CallBase &I,
unsigned IntrinsicID) {
// For now, we're only lowering an 'add' histogram.
// We can add others later, e.g. saturating adds, min/max.
@@ -6387,7 +6380,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
DAG.setRoot(Histogram);
}
-void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
+void SelectionDAGBuilder::visitVectorExtractLastActive(const CallBase &I,
unsigned Intrinsic) {
assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
"Tried lowering invalid vector extract last");
@@ -6415,7 +6408,7 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
}
/// Lower the call to the specified intrinsic function.
-void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
+void SelectionDAGBuilder::visitIntrinsicCall(const CallBase &I,
unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDLoc sdl = getCurSDLoc();
@@ -9047,7 +9040,7 @@ void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallBase &I) {
const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
const Value *Size = I.getArgOperand(2);
const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
@@ -9139,7 +9132,7 @@ bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitMemChrCall(const CallBase &I) {
const Value *Src = I.getArgOperand(0);
const Value *Char = I.getArgOperand(1);
const Value *Length = I.getArgOperand(2);
@@ -9163,7 +9156,7 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitMemPCpyCall(const CallBase &I) {
SDValue Dst = getValue(I.getArgOperand(0));
SDValue Src = getValue(I.getArgOperand(1));
SDValue Size = getValue(I.getArgOperand(2));
@@ -9202,7 +9195,7 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
+bool SelectionDAGBuilder::visitStrCpyCall(const CallBase &I, bool isStpcpy) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9225,7 +9218,7 @@ bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitStrCmpCall(const CallBase &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9248,7 +9241,7 @@ bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitStrLenCall(const CallBase &I) {
const Value *Arg0 = I.getArgOperand(0);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9269,7 +9262,7 @@ bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitStrNLenCall(const CallBase &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9291,7 +9284,7 @@ bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
/// false and it will be lowered like a normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
+bool SelectionDAGBuilder::visitUnaryFloatCall(const CallBase &I,
unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
@@ -9311,7 +9304,7 @@ bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
/// false, and it will be lowered like a normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
+bool SelectionDAGBuilder::visitBinaryFloatCall(const CallBase &I,
unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
@@ -9327,7 +9320,9 @@ bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
return true;
}
-void SelectionDAGBuilder::visitCall(const CallInst &I) {
+void SelectionDAGBuilder::visitCall(const CallInst &I) { visitCallBase(I); }
+
+void SelectionDAGBuilder::visitCallBase(const CallBase &I) {
// Handle inline assembly differently.
if (I.isInlineAsm()) {
visitInlineAsm(I);
@@ -10490,7 +10485,7 @@ void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
}
-void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
+void SelectionDAGBuilder::visitVAStart(const CallBase &I) {
DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
@@ -10512,14 +10507,14 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
setValue(&I, V);
}
-void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
+void SelectionDAGBuilder::visitVAEnd(const CallBase &I) {
DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
DAG.getSrcValue(I.getArgOperand(0))));
}
-void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
+void SelectionDAGBuilder::visitVACopy(const CallBase &I) {
DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
@@ -10637,7 +10632,7 @@ static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
}
/// Lower llvm.experimental.stackmap.
-void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
+void SelectionDAGBuilder::visitStackmap(const CallBase &CI) {
// void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
// [live variables...])
@@ -10849,7 +10844,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
FuncInfo.MF->getFrameInfo().setHasPatchPoint();
}
-void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
+void SelectionDAGBuilder::visitVectorReduce(const CallBase &I,
unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue Op1 = getValue(I.getArgOperand(0));
@@ -12524,14 +12519,14 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
}
}
-void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
+void SelectionDAGBuilder::visitStepVector(const CallBase &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto DL = getCurSDLoc();
EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
setValue(&I, DAG.getStepVector(DL, ResultVT));
}
-void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
+void SelectionDAGBuilder::visitVectorReverse(const CallBase &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
@@ -12554,7 +12549,7 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
}
-void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
+void SelectionDAGBuilder::visitVectorDeinterleave(const CallBase &I,
unsigned Factor) {
auto DL = getCurSDLoc();
SDValue InVec = getValue(I.getOperand(0));
@@ -12590,7 +12585,7 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
setValue(&I, Res);
}
-void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
+void SelectionDAGBuilder::visitVectorInterleave(const CallBase &I,
unsigned Factor) {
auto DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -12644,7 +12639,7 @@ void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
DAG.getVTList(ValueVTs), Values));
}
-void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
+void SelectionDAGBuilder::visitVectorSplice(const CallBase &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
@@ -12718,7 +12713,7 @@ static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
// otherwise we will end up with copies of virtregs only valid along direct
// edges.
-void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
+void SelectionDAGBuilder::visitCallBrLandingPad(const CallBase &I) {
SmallVector<EVT, 8> ResultVTs;
SmallVector<SDValue, 8> ResultValues;
const auto *CBR =
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 35c15bc269d4b..483e0b9d2db6c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -483,7 +483,7 @@ class SelectionDAGBuilder {
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee,
const BasicBlock *EHPadBB);
- void LowerDeoptimizeCall(const CallInst *CI);
+ void LowerDeoptimizeCall(const CallBase *CI);
void LowerDeoptimizingReturn();
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee,
@@ -537,7 +537,7 @@ class SelectionDAGBuilder {
// These all get lowered before this pass.
void visitInvoke(const InvokeInst &I);
void visitCallBr(const CallBrInst &I);
- void visitCallBrLandingPad(const CallInst &I);
+ void visitCallBrLandingPad(const CallBase &I);
void visitResume(const ResumeInst &I);
void visitUnary(const User &I, unsigned Opcode);
@@ -594,24 +594,25 @@ class SelectionDAGBuilder {
void visitAlloca(const AllocaInst &I);
void visitLoad(const LoadInst &I);
void visitStore(const StoreInst &I);
- void visitMaskedLoad(const CallInst &I, bool IsExpanding = false);
- void visitMaskedStore(const CallInst &I, bool IsCompressing = false);
- void visitMaskedGather(const CallInst &I);
- void visitMaskedScatter(const CallInst &I);
+ void visitMaskedLoad(const CallBase &I, bool IsExpanding = false);
+ void visitMaskedStore(const CallBase &I, bool IsCompressing = false);
+ void visitMaskedGather(const CallBase &I);
+ void visitMaskedScatter(const CallBase &I);
void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
void visitAtomicRMW(const AtomicRMWInst &I);
void visitFence(const FenceInst &I);
void visitPHI(const PHINode &I);
void visitCall(const CallInst &I);
- bool visitMemCmpBCmpCall(const CallInst &I);
- bool visitMemPCpyCall(const CallInst &I);
- bool visitMemChrCall(const CallInst &I);
- bool visitStrCpyCall(const CallInst &I, bool isStpcpy);
- bool visitStrCmpCall(const CallInst &I);
- bool visitStrLenCall(const CallInst &I);
- bool visitStrNLenCall(const CallInst &I);
- bool visitUnaryFloatCall(const CallInst &I, unsigned Opcode);
- bool visitBinaryFloatCall(const CallInst &I, unsigned Opcode);
+ void visitCallBase(const CallBase &I);
+ bool visitMemCmpBCmpCall(const CallBase &I);
+ bool visitMemPCpyCall(const CallBase &I);
+ bool visitMemChrCall(const CallBase &I);
+ bool visitStrCpyCall(const CallBase &I, bool isStpcpy);
+ bool visitStrCmpCall(const CallBase &I);
+ bool visitStrLenCall(const CallBase &I);
+ bool visitStrNLenCall(const CallBase &I);
+ bool visitUnaryFloatCall(const CallBase &I, unsigned Opcode);
+ bool visitBinaryFloatCall(const CallBase &I, unsigned Opcode);
void visitAtomicLoad(const LoadInst &I);
void visitAtomicStore(const StoreInst &I);
void visitLoadFromSwiftError(const LoadInst &I);
@@ -624,12 +625,12 @@ class SelectionDAGBuilder {
bool visitEntryValueDbgValue(ArrayRef<const Value *> Values,
DILocalVariable *Variable, DIExpression *Expr,
DebugLoc DbgLoc);
- void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
- void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
+ void visitIntrinsicCall(const CallBase &I, unsigned Intrinsic);
+ void visitTargetIntrinsic(const CallBase &I, unsigned Intrinsic);
void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
- void visitConvergenceControl(const CallInst &I, unsigned Intrinsic);
- void visitVectorHistogram(const CallInst &I, unsigned IntrinsicID);
- void visitVectorExtractLastActive(const CallInst &I, unsigned Intrinsic);
+ void visitConvergenceControl(const CallBase &I, unsigned Intrinsic);
+ void visitVectorHistogram(const CallBase &I, unsigned IntrinsicID);
+ void visitVectorExtractLastActive(const CallBase &I, unsigned Intrinsic);
void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT,
const SmallVectorImpl<SDValue> &OpValues);
void visitVPStore(const VPIntrinsic &VPIntrin,
@@ -645,23 +646,23 @@ class SelectionDAGBuilder {
void visitVPCmp(const VPCmpIntrinsic &VPIntrin);
void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin);
- void visitVAStart(const CallInst &I);
+ void visitVAStart(const CallBase &I);
void visitVAArg(const VAArgInst &I);
- void visitVAEnd(const CallInst &I);
- void visitVACopy(const CallInst &I);
- void visitStackmap(const CallInst &I);
+ void visitVAEnd(const CallBase &I);
+ void visitVACopy(const CallBase &I);
+ void visitStackmap(const CallBase &I);
void visitPatchpoint(const CallBase &CB, const BasicBlock *EHPadBB = nullptr);
// These two are implemented in StatepointLowering.cpp
void visitGCRelocate(const GCRelocateInst &Relocate);
void visitGCResult(const GCResultInst &I);
- void visitVectorReduce(const CallInst &I, unsigned Intrinsic);
- void visitVectorReverse(const CallInst &I);
- void visitVectorSplice(const CallInst &I);
- void visitVectorInterleave(const CallInst &I, unsigned Factor);
- void visitVectorDeinterleave(const CallInst &I, unsigned Factor);
- void visitStepVector(const CallInst &I);
+ void visitVectorReduce(const CallBase &I, unsigned Intrinsic);
+ void visitVectorReverse(const CallBase &I);
+ void visitVectorSplice(const CallBase &I);
+ void visitVectorInterleave(const CallBase &I, unsigned Factor);
+ void visitVectorDeinterleave(const CallBase &I, unsigned Factor);
+ void visitStepVector(const CallBase &I);
void visitUserOp1(const Instruction &I) {
llvm_unreachable("UserOp1 should not exist at instruction selection time!");
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 80aeefe8e068a..5227fd6a60e7f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -1303,7 +1303,7 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
setValue(&Relocate, SD);
}
-void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) {
+void SelectionDAGBuilder::LowerDeoptimizeCall(const CallBase *CI) {
const auto &TLI = DAG.getTargetLoweringInfo();
SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
TLI.getPointerTy(DAG.getDataLayout()));
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index f1649a3903fac..72e62e17a327a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5655,7 +5655,7 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
}
void TargetLowering::CollectTargetIntrinsicOperands(
- const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
+ const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
}
std::pair<unsigned, const TargetRegisterClass *>
diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp
index 0e526ada4b405..83b2f71c28185 100644
--- a/llvm/lib/IR/DiagnosticInfo.cpp
+++ b/llvm/lib/IR/DiagnosticInfo.cpp
@@ -456,7 +456,7 @@ void DiagnosticInfoMisExpect::print(DiagnosticPrinter &DP) const {
void OptimizationRemarkAnalysisFPCommute::anchor() {}
void OptimizationRemarkAnalysisAliasing::anchor() {}
-void llvm::diagnoseDontCall(const CallInst &CI) {
+void llvm::diagnoseDontCall(const CallBase &CI) {
const auto *F =
dyn_cast<Function>(CI.getCalledOperand()->stripPointerCasts());
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1c8e3afdfd718..3f628dbf49abb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -16373,7 +16373,7 @@ SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
template <unsigned NumVecs>
static bool
setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
- AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) {
+ AArch64TargetLowering::IntrinsicInfo &Info, const CallBase &CI) {
Info.opc = ISD::INTRINSIC_VOID;
// Retrieve EC from first vector argument.
const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType());
@@ -16398,7 +16398,7 @@ setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
/// specified in the intrinsic calls.
bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index bc0c3a832bb28..e1fadc641ebdb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -701,7 +701,7 @@ class AArch64TargetLowering : public TargetLowering {
EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *MBB) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b0c18715ef810..b42861147eb75 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1210,7 +1210,7 @@ MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
}
bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &CI,
+ const CallBase &CI,
MachineFunction &MF,
unsigned IntrID) const {
Info.flags = MachineMemOperand::MONone;
@@ -1496,7 +1496,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
}
void SITargetLowering::CollectTargetIntrinsicOperands(
- const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
+ const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
switch (cast<IntrinsicInst>(I).getIntrinsicID()) {
case Intrinsic::amdgcn_addrspacecast_nonnull: {
// The DAG's ValueType loses the addrspaces.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 8e4717a3f64ab..6d7f2d16a458e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -310,11 +310,11 @@ class SITargetLowering final : public AMDGPUTargetLowering {
MVT getPointerTy(const DataLayout &DL, unsigned AS) const override;
MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
+ bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
MachineFunction &MF,
unsigned IntrinsicID) const override;
- void CollectTargetIntrinsicOperands(const CallInst &I,
+ void CollectTargetIntrinsicOperands(const CallBase &I,
SmallVectorImpl<SDValue> &Ops,
SelectionDAG &DAG) const override;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index d2f9ec982ae01..f72070521b940 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -20953,7 +20953,7 @@ bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
/// specified in the intrinsic calls.
bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 9fad056edd3f1..883dbaa49d2ed 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -616,8 +616,7 @@ class VectorType;
bool isFPImmLegal(const APFloat &Imm, EVT VT,
bool ForCodeSize = false) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 1710488e4e292..ab6f0d5588e8d 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -2079,7 +2079,7 @@ static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
/// true and store the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 4df88b3a8abd7..2047c905fefd0 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -142,7 +142,7 @@ class HexagonTargetLowering : public TargetLowering {
const SmallVectorImpl<SDValue> &OutVals,
const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 900775eedfa7b..46c416255fca4 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -6043,7 +6043,7 @@ bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
}
bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 002fad0e20759..9021c09b6bc25 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -212,7 +212,7 @@ class LoongArchTargetLowering : public TargetLowering {
Value *NewVal, Value *Mask,
AtomicOrdering Ord) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 06e221777b7ea..a223190bf9f11 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -3632,9 +3632,10 @@ void NVPTXTargetLowering::LowerAsmOperandForConstraint(
// because we need the information that is only available in the "Value" type
// of destination
// pointer. In particular, the address space information.
-bool NVPTXTargetLowering::getTgtMemIntrinsic(
- IntrinsicInfo &Info, const CallInst &I,
- MachineFunction &MF, unsigned Intrinsic) const {
+bool NVPTXTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
+ const CallBase &I,
+ MachineFunction &MF,
+ unsigned Intrinsic) const {
switch (Intrinsic) {
default:
return false;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 39470be254efa..74ad5ef704c0d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -118,7 +118,7 @@ class NVPTXTargetLowering : public TargetLowering {
const char *getTargetNodeName(unsigned Opcode) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index ab78f33f5a630..0066ac5d41b8c 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -17478,9 +17478,8 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
}
-void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I,
- SmallVectorImpl<SDValue> &Ops,
- SelectionDAG &DAG) const {
+void PPCTargetLowering::CollectTargetIntrinsicOperands(
+ const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
if (I.getNumOperands() <= 1)
return;
if (!isa<ConstantSDNode>(Ops[1].getNode()))
@@ -17668,7 +17667,7 @@ PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
}
bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 1f22aa16a89be..64ef67923ef0e 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -1008,9 +1008,9 @@ namespace llvm {
return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
}
- void CollectTargetIntrinsicOperands(const CallInst &I,
- SmallVectorImpl<SDValue> &Ops,
- SelectionDAG &DAG) const override;
+ void CollectTargetIntrinsicOperands(const CallBase &I,
+ SmallVectorImpl<SDValue> &Ops,
+ SelectionDAG &DAG) const override;
/// isLegalAddressingMode - Return true if the addressing mode represented
/// by AM is legal for this target, for a load/store of the specified type.
@@ -1069,8 +1069,7 @@ namespace llvm {
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5b5dca4b541df..00cd0c5c87163 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1644,7 +1644,7 @@ bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
}
bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index f4d6cd86397a4..cc401a052c7f2 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -517,7 +517,7 @@ class RISCVTargetLowering : public TargetLowering {
const RISCVSubtarget &getSubtarget() const { return Subtarget; }
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index c347dde89256f..68bdab2da0e33 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -55,7 +55,7 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
}
bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const Callase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
unsigned AlignIdx = 3;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index eb78299b72f04..9e23ae3587fae 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -49,7 +49,7 @@ class SPIRVTargetLowering : public TargetLowering {
EVT VT) const override;
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
EVT VT) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 9ae46e709d823..0f8244dc50107 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1033,7 +1033,7 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
}
bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index 90d31e38a7076..a077fa6cbf7ea 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -72,7 +72,7 @@ class WebAssemblyTargetLowering final : public TargetLowering {
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
EVT VT) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b128a6dadbbb6..a5c299d3e682e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3101,7 +3101,7 @@ static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
}
bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
Info.flags = MachineMemOperand::MONone;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 4a2b35e9efe7c..1c776db401562 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1467,7 +1467,7 @@ namespace llvm {
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and stores the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
>From 46f44758efdb6d7829a0c200ca4c2bf3fc290a4f Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 2 Apr 2025 05:04:29 -0500
Subject: [PATCH 4/4] adapt and more tests
---
llvm/test/Assembler/callbr.ll | 22 +++++++++++++++++
llvm/test/CodeGen/AMDGPU/callbr.ll | 22 ++---------------
llvm/test/Verifier/callbr.ll | 39 ++++++++++++++++++++++++++++++
3 files changed, 63 insertions(+), 20 deletions(-)
create mode 100644 llvm/test/Assembler/callbr.ll
diff --git a/llvm/test/Assembler/callbr.ll b/llvm/test/Assembler/callbr.ll
new file mode 100644
index 0000000000000..0084e9763c62c
--- /dev/null
+++ b/llvm/test/Assembler/callbr.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+
+declare void @llvm.amdgcn.kill(i1)
+
+define void @test_kill(i1 %c) {
+; CHECK-LABEL: define void @test_kill(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: callbr void @llvm.amdgcn.kill(i1 [[C]])
+; CHECK-NEXT: to label %[[CONT:.*]] [label %kill]
+; CHECK: [[KILL:.*:]]
+; CHECK-NEXT: unreachable
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+;
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+kill:
+ unreachable
+cont:
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll
index e2e84dca96cbf..33c75f6cf5aab 100644
--- a/llvm/test/CodeGen/AMDGPU/callbr.ll
+++ b/llvm/test/CodeGen/AMDGPU/callbr.ll
@@ -1,14 +1,5 @@
-; RUN: rm -rf %t && split-file %s %t
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-seldag.s < %t/with-callbr.ll
-; RUN: FileCheck --check-prefix=SELDAG %s < %t/with-callbr-seldag.s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-gisel.s -global-isel < %t/with-callbr.ll
-; RUN: FileCheck --check-prefix=GISEL %s < %t/with-callbr-gisel.s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-seldag.s < %t/without-callbr.ll
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-gisel.s -global-isel < %t/without-callbr.ll
-; RUN: diff %t/with-callbr-seldag.s %t/without-callbr-seldag.s
-; RUN: diff %t/with-callbr-gisel.s %t/without-callbr-gisel.s
-
-;--- with-callbr.ll
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o - < %s | FileCheck --check-prefix=SELDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o - -global-isel < %s | FileCheck --check-prefix=GISEL %s
; SELDAG-LABEL: test_kill:
; SELDAG-NEXT: ; %bb.0:
@@ -59,12 +50,3 @@ cont:
store i32 %a, ptr %dst, align 4
ret void
}
-
-;--- without-callbr.ll
-
-define void @test_kill(ptr %src, ptr %dst, i1 %c) {
- %a = load i32, ptr %src, align 4
- call void @llvm.amdgcn.kill(i1 %c)
- store i32 %a, ptr %dst, align 4
- ret void
-}
diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll
index 9b819c5fed48b..8e125e723e6fc 100644
--- a/llvm/test/Verifier/callbr.ll
+++ b/llvm/test/Verifier/callbr.ll
@@ -120,3 +120,42 @@ landingpad:
%out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
ret i32 %out
}
+
+declare void @llvm.amdgcn.kill(i1)
+
+; CHECK-NEXT: Callbr amdgcn_kill only supports one indirect dest
+define void @test_callbr_intrinsic_indirect0(i1 %c) {
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont []
+kill:
+ unreachable
+cont:
+ ret void
+}
+
+; CHECK-NEXT: Callbr amdgcn_kill only supports one indirect dest
+define void @test_callbr_intrinsic_indirect2(i1 %c) {
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill1, label %kill2]
+kill1:
+ unreachable
+kill2:
+ unreachable
+cont:
+ ret void
+}
+
+; CHECK-NEXT: Callbr amdgcn_kill indirect dest needs to be unreachable
+define void @test_callbr_intrinsic_no_unreachable(i1 %c) {
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+kill:
+ ret void
+cont:
+ ret void
+}
+
+; CHECK-NEXT: Callbr currently only supports asm-goto and selected intrinsics
+declare i32 @llvm.amdgcn.workitem.id.x()
+define void @test_callbr_intrinsic_unsupported() {
+ callbr i32 @llvm.amdgcn.workitem.id.x() to label %cont []
+cont:
+ ret void
+}
More information about the llvm-commits
mailing list