[llvm] [polly] [IR] Add CallBr intrinsics support (PR #133907)
Robert Imschweiler via llvm-commits
llvm-commits at lists.llvm.org
Mon May 26 05:53:17 PDT 2025
https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/133907
>From aa88379c89e3f851a935b681128ce1d98c0aa144 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Tue, 1 Apr 2025 08:03:16 -0500
Subject: [PATCH 01/13] [IR] Add CallBr intrinsics support
This commit adds support for using intrinsics with callbr.
The uses of this will most of the time look like this example:
```llvm
callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
kill:
unreachable
cont:
...
```
---
llvm/include/llvm/Analysis/RegionInfoImpl.h | 12 ++++
.../llvm/Transforms/Utils/BasicBlockUtils.h | 6 +-
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 59 +++++++++++++++-
.../SelectionDAG/SelectionDAGBuilder.cpp | 45 +++++++++---
llvm/lib/IR/Verifier.cpp | 29 ++++++--
llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 11 ++-
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 4 +-
llvm/test/CodeGen/AMDGPU/callbr.ll | 70 +++++++++++++++++++
8 files changed, 207 insertions(+), 29 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/callbr.ll
diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h
index eb99d8bc6fb23..759e9c47bebb8 100644
--- a/llvm/include/llvm/Analysis/RegionInfoImpl.h
+++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h
@@ -553,6 +553,18 @@ bool RegionInfoBase<Tr>::isRegion(BlockT *entry, BlockT *exit) const {
using DST = typename DomFrontierT::DomSetType;
+ // TODO? post domination frontier?
+ if constexpr (std::is_same_v<BlockT, BasicBlock>) {
+ if (DomTreeNodeT *PDTNode = PDT->getNode(exit); PDTNode) {
+ for (DomTreeNodeT *PredNode : *PDTNode) {
+ for (BasicBlock *Pred : predecessors(PredNode->getBlock())) {
+ if (isa<CallBrInst>(Pred->getTerminator()))
+ return false;
+ }
+ }
+ }
+ }
+
DST *entrySuccs = &DF->find(entry)->second;
// Exit is the header of a loop that contains the entry. In this case,
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index adc1851c2ec2f..0e89604c7e59c 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -606,9 +606,9 @@ bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI,
// successors
void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder);
-// Check whether the function only has simple terminator:
-// br/brcond/unreachable/ret
-bool hasOnlySimpleTerminator(const Function &F);
+// Check whether the function only has blocks with simple terminators:
+// br/brcond/unreachable/ret (or callbr if AllowCallBr)
+bool hasOnlySimpleTerminator(const Function &F, bool AllowCallBr = true);
} // end namespace llvm
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index fe5dcd14d8804..65e36de76307d 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3008,8 +3008,63 @@ bool IRTranslator::translateInvoke(const User &U,
bool IRTranslator::translateCallBr(const User &U,
MachineIRBuilder &MIRBuilder) {
- // FIXME: Implement this.
- return false;
+ const CallBrInst &I = cast<CallBrInst>(U);
+ MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
+
+ // TODO: operand bundles (see SelDAG implementation of callbr)?
+ assert(!I.hasOperandBundles() &&
+ "Cannot lower callbrs with operand bundles yet");
+
+ if (I.isInlineAsm()) {
+ // FIXME: inline asm not yet supported
+ if (!translateInlineAsm(I, MIRBuilder))
+ return false;
+ } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
+ switch (I.getIntrinsicID()) {
+ default:
+ report_fatal_error("Unsupported intrinsic for callbr");
+ case Intrinsic::amdgcn_kill:
+ if (I.getNumIndirectDests() != 1)
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
+ CallInst *CI =
+ CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
+ SmallVector<Value *, 1>(I.args()));
+ bool Success = translateCall(*CI, MIRBuilder);
+ CI->deleteValue();
+ if (!Success)
+ return false;
+ break;
+ }
+ } else {
+ report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ }
+
+ // Retrieve successors.
+ SmallPtrSet<BasicBlock *, 8> Dests;
+ Dests.insert(I.getDefaultDest());
+ MachineBasicBlock *Return = &getMBB(*I.getDefaultDest());
+
+ // Update successor info.
+ addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
+ // TODO: For most of the cases where there is an intrinsic callbr, we're
+ // having exactly one indirect target, which will be unreachable. As soon as
+ // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
+ // or add something similar for intrinsic indirect branches.
+ if (I.isInlineAsm()) {
+ for (BasicBlock *Dest : I.getIndirectDests()) {
+ MachineBasicBlock *Target = &getMBB(*Dest);
+ Target->setIsInlineAsmBrIndirectTarget();
+ Target->setMachineBlockAddressTaken();
+ Target->setLabelMustBeEmitted();
+ // Don't add duplicate machine successors.
+ if (Dests.insert(Dest).second)
+ addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+ }
+ }
+ CallBrMBB->normalizeSuccProbs();
+
+ return true;
}
bool IRTranslator::translateLandingPad(const User &U,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 434484b671bf2..8dc6c837e8bb8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3384,8 +3384,26 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
{LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
"Cannot lower callbrs with arbitrary operand bundles yet!");
- assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
- visitInlineAsm(I);
+ if (I.isInlineAsm()) {
+ visitInlineAsm(I);
+ } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
+ switch (I.getIntrinsicID()) {
+ default:
+ report_fatal_error("Unsupported intrinsic for callbr");
+ case Intrinsic::amdgcn_kill:
+ if (I.getNumIndirectDests() != 1)
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
+ CallInst *CI =
+ CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
+ SmallVector<Value *, 1>(I.args()));
+ visitCall(*CI);
+ CI->deleteValue();
+ break;
+ }
+ } else {
+ report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ }
CopyToExportRegsIfNeeded(&I);
// Retrieve successors.
@@ -3395,15 +3413,20 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
// Update successor info.
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
- for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
- BasicBlock *Dest = I.getIndirectDest(i);
- MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
- Target->setIsInlineAsmBrIndirectTarget();
- Target->setMachineBlockAddressTaken();
- Target->setLabelMustBeEmitted();
- // Don't add duplicate machine successors.
- if (Dests.insert(Dest).second)
- addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+ // TODO: For most of the cases where there is an intrinsic callbr, we're
+ // having exactly one indirect target, which will be unreachable. As soon as
+ // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
+ // or add something similar for intrinsic indirect branches.
+ if (I.isInlineAsm()) {
+ for (BasicBlock *Dest : I.getIndirectDests()) {
+ MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
+ Target->setIsInlineAsmBrIndirectTarget();
+ Target->setMachineBlockAddressTaken();
+ Target->setLabelMustBeEmitted();
+ // Don't add duplicate machine successors.
+ if (Dests.insert(Dest).second)
+ addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+ }
}
CallBrMBB->normalizeSuccProbs();
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index fdb4ddaafbbcc..9d3edc2fe680f 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3278,11 +3278,30 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) {
}
void Verifier::visitCallBrInst(CallBrInst &CBI) {
- Check(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", &CBI);
- const InlineAsm *IA = cast<InlineAsm>(CBI.getCalledOperand());
- Check(!IA->canThrow(), "Unwinding from Callbr is not allowed");
+ if (!CBI.isInlineAsm()) {
+ switch (CBI.getIntrinsicID()) {
+ case Intrinsic::amdgcn_kill: {
+ Check(CBI.getNumIndirectDests() == 1,
+ "Callbr amdgcn_kill only supports one indirect dest");
+ bool Unreachable = isa<UnreachableInst>(CBI.getIndirectDest(0)->begin());
+ CallInst *Call = dyn_cast<CallInst>(CBI.getIndirectDest(0)->begin());
+ Check(Unreachable ||
+ (Call && Call->getIntrinsicID() == Intrinsic::amdgcn_unreachable),
+ "Callbr amdgcn_kill indirect dest needs to be unreachable");
+ visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI);
+ break;
+ }
+ default:
+ CheckFailed(
+ "Callbr currently only supports asm-goto and selected intrinsics");
+ }
+ visitIntrinsicCall(CBI.getIntrinsicID(), CBI);
+ } else {
+ const InlineAsm *IA = cast<InlineAsm>(CBI.getCalledOperand());
+ Check(!IA->canThrow(), "Unwinding from Callbr is not allowed");
- verifyInlineAsmCall(CBI);
+ verifyInlineAsmCall(CBI);
+ }
visitTerminator(CBI);
}
@@ -5254,7 +5273,7 @@ void Verifier::visitInstruction(Instruction &I) {
(CBI && &CBI->getCalledOperandUse() == &I.getOperandUse(i)) ||
IsAttachedCallOperand(F, CBI, i)),
"Cannot take the address of an intrinsic!", &I);
- Check(!F->isIntrinsic() || isa<CallInst>(I) ||
+ Check(!F->isIntrinsic() || isa<CallInst>(I) || isa<CallBrInst>(I) ||
F->getIntrinsicID() == Intrinsic::donothing ||
F->getIntrinsicID() == Intrinsic::seh_try_begin ||
F->getIntrinsicID() == Intrinsic::seh_try_end ||
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index a69d64956d6d9..18d3381b420bd 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -480,11 +480,10 @@ void StructurizeCFG::analyzeLoops(RegionNode *N) {
} else {
// Test for successors as back edge
BasicBlock *BB = N->getNodeAs<BasicBlock>();
- BranchInst *Term = cast<BranchInst>(BB->getTerminator());
-
- for (BasicBlock *Succ : Term->successors())
- if (Visited.count(Succ))
- Loops[Succ] = BB;
+ if (BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator()); Term)
+ for (BasicBlock *Succ : Term->successors())
+ if (Visited.count(Succ))
+ Loops[Succ] = BB;
}
}
@@ -516,7 +515,7 @@ void StructurizeCFG::gatherPredicates(RegionNode *N) {
for (BasicBlock *P : predecessors(BB)) {
// Ignore it if it's a branch from outside into our region entry
- if (!ParentRegion->contains(P))
+ if (!ParentRegion->contains(P) || !dyn_cast<BranchInst>(P->getTerminator()))
continue;
Region *R = RI->getRegionFor(P);
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 6608515e1cbbc..41036b5ec9194 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1900,11 +1900,11 @@ void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) {
PBI->swapSuccessors();
}
-bool llvm::hasOnlySimpleTerminator(const Function &F) {
+bool llvm::hasOnlySimpleTerminator(const Function &F, bool AllowCallBr) {
for (auto &BB : F) {
auto *Term = BB.getTerminator();
if (!(isa<ReturnInst>(Term) || isa<UnreachableInst>(Term) ||
- isa<BranchInst>(Term)))
+ isa<BranchInst>(Term) || (AllowCallBr && isa<CallBrInst>(Term))))
return false;
}
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll
new file mode 100644
index 0000000000000..e2e84dca96cbf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/callbr.ll
@@ -0,0 +1,70 @@
+; RUN: rm -rf %t && split-file %s %t
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-seldag.s < %t/with-callbr.ll
+; RUN: FileCheck --check-prefix=SELDAG %s < %t/with-callbr-seldag.s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-gisel.s -global-isel < %t/with-callbr.ll
+; RUN: FileCheck --check-prefix=GISEL %s < %t/with-callbr-gisel.s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-seldag.s < %t/without-callbr.ll
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-gisel.s -global-isel < %t/without-callbr.ll
+; RUN: diff %t/with-callbr-seldag.s %t/without-callbr-seldag.s
+; RUN: diff %t/with-callbr-gisel.s %t/without-callbr-gisel.s
+
+;--- with-callbr.ll
+
+; SELDAG-LABEL: test_kill:
+; SELDAG-NEXT: ; %bb.0:
+; SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: flat_load_dword v0, v[0:1]
+; SELDAG-NEXT: v_and_b32_e32 v1, 1, v4
+; SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; SELDAG-NEXT: s_mov_b64 s[4:5], exec
+; SELDAG-NEXT: s_andn2_b64 s[6:7], exec, vcc
+; SELDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; SELDAG-NEXT: s_cbranch_scc0 .LBB0_2
+; SELDAG-NEXT: ; %bb.1:
+; SELDAG-NEXT: s_and_b64 exec, exec, s[4:5]
+; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: flat_store_dword v[2:3], v0
+; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: s_setpc_b64 s[30:31]
+; SELDAG-NEXT: .LBB0_2:
+; SELDAG-NEXT: s_mov_b64 exec, 0
+; SELDAG-NEXT: s_endpgm
+
+; GISEL-LABEL: test_kill:
+; GISEL-NEXT: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: flat_load_dword v0, v[0:1]
+; GISEL-NEXT: v_and_b32_e32 v1, 1, v4
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: s_andn2_b64 s[6:7], exec, vcc
+; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GISEL-NEXT: s_cbranch_scc0 .LBB0_2
+; GISEL-NEXT: ; %bb.1:
+; GISEL-NEXT: s_and_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT: flat_store_dword v[2:3], v0
+; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB0_2:
+; GISEL-NEXT: s_mov_b64 exec, 0
+; GISEL-NEXT: s_endpgm
+
+define void @test_kill(ptr %src, ptr %dst, i1 %c) {
+ %a = load i32, ptr %src, align 4
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+kill:
+ unreachable
+cont:
+ store i32 %a, ptr %dst, align 4
+ ret void
+}
+
+;--- without-callbr.ll
+
+define void @test_kill(ptr %src, ptr %dst, i1 %c) {
+ %a = load i32, ptr %src, align 4
+ call void @llvm.amdgcn.kill(i1 %c)
+ store i32 %a, ptr %dst, align 4
+ ret void
+}
>From 268d144af88ce8fd583c49d2d33bea63705ba122 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Tue, 1 Apr 2025 08:07:44 -0500
Subject: [PATCH 02/13] fix formatting
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++--
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 9 +++++----
llvm/lib/IR/Verifier.cpp | 4 ++--
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 65e36de76307d..3b33d3784c1f9 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3049,8 +3049,9 @@ bool IRTranslator::translateCallBr(const User &U,
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
// TODO: For most of the cases where there is an intrinsic callbr, we're
// having exactly one indirect target, which will be unreachable. As soon as
- // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
- // or add something similar for intrinsic indirect branches.
+ // this changes, we might need to enhance
+ // Target->setIsInlineAsmBrIndirectTarget or add something similar for
+ // intrinsic indirect branches.
if (I.isInlineAsm()) {
for (BasicBlock *Dest : I.getIndirectDests()) {
MachineBasicBlock *Target = &getMBB(*Dest);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 8dc6c837e8bb8..68f6a0445b1af 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3392,8 +3392,8 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
CallInst *CI =
CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
SmallVector<Value *, 1>(I.args()));
@@ -3415,8 +3415,9 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
// TODO: For most of the cases where there is an intrinsic callbr, we're
// having exactly one indirect target, which will be unreachable. As soon as
- // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget
- // or add something similar for intrinsic indirect branches.
+ // this changes, we might need to enhance
+ // Target->setIsInlineAsmBrIndirectTarget or add something similar for
+ // intrinsic indirect branches.
if (I.isInlineAsm()) {
for (BasicBlock *Dest : I.getIndirectDests()) {
MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 9d3edc2fe680f..2811c6c712b3b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3285,8 +3285,8 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) {
"Callbr amdgcn_kill only supports one indirect dest");
bool Unreachable = isa<UnreachableInst>(CBI.getIndirectDest(0)->begin());
CallInst *Call = dyn_cast<CallInst>(CBI.getIndirectDest(0)->begin());
- Check(Unreachable ||
- (Call && Call->getIntrinsicID() == Intrinsic::amdgcn_unreachable),
+ Check(Unreachable || (Call && Call->getIntrinsicID() ==
+ Intrinsic::amdgcn_unreachable),
"Callbr amdgcn_kill indirect dest needs to be unreachable");
visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI);
break;
>From df990f2e97ee329858d9cbba7dd5b440e3b72ca0 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 2 Apr 2025 04:26:22 -0500
Subject: [PATCH 03/13] implement feedback
---
llvm/include/llvm/CodeGen/Analysis.h | 2 +-
.../llvm/CodeGen/GlobalISel/IRTranslator.h | 18 ++---
llvm/include/llvm/CodeGen/SelectionDAG.h | 6 +-
llvm/include/llvm/CodeGen/TargetLowering.h | 6 +-
llvm/include/llvm/IR/DiagnosticInfo.h | 4 +-
llvm/include/llvm/IR/IntrinsicInst.h | 8 ++-
llvm/lib/CodeGen/Analysis.cpp | 2 +-
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 34 ++++-----
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 71 +++++++++----------
.../SelectionDAG/SelectionDAGBuilder.h | 61 ++++++++--------
.../SelectionDAG/StatepointLowering.cpp | 2 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 2 +-
llvm/lib/IR/DiagnosticInfo.cpp | 2 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 4 +-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +-
llvm/lib/Target/ARM/ARMISelLowering.cpp | 2 +-
llvm/lib/Target/ARM/ARMISelLowering.h | 3 +-
.../Target/Hexagon/HexagonISelLowering.cpp | 2 +-
llvm/lib/Target/Hexagon/HexagonISelLowering.h | 2 +-
.../LoongArch/LoongArchISelLowering.cpp | 2 +-
.../Target/LoongArch/LoongArchISelLowering.h | 2 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 7 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 2 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 7 +-
llvm/lib/Target/PowerPC/PPCISelLowering.h | 9 ++-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 2 +-
.../WebAssembly/WebAssemblyISelLowering.cpp | 2 +-
.../WebAssembly/WebAssemblyISelLowering.h | 2 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
llvm/lib/Target/X86/X86ISelLowering.h | 2 +-
36 files changed, 142 insertions(+), 150 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index 362cc30bbd06a..a4a604dd2c608 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -150,7 +150,7 @@ bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I,
/// Returns true if the parent of \p CI returns CI's first argument after
/// calling \p CI.
-bool funcReturnsFirstArgOfCall(const CallInst &CI);
+bool funcReturnsFirstArgOfCall(const CallBase &CI);
DenseMap<const MachineBasicBlock *, int>
getEHScopeMembership(const MachineFunction &MF);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 6fd05c8fddd5f..d3f731cf02be9 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -235,26 +235,26 @@ class IRTranslator : public MachineFunctionPass {
bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
/// Translate an LLVM string intrinsic (memcpy, memset, ...).
- bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
+ bool translateMemFunc(const CallBase &CI, MachineIRBuilder &MIRBuilder,
unsigned Opcode);
/// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
- bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
+ bool translateTrap(const CallBase &U, MachineIRBuilder &MIRBuilder,
unsigned Opcode);
// Translate @llvm.vector.interleave2 and
// @llvm.vector.deinterleave2 intrinsics for fixed-width vector
// types into vector shuffles.
- bool translateVectorInterleave2Intrinsic(const CallInst &CI,
+ bool translateVectorInterleave2Intrinsic(const CallBase &CI,
MachineIRBuilder &MIRBuilder);
- bool translateVectorDeinterleave2Intrinsic(const CallInst &CI,
+ bool translateVectorDeinterleave2Intrinsic(const CallBase &CI,
MachineIRBuilder &MIRBuilder);
void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
- bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
+ bool translateOverflowIntrinsic(const CallBase &CI, unsigned Op,
MachineIRBuilder &MIRBuilder);
- bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
+ bool translateFixedPointIntrinsic(unsigned Op, const CallBase &CI,
MachineIRBuilder &MIRBuilder);
/// Helper function for translateSimpleIntrinsic.
@@ -265,13 +265,13 @@ class IRTranslator : public MachineFunctionPass {
/// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
/// \return true if the translation succeeded.
- bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
+ bool translateSimpleIntrinsic(const CallBase &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI,
MachineIRBuilder &MIRBuilder);
- bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
+ bool translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
/// Returns the single livein physical register Arg was lowered to, if
@@ -588,7 +588,7 @@ class IRTranslator : public MachineFunctionPass {
return false;
}
- bool translateConvergenceControlIntrinsic(const CallInst &CI,
+ bool translateConvergenceControlIntrinsic(const CallBase &CI,
Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 87b6914f8a0ee..00f25e753f292 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1238,7 +1238,7 @@ class SelectionDAG {
* the tail call optimization decision. */
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, const CallInst *CI,
+ bool AlwaysInline, const CallBase *CI,
std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
@@ -1250,7 +1250,7 @@ class SelectionDAG {
* the tail call optimization decision. */
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- const CallInst *CI, std::optional<bool> OverrideTailCall,
+ const CallBase *CI, std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1258,7 +1258,7 @@ class SelectionDAG {
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, const CallInst *CI,
+ bool AlwaysInline, const CallBase *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes());
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index d9fce57c6e43a..dfc4320ed92f5 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1239,7 +1239,7 @@ class TargetLoweringBase {
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and store the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
- virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
+ virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
MachineFunction &,
unsigned /*Intrinsic*/) const {
return false;
@@ -5222,9 +5222,9 @@ class TargetLowering : public TargetLoweringBase {
const AsmOperandInfo &OpInfo,
SelectionDAG &DAG) const;
- // Targets may override this function to collect operands from the CallInst
+ // Targets may override this function to collect operands from the CallBase
// and for example, lower them into the SelectionDAG operands.
- virtual void CollectTargetIntrinsicOperands(const CallInst &I,
+ virtual void CollectTargetIntrinsicOperands(const CallBase &I,
SmallVectorImpl<SDValue> &Ops,
SelectionDAG &DAG) const;
diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h
index a1113134f6a34..b43ef657162c8 100644
--- a/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -38,7 +38,7 @@ namespace llvm {
class DiagnosticPrinter;
class DIFile;
class DISubprogram;
-class CallInst;
+class CallBase;
class Function;
class Instruction;
class InstructionCost;
@@ -1151,7 +1151,7 @@ class DiagnosticInfoSrcMgr : public DiagnosticInfo {
}
};
-void diagnoseDontCall(const CallInst &CI);
+void diagnoseDontCall(const CallBase &CI);
class DiagnosticInfoDontCall : public DiagnosticInfo {
StringRef CalleeName;
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 1217a628dcb29..4828240f06967 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -134,8 +134,14 @@ class IntrinsicInst : public CallInst {
return CF->isIntrinsic();
return false;
}
+ static bool classof(const CallBase *I) {
+ if (const Function *CF = I->getCalledFunction())
+ return CF->isIntrinsic();
+ return false;
+ }
static bool classof(const Value *V) {
- return isa<CallInst>(V) && classof(cast<CallInst>(V));
+ return (isa<CallInst>(V) && classof(cast<CallInst>(V))) ||
+ (isa<CallBase>(V) && classof(cast<CallBase>(V)));
}
};
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index e7b9417de8c9f..f950bb756fb2b 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -712,7 +712,7 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
return true;
}
-bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
+bool llvm::funcReturnsFirstArgOfCall(const CallBase &CI) {
const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
bool ReturnsFirstArg = false;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 3b33d3784c1f9..1b6efe1c91fc4 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1693,7 +1693,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
return true;
}
-bool IRTranslator::translateMemFunc(const CallInst &CI,
+bool IRTranslator::translateMemFunc(const CallBase &CI,
MachineIRBuilder &MIRBuilder,
unsigned Opcode) {
const Value *SrcPtr = CI.getArgOperand(1);
@@ -1780,7 +1780,7 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
return true;
}
-bool IRTranslator::translateTrap(const CallInst &CI,
+bool IRTranslator::translateTrap(const CallBase &CI,
MachineIRBuilder &MIRBuilder,
unsigned Opcode) {
StringRef TrapFuncName =
@@ -1807,7 +1807,7 @@ bool IRTranslator::translateTrap(const CallInst &CI,
}
bool IRTranslator::translateVectorInterleave2Intrinsic(
- const CallInst &CI, MachineIRBuilder &MIRBuilder) {
+ const CallBase &CI, MachineIRBuilder &MIRBuilder) {
assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
"This function can only be called on the interleave2 intrinsic!");
// Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
@@ -1823,7 +1823,7 @@ bool IRTranslator::translateVectorInterleave2Intrinsic(
}
bool IRTranslator::translateVectorDeinterleave2Intrinsic(
- const CallInst &CI, MachineIRBuilder &MIRBuilder) {
+ const CallBase &CI, MachineIRBuilder &MIRBuilder) {
assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
"This function can only be called on the deinterleave2 intrinsic!");
// Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
@@ -1863,7 +1863,7 @@ void IRTranslator::getStackGuard(Register DstReg,
MIB.setMemRefs({MemRef});
}
-bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
+bool IRTranslator::translateOverflowIntrinsic(const CallBase &CI, unsigned Op,
MachineIRBuilder &MIRBuilder) {
ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
MIRBuilder.buildInstr(
@@ -1873,7 +1873,7 @@ bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
return true;
}
-bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
+bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallBase &CI,
MachineIRBuilder &MIRBuilder) {
Register Dst = getOrCreateVReg(CI);
Register Src0 = getOrCreateVReg(*CI.getOperand(0));
@@ -2022,7 +2022,7 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return Intrinsic::not_intrinsic;
}
-bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
+bool IRTranslator::translateSimpleIntrinsic(const CallBase &CI,
Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
@@ -2144,7 +2144,7 @@ static unsigned getConvOpcode(Intrinsic::ID ID) {
}
bool IRTranslator::translateConvergenceControlIntrinsic(
- const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
+ const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
MachineInstrBuilder MIB = MIRBuilder.buildInstr(getConvOpcode(ID));
Register OutputReg = getOrCreateConvergenceTokenVReg(CI);
MIB.addDef(OutputReg);
@@ -2160,7 +2160,7 @@ bool IRTranslator::translateConvergenceControlIntrinsic(
return true;
}
-bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
+bool IRTranslator::translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
if (ORE->enabled()) {
@@ -2754,7 +2754,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
if (containsBF16Type(U))
return false;
- const CallInst &CI = cast<CallInst>(U);
+ const CallBase &CI = cast<CallBase>(U);
const Function *F = CI.getCalledFunction();
// FIXME: support Windows dllimport function calls and calls through
@@ -3022,22 +3022,14 @@ bool IRTranslator::translateCallBr(const User &U,
} else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
switch (I.getIntrinsicID()) {
default:
- report_fatal_error("Unsupported intrinsic for callbr");
+ return false;
case Intrinsic::amdgcn_kill:
- if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
- CallInst *CI =
- CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
- SmallVector<Value *, 1>(I.args()));
- bool Success = translateCall(*CI, MIRBuilder);
- CI->deleteValue();
- if (!Success)
+ if (!translateCall(I, MIRBuilder))
return false;
break;
}
} else {
- report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ return false;
}
// Retrieve successors.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index e6a7d092b7b79..e14b71106e975 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8674,7 +8674,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
SDValue SelectionDAG::getMemcpy(
SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
- Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
+ Align Alignment, bool isVol, bool AlwaysInline, const CallBase *CI,
std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
BatchAAResults *BatchAA) {
@@ -8800,7 +8800,7 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, const CallInst *CI,
+ bool isVol, const CallBase *CI,
std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
@@ -8918,7 +8918,7 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
bool isVol, bool AlwaysInline,
- const CallInst *CI,
+ const CallBase *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo) {
// Check to see if we should lower the memset to stores first.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 68f6a0445b1af..1f41d57c0872c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3391,14 +3391,7 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
default:
report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
- if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
- CallInst *CI =
- CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
- SmallVector<Value *, 1>(I.args()));
- visitCall(*CI);
- CI->deleteValue();
+ visitCallBase(I);
break;
}
} else {
@@ -4757,7 +4750,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
DAG.setRoot(StoreNode);
}
-void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
+void SelectionDAGBuilder::visitMaskedStore(const CallBase &I,
bool IsCompressing) {
SDLoc sdl = getCurSDLoc();
@@ -4888,7 +4881,7 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
return true;
}
-void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
+void SelectionDAGBuilder::visitMaskedScatter(const CallBase &I) {
SDLoc sdl = getCurSDLoc();
// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
@@ -4933,7 +4926,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
setValue(&I, Scatter);
}
-void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
+void SelectionDAGBuilder::visitMaskedLoad(const CallBase &I, bool IsExpanding) {
SDLoc sdl = getCurSDLoc();
auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
@@ -5002,7 +4995,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
setValue(&I, Res);
}
-void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
+void SelectionDAGBuilder::visitMaskedGather(const CallBase &I) {
SDLoc sdl = getCurSDLoc();
// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
@@ -5236,7 +5229,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
/// node.
-void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
+void SelectionDAGBuilder::visitTargetIntrinsic(const CallBase &I,
unsigned Intrinsic) {
// Ignore the callsite's attributes. A specific call site may be marked with
// readnone, but the lowering code will expect the chain based on the
@@ -6319,7 +6312,7 @@ bool SelectionDAGBuilder::visitEntryValueDbgValue(
}
/// Lower the call to the specified intrinsic function.
-void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
+void SelectionDAGBuilder::visitConvergenceControl(const CallBase &I,
unsigned Intrinsic) {
SDLoc sdl = getCurSDLoc();
switch (Intrinsic) {
@@ -6339,7 +6332,7 @@ void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
}
}
-void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
+void SelectionDAGBuilder::visitVectorHistogram(const CallBase &I,
unsigned IntrinsicID) {
// For now, we're only lowering an 'add' histogram.
// We can add others later, e.g. saturating adds, min/max.
@@ -6397,7 +6390,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
DAG.setRoot(Histogram);
}
-void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
+void SelectionDAGBuilder::visitVectorExtractLastActive(const CallBase &I,
unsigned Intrinsic) {
assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
"Tried lowering invalid vector extract last");
@@ -6425,7 +6418,7 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
}
/// Lower the call to the specified intrinsic function.
-void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
+void SelectionDAGBuilder::visitIntrinsicCall(const CallBase &I,
unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDLoc sdl = getCurSDLoc();
@@ -9035,7 +9028,7 @@ void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallBase &I) {
const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
const Value *Size = I.getArgOperand(2);
const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
@@ -9127,7 +9120,7 @@ bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitMemChrCall(const CallBase &I) {
const Value *Src = I.getArgOperand(0);
const Value *Char = I.getArgOperand(1);
const Value *Length = I.getArgOperand(2);
@@ -9151,7 +9144,7 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitMemPCpyCall(const CallBase &I) {
SDValue Dst = getValue(I.getArgOperand(0));
SDValue Src = getValue(I.getArgOperand(1));
SDValue Size = getValue(I.getArgOperand(2));
@@ -9190,7 +9183,7 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
+bool SelectionDAGBuilder::visitStrCpyCall(const CallBase &I, bool isStpcpy) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9213,7 +9206,7 @@ bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitStrCmpCall(const CallBase &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9236,7 +9229,7 @@ bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitStrLenCall(const CallBase &I) {
const Value *Arg0 = I.getArgOperand(0);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9257,7 +9250,7 @@ bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
+bool SelectionDAGBuilder::visitStrNLenCall(const CallBase &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9279,7 +9272,7 @@ bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
/// false and it will be lowered like a normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
+bool SelectionDAGBuilder::visitUnaryFloatCall(const CallBase &I,
unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
@@ -9299,7 +9292,7 @@ bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
/// false, and it will be lowered like a normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
+bool SelectionDAGBuilder::visitBinaryFloatCall(const CallBase &I,
unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
@@ -9315,7 +9308,9 @@ bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
return true;
}
-void SelectionDAGBuilder::visitCall(const CallInst &I) {
+void SelectionDAGBuilder::visitCall(const CallInst &I) { visitCallBase(I); }
+
+void SelectionDAGBuilder::visitCallBase(const CallBase &I) {
// Handle inline assembly differently.
if (I.isInlineAsm()) {
visitInlineAsm(I);
@@ -10478,7 +10473,7 @@ void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
}
-void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
+void SelectionDAGBuilder::visitVAStart(const CallBase &I) {
DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
@@ -10500,14 +10495,14 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
setValue(&I, V);
}
-void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
+void SelectionDAGBuilder::visitVAEnd(const CallBase &I) {
DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
DAG.getSrcValue(I.getArgOperand(0))));
}
-void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
+void SelectionDAGBuilder::visitVACopy(const CallBase &I) {
DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
@@ -10625,7 +10620,7 @@ static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
}
/// Lower llvm.experimental.stackmap.
-void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
+void SelectionDAGBuilder::visitStackmap(const CallBase &CI) {
// void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
// [live variables...])
@@ -10837,7 +10832,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
FuncInfo.MF->getFrameInfo().setHasPatchPoint();
}
-void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
+void SelectionDAGBuilder::visitVectorReduce(const CallBase &I,
unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue Op1 = getValue(I.getArgOperand(0));
@@ -12521,14 +12516,14 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
}
}
-void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
+void SelectionDAGBuilder::visitStepVector(const CallBase &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto DL = getCurSDLoc();
EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
setValue(&I, DAG.getStepVector(DL, ResultVT));
}
-void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
+void SelectionDAGBuilder::visitVectorReverse(const CallBase &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
@@ -12551,7 +12546,7 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
}
-void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
+void SelectionDAGBuilder::visitVectorDeinterleave(const CallBase &I,
unsigned Factor) {
auto DL = getCurSDLoc();
SDValue InVec = getValue(I.getOperand(0));
@@ -12587,7 +12582,7 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
setValue(&I, Res);
}
-void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
+void SelectionDAGBuilder::visitVectorInterleave(const CallBase &I,
unsigned Factor) {
auto DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -12641,7 +12636,7 @@ void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
DAG.getVTList(ValueVTs), Values));
}
-void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
+void SelectionDAGBuilder::visitVectorSplice(const CallBase &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
@@ -12715,7 +12710,7 @@ static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
// otherwise we will end up with copies of virtregs only valid along direct
// edges.
-void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
+void SelectionDAGBuilder::visitCallBrLandingPad(const CallBase &I) {
SmallVector<EVT, 8> ResultVTs;
SmallVector<SDValue, 8> ResultValues;
const auto *CBR =
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 35c15bc269d4b..483e0b9d2db6c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -483,7 +483,7 @@ class SelectionDAGBuilder {
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee,
const BasicBlock *EHPadBB);
- void LowerDeoptimizeCall(const CallInst *CI);
+ void LowerDeoptimizeCall(const CallBase *CI);
void LowerDeoptimizingReturn();
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee,
@@ -537,7 +537,7 @@ class SelectionDAGBuilder {
// These all get lowered before this pass.
void visitInvoke(const InvokeInst &I);
void visitCallBr(const CallBrInst &I);
- void visitCallBrLandingPad(const CallInst &I);
+ void visitCallBrLandingPad(const CallBase &I);
void visitResume(const ResumeInst &I);
void visitUnary(const User &I, unsigned Opcode);
@@ -594,24 +594,25 @@ class SelectionDAGBuilder {
void visitAlloca(const AllocaInst &I);
void visitLoad(const LoadInst &I);
void visitStore(const StoreInst &I);
- void visitMaskedLoad(const CallInst &I, bool IsExpanding = false);
- void visitMaskedStore(const CallInst &I, bool IsCompressing = false);
- void visitMaskedGather(const CallInst &I);
- void visitMaskedScatter(const CallInst &I);
+ void visitMaskedLoad(const CallBase &I, bool IsExpanding = false);
+ void visitMaskedStore(const CallBase &I, bool IsCompressing = false);
+ void visitMaskedGather(const CallBase &I);
+ void visitMaskedScatter(const CallBase &I);
void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
void visitAtomicRMW(const AtomicRMWInst &I);
void visitFence(const FenceInst &I);
void visitPHI(const PHINode &I);
void visitCall(const CallInst &I);
- bool visitMemCmpBCmpCall(const CallInst &I);
- bool visitMemPCpyCall(const CallInst &I);
- bool visitMemChrCall(const CallInst &I);
- bool visitStrCpyCall(const CallInst &I, bool isStpcpy);
- bool visitStrCmpCall(const CallInst &I);
- bool visitStrLenCall(const CallInst &I);
- bool visitStrNLenCall(const CallInst &I);
- bool visitUnaryFloatCall(const CallInst &I, unsigned Opcode);
- bool visitBinaryFloatCall(const CallInst &I, unsigned Opcode);
+ void visitCallBase(const CallBase &I);
+ bool visitMemCmpBCmpCall(const CallBase &I);
+ bool visitMemPCpyCall(const CallBase &I);
+ bool visitMemChrCall(const CallBase &I);
+ bool visitStrCpyCall(const CallBase &I, bool isStpcpy);
+ bool visitStrCmpCall(const CallBase &I);
+ bool visitStrLenCall(const CallBase &I);
+ bool visitStrNLenCall(const CallBase &I);
+ bool visitUnaryFloatCall(const CallBase &I, unsigned Opcode);
+ bool visitBinaryFloatCall(const CallBase &I, unsigned Opcode);
void visitAtomicLoad(const LoadInst &I);
void visitAtomicStore(const StoreInst &I);
void visitLoadFromSwiftError(const LoadInst &I);
@@ -624,12 +625,12 @@ class SelectionDAGBuilder {
bool visitEntryValueDbgValue(ArrayRef<const Value *> Values,
DILocalVariable *Variable, DIExpression *Expr,
DebugLoc DbgLoc);
- void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
- void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
+ void visitIntrinsicCall(const CallBase &I, unsigned Intrinsic);
+ void visitTargetIntrinsic(const CallBase &I, unsigned Intrinsic);
void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
- void visitConvergenceControl(const CallInst &I, unsigned Intrinsic);
- void visitVectorHistogram(const CallInst &I, unsigned IntrinsicID);
- void visitVectorExtractLastActive(const CallInst &I, unsigned Intrinsic);
+ void visitConvergenceControl(const CallBase &I, unsigned Intrinsic);
+ void visitVectorHistogram(const CallBase &I, unsigned IntrinsicID);
+ void visitVectorExtractLastActive(const CallBase &I, unsigned Intrinsic);
void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT,
const SmallVectorImpl<SDValue> &OpValues);
void visitVPStore(const VPIntrinsic &VPIntrin,
@@ -645,23 +646,23 @@ class SelectionDAGBuilder {
void visitVPCmp(const VPCmpIntrinsic &VPIntrin);
void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin);
- void visitVAStart(const CallInst &I);
+ void visitVAStart(const CallBase &I);
void visitVAArg(const VAArgInst &I);
- void visitVAEnd(const CallInst &I);
- void visitVACopy(const CallInst &I);
- void visitStackmap(const CallInst &I);
+ void visitVAEnd(const CallBase &I);
+ void visitVACopy(const CallBase &I);
+ void visitStackmap(const CallBase &I);
void visitPatchpoint(const CallBase &CB, const BasicBlock *EHPadBB = nullptr);
// These two are implemented in StatepointLowering.cpp
void visitGCRelocate(const GCRelocateInst &Relocate);
void visitGCResult(const GCResultInst &I);
- void visitVectorReduce(const CallInst &I, unsigned Intrinsic);
- void visitVectorReverse(const CallInst &I);
- void visitVectorSplice(const CallInst &I);
- void visitVectorInterleave(const CallInst &I, unsigned Factor);
- void visitVectorDeinterleave(const CallInst &I, unsigned Factor);
- void visitStepVector(const CallInst &I);
+ void visitVectorReduce(const CallBase &I, unsigned Intrinsic);
+ void visitVectorReverse(const CallBase &I);
+ void visitVectorSplice(const CallBase &I);
+ void visitVectorInterleave(const CallBase &I, unsigned Factor);
+ void visitVectorDeinterleave(const CallBase &I, unsigned Factor);
+ void visitStepVector(const CallBase &I);
void visitUserOp1(const Instruction &I) {
llvm_unreachable("UserOp1 should not exist at instruction selection time!");
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 80aeefe8e068a..5227fd6a60e7f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -1303,7 +1303,7 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
setValue(&Relocate, SD);
}
-void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) {
+void SelectionDAGBuilder::LowerDeoptimizeCall(const CallBase *CI) {
const auto &TLI = DAG.getTargetLoweringInfo();
SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
TLI.getPointerTy(DAG.getDataLayout()));
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 22d0bc9914585..49f0aaa5387e6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5683,7 +5683,7 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
}
void TargetLowering::CollectTargetIntrinsicOperands(
- const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
+ const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
}
std::pair<unsigned, const TargetRegisterClass *>
diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp
index 4315f63cce4f8..05bd88981b24a 100644
--- a/llvm/lib/IR/DiagnosticInfo.cpp
+++ b/llvm/lib/IR/DiagnosticInfo.cpp
@@ -456,7 +456,7 @@ void DiagnosticInfoMisExpect::print(DiagnosticPrinter &DP) const {
void OptimizationRemarkAnalysisFPCommute::anchor() {}
void OptimizationRemarkAnalysisAliasing::anchor() {}
-void llvm::diagnoseDontCall(const CallInst &CI) {
+void llvm::diagnoseDontCall(const CallBase &CI) {
const auto *F =
dyn_cast<Function>(CI.getCalledOperand()->stripPointerCasts());
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4dacd2273306e..3a5d87e518055 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -16283,7 +16283,7 @@ SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
template <unsigned NumVecs>
static bool
setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
- AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) {
+ AArch64TargetLowering::IntrinsicInfo &Info, const CallBase &CI) {
Info.opc = ISD::INTRINSIC_VOID;
// Retrieve EC from first vector argument.
const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType());
@@ -16308,7 +16308,7 @@ setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
/// specified in the intrinsic calls.
bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index b59526bf01888..30b7c6d2e7a65 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -680,7 +680,7 @@ class AArch64TargetLowering : public TargetLowering {
EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *MBB) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index ade88a16193b8..0b001d3a08aa7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1214,7 +1214,7 @@ MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
}
bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &CI,
+ const CallBase &CI,
MachineFunction &MF,
unsigned IntrID) const {
Info.flags = MachineMemOperand::MONone;
@@ -1501,7 +1501,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
}
void SITargetLowering::CollectTargetIntrinsicOperands(
- const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
+ const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
switch (cast<IntrinsicInst>(I).getIntrinsicID()) {
case Intrinsic::amdgcn_addrspacecast_nonnull: {
// The DAG's ValueType loses the addrspaces.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index c42366a1c04c8..6109dfa6efe8f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -310,11 +310,11 @@ class SITargetLowering final : public AMDGPUTargetLowering {
MVT getPointerTy(const DataLayout &DL, unsigned AS) const override;
MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
+ bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
MachineFunction &MF,
unsigned IntrinsicID) const override;
- void CollectTargetIntrinsicOperands(const CallInst &I,
+ void CollectTargetIntrinsicOperands(const CallBase &I,
SmallVectorImpl<SDValue> &Ops,
SelectionDAG &DAG) const override;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 1f7ab8ce3a0e0..900b6f21de8e9 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -20953,7 +20953,7 @@ bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
/// specified in the intrinsic calls.
bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 9fad056edd3f1..883dbaa49d2ed 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -616,8 +616,7 @@ class VectorType;
bool isFPImmLegal(const APFloat &Imm, EVT VT,
bool ForCodeSize = false) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 01efcedebc808..ae81b53c47a4d 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -2083,7 +2083,7 @@ static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
/// true and store the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 1321bee44a295..9f315681ab95f 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -142,7 +142,7 @@ class HexagonTargetLowering : public TargetLowering {
const SmallVectorImpl<SDValue> &OutVals,
const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 9f5c94ddea44f..3284b7fa801d6 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -6918,7 +6918,7 @@ bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
}
bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 6bf295984dfc5..4599db4656ef3 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -226,7 +226,7 @@ class LoongArchTargetLowering : public TargetLowering {
Value *NewVal, Value *Mask,
AtomicOrdering Ord) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index d7ca59ca04cf3..ec2cd3e628fbc 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -3655,9 +3655,10 @@ void NVPTXTargetLowering::LowerAsmOperandForConstraint(
// because we need the information that is only available in the "Value" type
// of destination
// pointer. In particular, the address space information.
-bool NVPTXTargetLowering::getTgtMemIntrinsic(
- IntrinsicInfo &Info, const CallInst &I,
- MachineFunction &MF, unsigned Intrinsic) const {
+bool NVPTXTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
+ const CallBase &I,
+ MachineFunction &MF,
+ unsigned Intrinsic) const {
switch (Intrinsic) {
default:
return false;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 8d71022a1f102..0530cecde00e7 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -124,7 +124,7 @@ class NVPTXTargetLowering : public TargetLowering {
const char *getTargetNodeName(unsigned Opcode) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index c39b9d55cc212..e1c1f6929807d 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -17664,9 +17664,8 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
}
-void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I,
- SmallVectorImpl<SDValue> &Ops,
- SelectionDAG &DAG) const {
+void PPCTargetLowering::CollectTargetIntrinsicOperands(
+ const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
if (I.getNumOperands() <= 1)
return;
if (!isa<ConstantSDNode>(Ops[1].getNode()))
@@ -17854,7 +17853,7 @@ PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
}
bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 2c55b5427297a..404b3c8ae59fd 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -1014,9 +1014,9 @@ namespace llvm {
return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
}
- void CollectTargetIntrinsicOperands(const CallInst &I,
- SmallVectorImpl<SDValue> &Ops,
- SelectionDAG &DAG) const override;
+ void CollectTargetIntrinsicOperands(const CallBase &I,
+ SmallVectorImpl<SDValue> &Ops,
+ SelectionDAG &DAG) const override;
/// isLegalAddressingMode - Return true if the addressing mode represented
/// by AM is legal for this target, for a load/store of the specified type.
@@ -1075,8 +1075,7 @@ namespace llvm {
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 0a849f49116ee..45893a879afc8 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1709,7 +1709,7 @@ bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
}
bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 78f2044ba83a7..8c1a629864bdd 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -35,7 +35,7 @@ class RISCVTargetLowering : public TargetLowering {
const RISCVSubtarget &getSubtarget() const { return Subtarget; }
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 8a873426e78d8..36ae2e3eccc03 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -91,7 +91,7 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
}
bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const Callase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
unsigned AlignIdx = 3;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index 9025e6eb0842e..d6e10d611704a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -49,7 +49,7 @@ class SPIRVTargetLowering : public TargetLowering {
EVT VT) const override;
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
EVT VT) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index aac3473311192..c77a9375fbce8 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1034,7 +1034,7 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
}
bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index 72401a7a259c0..f8f23c932177b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -72,7 +72,7 @@ class WebAssemblyTargetLowering final : public TargetLowering {
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
EVT VT) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 92e980574a187..e1971d12818cb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3105,7 +3105,7 @@ static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
}
bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallInst &I,
+ const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const {
Info.flags = MachineMemOperand::MONone;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 359f24768b3da..95ddde8b17e90 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1474,7 +1474,7 @@ namespace llvm {
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and stores the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
>From 284d18ec922d05b19e35347bc3e415ece82bc4ad Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 2 Apr 2025 05:04:29 -0500
Subject: [PATCH 04/13] adapt and more tests
---
llvm/test/Assembler/callbr.ll | 22 +++++++++++++++++
llvm/test/CodeGen/AMDGPU/callbr.ll | 22 ++---------------
llvm/test/Verifier/callbr.ll | 39 ++++++++++++++++++++++++++++++
3 files changed, 63 insertions(+), 20 deletions(-)
create mode 100644 llvm/test/Assembler/callbr.ll
diff --git a/llvm/test/Assembler/callbr.ll b/llvm/test/Assembler/callbr.ll
new file mode 100644
index 0000000000000..0084e9763c62c
--- /dev/null
+++ b/llvm/test/Assembler/callbr.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+
+declare void @llvm.amdgcn.kill(i1)
+
+define void @test_kill(i1 %c) {
+; CHECK-LABEL: define void @test_kill(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: callbr void @llvm.amdgcn.kill(i1 [[C]])
+; CHECK-NEXT: to label %[[CONT:.*]] [label %kill]
+; CHECK: [[KILL:.*:]]
+; CHECK-NEXT: unreachable
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+;
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+kill:
+ unreachable
+cont:
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll
index e2e84dca96cbf..33c75f6cf5aab 100644
--- a/llvm/test/CodeGen/AMDGPU/callbr.ll
+++ b/llvm/test/CodeGen/AMDGPU/callbr.ll
@@ -1,14 +1,5 @@
-; RUN: rm -rf %t && split-file %s %t
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-seldag.s < %t/with-callbr.ll
-; RUN: FileCheck --check-prefix=SELDAG %s < %t/with-callbr-seldag.s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-gisel.s -global-isel < %t/with-callbr.ll
-; RUN: FileCheck --check-prefix=GISEL %s < %t/with-callbr-gisel.s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-seldag.s < %t/without-callbr.ll
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-gisel.s -global-isel < %t/without-callbr.ll
-; RUN: diff %t/with-callbr-seldag.s %t/without-callbr-seldag.s
-; RUN: diff %t/with-callbr-gisel.s %t/without-callbr-gisel.s
-
-;--- with-callbr.ll
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o - < %s | FileCheck --check-prefix=SELDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o - -global-isel < %s | FileCheck --check-prefix=GISEL %s
; SELDAG-LABEL: test_kill:
; SELDAG-NEXT: ; %bb.0:
@@ -59,12 +50,3 @@ cont:
store i32 %a, ptr %dst, align 4
ret void
}
-
-;--- without-callbr.ll
-
-define void @test_kill(ptr %src, ptr %dst, i1 %c) {
- %a = load i32, ptr %src, align 4
- call void @llvm.amdgcn.kill(i1 %c)
- store i32 %a, ptr %dst, align 4
- ret void
-}
diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll
index 9b819c5fed48b..8e125e723e6fc 100644
--- a/llvm/test/Verifier/callbr.ll
+++ b/llvm/test/Verifier/callbr.ll
@@ -120,3 +120,42 @@ landingpad:
%out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
ret i32 %out
}
+
+declare void @llvm.amdgcn.kill(i1)
+
+; CHECK-NEXT: Callbr amdgcn_kill only supports one indirect dest
+define void @test_callbr_intrinsic_indirect0(i1 %c) {
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont []
+kill:
+ unreachable
+cont:
+ ret void
+}
+
+; CHECK-NEXT: Callbr amdgcn_kill only supports one indirect dest
+define void @test_callbr_intrinsic_indirect2(i1 %c) {
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill1, label %kill2]
+kill1:
+ unreachable
+kill2:
+ unreachable
+cont:
+ ret void
+}
+
+; CHECK-NEXT: Callbr amdgcn_kill indirect dest needs to be unreachable
+define void @test_callbr_intrinsic_no_unreachable(i1 %c) {
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+kill:
+ ret void
+cont:
+ ret void
+}
+
+; CHECK-NEXT: Callbr currently only supports asm-goto and selected intrinsics
+declare i32 @llvm.amdgcn.workitem.id.x()
+define void @test_callbr_intrinsic_unsupported() {
+ callbr i32 @llvm.amdgcn.workitem.id.x() to label %cont []
+cont:
+ ret void
+}
>From 789a7297627a4591a0e3d8a2eaa9736ede7a93d7 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 3 Apr 2025 02:44:58 -0500
Subject: [PATCH 05/13] Revert "implement feedback"
This reverts commit db57ce26f1780f285db04481f22c56d8262d60ba.
---
llvm/include/llvm/CodeGen/Analysis.h | 2 +-
.../llvm/CodeGen/GlobalISel/IRTranslator.h | 18 ++---
llvm/include/llvm/CodeGen/SelectionDAG.h | 6 +-
llvm/include/llvm/CodeGen/TargetLowering.h | 6 +-
llvm/include/llvm/IR/DiagnosticInfo.h | 4 +-
llvm/include/llvm/IR/IntrinsicInst.h | 8 +--
llvm/lib/CodeGen/Analysis.cpp | 2 +-
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 34 +++++----
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 71 ++++++++++---------
.../SelectionDAG/SelectionDAGBuilder.h | 61 ++++++++--------
.../SelectionDAG/StatepointLowering.cpp | 2 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 2 +-
llvm/lib/IR/DiagnosticInfo.cpp | 2 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 4 +-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +-
llvm/lib/Target/ARM/ARMISelLowering.cpp | 2 +-
llvm/lib/Target/ARM/ARMISelLowering.h | 3 +-
.../Target/Hexagon/HexagonISelLowering.cpp | 2 +-
llvm/lib/Target/Hexagon/HexagonISelLowering.h | 2 +-
.../LoongArch/LoongArchISelLowering.cpp | 2 +-
.../Target/LoongArch/LoongArchISelLowering.h | 2 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 7 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 2 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 7 +-
llvm/lib/Target/PowerPC/PPCISelLowering.h | 9 +--
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 2 +-
.../WebAssembly/WebAssemblyISelLowering.cpp | 2 +-
.../WebAssembly/WebAssemblyISelLowering.h | 2 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
llvm/lib/Target/X86/X86ISelLowering.h | 2 +-
36 files changed, 150 insertions(+), 142 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index a4a604dd2c608..362cc30bbd06a 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -150,7 +150,7 @@ bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I,
/// Returns true if the parent of \p CI returns CI's first argument after
/// calling \p CI.
-bool funcReturnsFirstArgOfCall(const CallBase &CI);
+bool funcReturnsFirstArgOfCall(const CallInst &CI);
DenseMap<const MachineBasicBlock *, int>
getEHScopeMembership(const MachineFunction &MF);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index d3f731cf02be9..6fd05c8fddd5f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -235,26 +235,26 @@ class IRTranslator : public MachineFunctionPass {
bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
/// Translate an LLVM string intrinsic (memcpy, memset, ...).
- bool translateMemFunc(const CallBase &CI, MachineIRBuilder &MIRBuilder,
+ bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
unsigned Opcode);
/// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
- bool translateTrap(const CallBase &U, MachineIRBuilder &MIRBuilder,
+ bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
unsigned Opcode);
// Translate @llvm.vector.interleave2 and
// @llvm.vector.deinterleave2 intrinsics for fixed-width vector
// types into vector shuffles.
- bool translateVectorInterleave2Intrinsic(const CallBase &CI,
+ bool translateVectorInterleave2Intrinsic(const CallInst &CI,
MachineIRBuilder &MIRBuilder);
- bool translateVectorDeinterleave2Intrinsic(const CallBase &CI,
+ bool translateVectorDeinterleave2Intrinsic(const CallInst &CI,
MachineIRBuilder &MIRBuilder);
void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
- bool translateOverflowIntrinsic(const CallBase &CI, unsigned Op,
+ bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
MachineIRBuilder &MIRBuilder);
- bool translateFixedPointIntrinsic(unsigned Op, const CallBase &CI,
+ bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
MachineIRBuilder &MIRBuilder);
/// Helper function for translateSimpleIntrinsic.
@@ -265,13 +265,13 @@ class IRTranslator : public MachineFunctionPass {
/// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
/// \return true if the translation succeeded.
- bool translateSimpleIntrinsic(const CallBase &CI, Intrinsic::ID ID,
+ bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI,
MachineIRBuilder &MIRBuilder);
- bool translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID,
+ bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
/// Returns the single livein physical register Arg was lowered to, if
@@ -588,7 +588,7 @@ class IRTranslator : public MachineFunctionPass {
return false;
}
- bool translateConvergenceControlIntrinsic(const CallBase &CI,
+ bool translateConvergenceControlIntrinsic(const CallInst &CI,
Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 00f25e753f292..87b6914f8a0ee 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1238,7 +1238,7 @@ class SelectionDAG {
* the tail call optimization decision. */
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, const CallBase *CI,
+ bool AlwaysInline, const CallInst *CI,
std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
@@ -1250,7 +1250,7 @@ class SelectionDAG {
* the tail call optimization decision. */
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- const CallBase *CI, std::optional<bool> OverrideTailCall,
+ const CallInst *CI, std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1258,7 +1258,7 @@ class SelectionDAG {
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, const CallBase *CI,
+ bool AlwaysInline, const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes());
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index dfc4320ed92f5..d9fce57c6e43a 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1239,7 +1239,7 @@ class TargetLoweringBase {
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and store the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
- virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
+ virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
MachineFunction &,
unsigned /*Intrinsic*/) const {
return false;
@@ -5222,9 +5222,9 @@ class TargetLowering : public TargetLoweringBase {
const AsmOperandInfo &OpInfo,
SelectionDAG &DAG) const;
- // Targets may override this function to collect operands from the CallBase
+ // Targets may override this function to collect operands from the CallInst
// and for example, lower them into the SelectionDAG operands.
- virtual void CollectTargetIntrinsicOperands(const CallBase &I,
+ virtual void CollectTargetIntrinsicOperands(const CallInst &I,
SmallVectorImpl<SDValue> &Ops,
SelectionDAG &DAG) const;
diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h
index b43ef657162c8..a1113134f6a34 100644
--- a/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -38,7 +38,7 @@ namespace llvm {
class DiagnosticPrinter;
class DIFile;
class DISubprogram;
-class CallBase;
+class CallInst;
class Function;
class Instruction;
class InstructionCost;
@@ -1151,7 +1151,7 @@ class DiagnosticInfoSrcMgr : public DiagnosticInfo {
}
};
-void diagnoseDontCall(const CallBase &CI);
+void diagnoseDontCall(const CallInst &CI);
class DiagnosticInfoDontCall : public DiagnosticInfo {
StringRef CalleeName;
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 4828240f06967..1217a628dcb29 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -134,14 +134,8 @@ class IntrinsicInst : public CallInst {
return CF->isIntrinsic();
return false;
}
- static bool classof(const CallBase *I) {
- if (const Function *CF = I->getCalledFunction())
- return CF->isIntrinsic();
- return false;
- }
static bool classof(const Value *V) {
- return (isa<CallInst>(V) && classof(cast<CallInst>(V))) ||
- (isa<CallBase>(V) && classof(cast<CallBase>(V)));
+ return isa<CallInst>(V) && classof(cast<CallInst>(V));
}
};
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index f950bb756fb2b..e7b9417de8c9f 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -712,7 +712,7 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
return true;
}
-bool llvm::funcReturnsFirstArgOfCall(const CallBase &CI) {
+bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
bool ReturnsFirstArg = false;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 1b6efe1c91fc4..3b33d3784c1f9 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1693,7 +1693,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
return true;
}
-bool IRTranslator::translateMemFunc(const CallBase &CI,
+bool IRTranslator::translateMemFunc(const CallInst &CI,
MachineIRBuilder &MIRBuilder,
unsigned Opcode) {
const Value *SrcPtr = CI.getArgOperand(1);
@@ -1780,7 +1780,7 @@ bool IRTranslator::translateMemFunc(const CallBase &CI,
return true;
}
-bool IRTranslator::translateTrap(const CallBase &CI,
+bool IRTranslator::translateTrap(const CallInst &CI,
MachineIRBuilder &MIRBuilder,
unsigned Opcode) {
StringRef TrapFuncName =
@@ -1807,7 +1807,7 @@ bool IRTranslator::translateTrap(const CallBase &CI,
}
bool IRTranslator::translateVectorInterleave2Intrinsic(
- const CallBase &CI, MachineIRBuilder &MIRBuilder) {
+ const CallInst &CI, MachineIRBuilder &MIRBuilder) {
assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
"This function can only be called on the interleave2 intrinsic!");
// Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
@@ -1823,7 +1823,7 @@ bool IRTranslator::translateVectorInterleave2Intrinsic(
}
bool IRTranslator::translateVectorDeinterleave2Intrinsic(
- const CallBase &CI, MachineIRBuilder &MIRBuilder) {
+ const CallInst &CI, MachineIRBuilder &MIRBuilder) {
assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
"This function can only be called on the deinterleave2 intrinsic!");
// Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
@@ -1863,7 +1863,7 @@ void IRTranslator::getStackGuard(Register DstReg,
MIB.setMemRefs({MemRef});
}
-bool IRTranslator::translateOverflowIntrinsic(const CallBase &CI, unsigned Op,
+bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
MachineIRBuilder &MIRBuilder) {
ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
MIRBuilder.buildInstr(
@@ -1873,7 +1873,7 @@ bool IRTranslator::translateOverflowIntrinsic(const CallBase &CI, unsigned Op,
return true;
}
-bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallBase &CI,
+bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
MachineIRBuilder &MIRBuilder) {
Register Dst = getOrCreateVReg(CI);
Register Src0 = getOrCreateVReg(*CI.getOperand(0));
@@ -2022,7 +2022,7 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return Intrinsic::not_intrinsic;
}
-bool IRTranslator::translateSimpleIntrinsic(const CallBase &CI,
+bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
@@ -2144,7 +2144,7 @@ static unsigned getConvOpcode(Intrinsic::ID ID) {
}
bool IRTranslator::translateConvergenceControlIntrinsic(
- const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
+ const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
MachineInstrBuilder MIB = MIRBuilder.buildInstr(getConvOpcode(ID));
Register OutputReg = getOrCreateConvergenceTokenVReg(CI);
MIB.addDef(OutputReg);
@@ -2160,7 +2160,7 @@ bool IRTranslator::translateConvergenceControlIntrinsic(
return true;
}
-bool IRTranslator::translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID,
+bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
if (ORE->enabled()) {
@@ -2754,7 +2754,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
if (containsBF16Type(U))
return false;
- const CallBase &CI = cast<CallBase>(U);
+ const CallInst &CI = cast<CallInst>(U);
const Function *F = CI.getCalledFunction();
// FIXME: support Windows dllimport function calls and calls through
@@ -3022,14 +3022,22 @@ bool IRTranslator::translateCallBr(const User &U,
} else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
switch (I.getIntrinsicID()) {
default:
- return false;
+ report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
- if (!translateCall(I, MIRBuilder))
+ if (I.getNumIndirectDests() != 1)
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
+ CallInst *CI =
+ CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
+ SmallVector<Value *, 1>(I.args()));
+ bool Success = translateCall(*CI, MIRBuilder);
+ CI->deleteValue();
+ if (!Success)
return false;
break;
}
} else {
- return false;
+ report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
}
// Retrieve successors.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index e14b71106e975..e6a7d092b7b79 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8674,7 +8674,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
SDValue SelectionDAG::getMemcpy(
SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
- Align Alignment, bool isVol, bool AlwaysInline, const CallBase *CI,
+ Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
BatchAAResults *BatchAA) {
@@ -8800,7 +8800,7 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, const CallBase *CI,
+ bool isVol, const CallInst *CI,
std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
@@ -8918,7 +8918,7 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
bool isVol, bool AlwaysInline,
- const CallBase *CI,
+ const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo) {
// Check to see if we should lower the memset to stores first.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1f41d57c0872c..68f6a0445b1af 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3391,7 +3391,14 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
default:
report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
- visitCallBase(I);
+ if (I.getNumIndirectDests() != 1)
+ report_fatal_error(
+ "amdgcn.kill supportes exactly one indirect destination");
+ CallInst *CI =
+ CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
+ SmallVector<Value *, 1>(I.args()));
+ visitCall(*CI);
+ CI->deleteValue();
break;
}
} else {
@@ -4750,7 +4757,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
DAG.setRoot(StoreNode);
}
-void SelectionDAGBuilder::visitMaskedStore(const CallBase &I,
+void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
bool IsCompressing) {
SDLoc sdl = getCurSDLoc();
@@ -4881,7 +4888,7 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
return true;
}
-void SelectionDAGBuilder::visitMaskedScatter(const CallBase &I) {
+void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
SDLoc sdl = getCurSDLoc();
// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
@@ -4926,7 +4933,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallBase &I) {
setValue(&I, Scatter);
}
-void SelectionDAGBuilder::visitMaskedLoad(const CallBase &I, bool IsExpanding) {
+void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
SDLoc sdl = getCurSDLoc();
auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
@@ -4995,7 +5002,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallBase &I, bool IsExpanding) {
setValue(&I, Res);
}
-void SelectionDAGBuilder::visitMaskedGather(const CallBase &I) {
+void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
SDLoc sdl = getCurSDLoc();
// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
@@ -5229,7 +5236,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
/// node.
-void SelectionDAGBuilder::visitTargetIntrinsic(const CallBase &I,
+void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
unsigned Intrinsic) {
// Ignore the callsite's attributes. A specific call site may be marked with
// readnone, but the lowering code will expect the chain based on the
@@ -6312,7 +6319,7 @@ bool SelectionDAGBuilder::visitEntryValueDbgValue(
}
/// Lower the call to the specified intrinsic function.
-void SelectionDAGBuilder::visitConvergenceControl(const CallBase &I,
+void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
unsigned Intrinsic) {
SDLoc sdl = getCurSDLoc();
switch (Intrinsic) {
@@ -6332,7 +6339,7 @@ void SelectionDAGBuilder::visitConvergenceControl(const CallBase &I,
}
}
-void SelectionDAGBuilder::visitVectorHistogram(const CallBase &I,
+void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
unsigned IntrinsicID) {
// For now, we're only lowering an 'add' histogram.
// We can add others later, e.g. saturating adds, min/max.
@@ -6390,7 +6397,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallBase &I,
DAG.setRoot(Histogram);
}
-void SelectionDAGBuilder::visitVectorExtractLastActive(const CallBase &I,
+void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
unsigned Intrinsic) {
assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
"Tried lowering invalid vector extract last");
@@ -6418,7 +6425,7 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallBase &I,
}
/// Lower the call to the specified intrinsic function.
-void SelectionDAGBuilder::visitIntrinsicCall(const CallBase &I,
+void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDLoc sdl = getCurSDLoc();
@@ -9028,7 +9035,7 @@ void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallBase &I) {
+bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
const Value *Size = I.getArgOperand(2);
const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
@@ -9120,7 +9127,7 @@ bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallBase &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemChrCall(const CallBase &I) {
+bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
const Value *Src = I.getArgOperand(0);
const Value *Char = I.getArgOperand(1);
const Value *Length = I.getArgOperand(2);
@@ -9144,7 +9151,7 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallBase &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitMemPCpyCall(const CallBase &I) {
+bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
SDValue Dst = getValue(I.getArgOperand(0));
SDValue Src = getValue(I.getArgOperand(1));
SDValue Size = getValue(I.getArgOperand(2));
@@ -9183,7 +9190,7 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallBase &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrCpyCall(const CallBase &I, bool isStpcpy) {
+bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9206,7 +9213,7 @@ bool SelectionDAGBuilder::visitStrCpyCall(const CallBase &I, bool isStpcpy) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrCmpCall(const CallBase &I) {
+bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9229,7 +9236,7 @@ bool SelectionDAGBuilder::visitStrCmpCall(const CallBase &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrLenCall(const CallBase &I) {
+bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
const Value *Arg0 = I.getArgOperand(0);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9250,7 +9257,7 @@ bool SelectionDAGBuilder::visitStrLenCall(const CallBase &I) {
/// normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitStrNLenCall(const CallBase &I) {
+bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
@@ -9272,7 +9279,7 @@ bool SelectionDAGBuilder::visitStrNLenCall(const CallBase &I) {
/// false and it will be lowered like a normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitUnaryFloatCall(const CallBase &I,
+bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
@@ -9292,7 +9299,7 @@ bool SelectionDAGBuilder::visitUnaryFloatCall(const CallBase &I,
/// false, and it will be lowered like a normal call.
/// The caller already checked that \p I calls the appropriate LibFunc with a
/// correct prototype.
-bool SelectionDAGBuilder::visitBinaryFloatCall(const CallBase &I,
+bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
@@ -9308,9 +9315,7 @@ bool SelectionDAGBuilder::visitBinaryFloatCall(const CallBase &I,
return true;
}
-void SelectionDAGBuilder::visitCall(const CallInst &I) { visitCallBase(I); }
-
-void SelectionDAGBuilder::visitCallBase(const CallBase &I) {
+void SelectionDAGBuilder::visitCall(const CallInst &I) {
// Handle inline assembly differently.
if (I.isInlineAsm()) {
visitInlineAsm(I);
@@ -10473,7 +10478,7 @@ void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
}
-void SelectionDAGBuilder::visitVAStart(const CallBase &I) {
+void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
@@ -10495,14 +10500,14 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
setValue(&I, V);
}
-void SelectionDAGBuilder::visitVAEnd(const CallBase &I) {
+void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
DAG.getSrcValue(I.getArgOperand(0))));
}
-void SelectionDAGBuilder::visitVACopy(const CallBase &I) {
+void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
@@ -10620,7 +10625,7 @@ static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
}
/// Lower llvm.experimental.stackmap.
-void SelectionDAGBuilder::visitStackmap(const CallBase &CI) {
+void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
// void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
// [live variables...])
@@ -10832,7 +10837,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
FuncInfo.MF->getFrameInfo().setHasPatchPoint();
}
-void SelectionDAGBuilder::visitVectorReduce(const CallBase &I,
+void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue Op1 = getValue(I.getArgOperand(0));
@@ -12516,14 +12521,14 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
}
}
-void SelectionDAGBuilder::visitStepVector(const CallBase &I) {
+void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto DL = getCurSDLoc();
EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
setValue(&I, DAG.getStepVector(DL, ResultVT));
}
-void SelectionDAGBuilder::visitVectorReverse(const CallBase &I) {
+void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
@@ -12546,7 +12551,7 @@ void SelectionDAGBuilder::visitVectorReverse(const CallBase &I) {
setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
}
-void SelectionDAGBuilder::visitVectorDeinterleave(const CallBase &I,
+void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
unsigned Factor) {
auto DL = getCurSDLoc();
SDValue InVec = getValue(I.getOperand(0));
@@ -12582,7 +12587,7 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallBase &I,
setValue(&I, Res);
}
-void SelectionDAGBuilder::visitVectorInterleave(const CallBase &I,
+void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
unsigned Factor) {
auto DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -12636,7 +12641,7 @@ void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
DAG.getVTList(ValueVTs), Values));
}
-void SelectionDAGBuilder::visitVectorSplice(const CallBase &I) {
+void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
@@ -12710,7 +12715,7 @@ static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
// otherwise we will end up with copies of virtregs only valid along direct
// edges.
-void SelectionDAGBuilder::visitCallBrLandingPad(const CallBase &I) {
+void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
SmallVector<EVT, 8> ResultVTs;
SmallVector<SDValue, 8> ResultValues;
const auto *CBR =
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 483e0b9d2db6c..35c15bc269d4b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -483,7 +483,7 @@ class SelectionDAGBuilder {
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee,
const BasicBlock *EHPadBB);
- void LowerDeoptimizeCall(const CallBase *CI);
+ void LowerDeoptimizeCall(const CallInst *CI);
void LowerDeoptimizingReturn();
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee,
@@ -537,7 +537,7 @@ class SelectionDAGBuilder {
// These all get lowered before this pass.
void visitInvoke(const InvokeInst &I);
void visitCallBr(const CallBrInst &I);
- void visitCallBrLandingPad(const CallBase &I);
+ void visitCallBrLandingPad(const CallInst &I);
void visitResume(const ResumeInst &I);
void visitUnary(const User &I, unsigned Opcode);
@@ -594,25 +594,24 @@ class SelectionDAGBuilder {
void visitAlloca(const AllocaInst &I);
void visitLoad(const LoadInst &I);
void visitStore(const StoreInst &I);
- void visitMaskedLoad(const CallBase &I, bool IsExpanding = false);
- void visitMaskedStore(const CallBase &I, bool IsCompressing = false);
- void visitMaskedGather(const CallBase &I);
- void visitMaskedScatter(const CallBase &I);
+ void visitMaskedLoad(const CallInst &I, bool IsExpanding = false);
+ void visitMaskedStore(const CallInst &I, bool IsCompressing = false);
+ void visitMaskedGather(const CallInst &I);
+ void visitMaskedScatter(const CallInst &I);
void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
void visitAtomicRMW(const AtomicRMWInst &I);
void visitFence(const FenceInst &I);
void visitPHI(const PHINode &I);
void visitCall(const CallInst &I);
- void visitCallBase(const CallBase &I);
- bool visitMemCmpBCmpCall(const CallBase &I);
- bool visitMemPCpyCall(const CallBase &I);
- bool visitMemChrCall(const CallBase &I);
- bool visitStrCpyCall(const CallBase &I, bool isStpcpy);
- bool visitStrCmpCall(const CallBase &I);
- bool visitStrLenCall(const CallBase &I);
- bool visitStrNLenCall(const CallBase &I);
- bool visitUnaryFloatCall(const CallBase &I, unsigned Opcode);
- bool visitBinaryFloatCall(const CallBase &I, unsigned Opcode);
+ bool visitMemCmpBCmpCall(const CallInst &I);
+ bool visitMemPCpyCall(const CallInst &I);
+ bool visitMemChrCall(const CallInst &I);
+ bool visitStrCpyCall(const CallInst &I, bool isStpcpy);
+ bool visitStrCmpCall(const CallInst &I);
+ bool visitStrLenCall(const CallInst &I);
+ bool visitStrNLenCall(const CallInst &I);
+ bool visitUnaryFloatCall(const CallInst &I, unsigned Opcode);
+ bool visitBinaryFloatCall(const CallInst &I, unsigned Opcode);
void visitAtomicLoad(const LoadInst &I);
void visitAtomicStore(const StoreInst &I);
void visitLoadFromSwiftError(const LoadInst &I);
@@ -625,12 +624,12 @@ class SelectionDAGBuilder {
bool visitEntryValueDbgValue(ArrayRef<const Value *> Values,
DILocalVariable *Variable, DIExpression *Expr,
DebugLoc DbgLoc);
- void visitIntrinsicCall(const CallBase &I, unsigned Intrinsic);
- void visitTargetIntrinsic(const CallBase &I, unsigned Intrinsic);
+ void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
+ void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
- void visitConvergenceControl(const CallBase &I, unsigned Intrinsic);
- void visitVectorHistogram(const CallBase &I, unsigned IntrinsicID);
- void visitVectorExtractLastActive(const CallBase &I, unsigned Intrinsic);
+ void visitConvergenceControl(const CallInst &I, unsigned Intrinsic);
+ void visitVectorHistogram(const CallInst &I, unsigned IntrinsicID);
+ void visitVectorExtractLastActive(const CallInst &I, unsigned Intrinsic);
void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT,
const SmallVectorImpl<SDValue> &OpValues);
void visitVPStore(const VPIntrinsic &VPIntrin,
@@ -646,23 +645,23 @@ class SelectionDAGBuilder {
void visitVPCmp(const VPCmpIntrinsic &VPIntrin);
void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin);
- void visitVAStart(const CallBase &I);
+ void visitVAStart(const CallInst &I);
void visitVAArg(const VAArgInst &I);
- void visitVAEnd(const CallBase &I);
- void visitVACopy(const CallBase &I);
- void visitStackmap(const CallBase &I);
+ void visitVAEnd(const CallInst &I);
+ void visitVACopy(const CallInst &I);
+ void visitStackmap(const CallInst &I);
void visitPatchpoint(const CallBase &CB, const BasicBlock *EHPadBB = nullptr);
// These two are implemented in StatepointLowering.cpp
void visitGCRelocate(const GCRelocateInst &Relocate);
void visitGCResult(const GCResultInst &I);
- void visitVectorReduce(const CallBase &I, unsigned Intrinsic);
- void visitVectorReverse(const CallBase &I);
- void visitVectorSplice(const CallBase &I);
- void visitVectorInterleave(const CallBase &I, unsigned Factor);
- void visitVectorDeinterleave(const CallBase &I, unsigned Factor);
- void visitStepVector(const CallBase &I);
+ void visitVectorReduce(const CallInst &I, unsigned Intrinsic);
+ void visitVectorReverse(const CallInst &I);
+ void visitVectorSplice(const CallInst &I);
+ void visitVectorInterleave(const CallInst &I, unsigned Factor);
+ void visitVectorDeinterleave(const CallInst &I, unsigned Factor);
+ void visitStepVector(const CallInst &I);
void visitUserOp1(const Instruction &I) {
llvm_unreachable("UserOp1 should not exist at instruction selection time!");
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 5227fd6a60e7f..80aeefe8e068a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -1303,7 +1303,7 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
setValue(&Relocate, SD);
}
-void SelectionDAGBuilder::LowerDeoptimizeCall(const CallBase *CI) {
+void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) {
const auto &TLI = DAG.getTargetLoweringInfo();
SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
TLI.getPointerTy(DAG.getDataLayout()));
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 49f0aaa5387e6..22d0bc9914585 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5683,7 +5683,7 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
}
void TargetLowering::CollectTargetIntrinsicOperands(
- const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
+ const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
}
std::pair<unsigned, const TargetRegisterClass *>
diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp
index 05bd88981b24a..4315f63cce4f8 100644
--- a/llvm/lib/IR/DiagnosticInfo.cpp
+++ b/llvm/lib/IR/DiagnosticInfo.cpp
@@ -456,7 +456,7 @@ void DiagnosticInfoMisExpect::print(DiagnosticPrinter &DP) const {
void OptimizationRemarkAnalysisFPCommute::anchor() {}
void OptimizationRemarkAnalysisAliasing::anchor() {}
-void llvm::diagnoseDontCall(const CallBase &CI) {
+void llvm::diagnoseDontCall(const CallInst &CI) {
const auto *F =
dyn_cast<Function>(CI.getCalledOperand()->stripPointerCasts());
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 3a5d87e518055..4dacd2273306e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -16283,7 +16283,7 @@ SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
template <unsigned NumVecs>
static bool
setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
- AArch64TargetLowering::IntrinsicInfo &Info, const CallBase &CI) {
+ AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) {
Info.opc = ISD::INTRINSIC_VOID;
// Retrieve EC from first vector argument.
const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType());
@@ -16308,7 +16308,7 @@ setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
/// specified in the intrinsic calls.
bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 30b7c6d2e7a65..b59526bf01888 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -680,7 +680,7 @@ class AArch64TargetLowering : public TargetLowering {
EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *MBB) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0b001d3a08aa7..ade88a16193b8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1214,7 +1214,7 @@ MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
}
bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &CI,
+ const CallInst &CI,
MachineFunction &MF,
unsigned IntrID) const {
Info.flags = MachineMemOperand::MONone;
@@ -1501,7 +1501,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
}
void SITargetLowering::CollectTargetIntrinsicOperands(
- const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
+ const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
switch (cast<IntrinsicInst>(I).getIntrinsicID()) {
case Intrinsic::amdgcn_addrspacecast_nonnull: {
// The DAG's ValueType loses the addrspaces.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 6109dfa6efe8f..c42366a1c04c8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -310,11 +310,11 @@ class SITargetLowering final : public AMDGPUTargetLowering {
MVT getPointerTy(const DataLayout &DL, unsigned AS) const override;
MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
+ bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
MachineFunction &MF,
unsigned IntrinsicID) const override;
- void CollectTargetIntrinsicOperands(const CallBase &I,
+ void CollectTargetIntrinsicOperands(const CallInst &I,
SmallVectorImpl<SDValue> &Ops,
SelectionDAG &DAG) const override;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 900b6f21de8e9..1f7ab8ce3a0e0 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -20953,7 +20953,7 @@ bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
/// specified in the intrinsic calls.
bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 883dbaa49d2ed..9fad056edd3f1 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -616,7 +616,8 @@ class VectorType;
bool isFPImmLegal(const APFloat &Imm, EVT VT,
bool ForCodeSize = false) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index ae81b53c47a4d..01efcedebc808 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -2083,7 +2083,7 @@ static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
/// true and store the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 9f315681ab95f..1321bee44a295 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -142,7 +142,7 @@ class HexagonTargetLowering : public TargetLowering {
const SmallVectorImpl<SDValue> &OutVals,
const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 3284b7fa801d6..9f5c94ddea44f 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -6918,7 +6918,7 @@ bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
}
bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 4599db4656ef3..6bf295984dfc5 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -226,7 +226,7 @@ class LoongArchTargetLowering : public TargetLowering {
Value *NewVal, Value *Mask,
AtomicOrdering Ord) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index ec2cd3e628fbc..d7ca59ca04cf3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -3655,10 +3655,9 @@ void NVPTXTargetLowering::LowerAsmOperandForConstraint(
// because we need the information that is only available in the "Value" type
// of destination
// pointer. In particular, the address space information.
-bool NVPTXTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
- MachineFunction &MF,
- unsigned Intrinsic) const {
+bool NVPTXTargetLowering::getTgtMemIntrinsic(
+ IntrinsicInfo &Info, const CallInst &I,
+ MachineFunction &MF, unsigned Intrinsic) const {
switch (Intrinsic) {
default:
return false;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 0530cecde00e7..8d71022a1f102 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -124,7 +124,7 @@ class NVPTXTargetLowering : public TargetLowering {
const char *getTargetNodeName(unsigned Opcode) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index e1c1f6929807d..c39b9d55cc212 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -17664,8 +17664,9 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
}
-void PPCTargetLowering::CollectTargetIntrinsicOperands(
- const CallBase &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
+void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I,
+ SmallVectorImpl<SDValue> &Ops,
+ SelectionDAG &DAG) const {
if (I.getNumOperands() <= 1)
return;
if (!isa<ConstantSDNode>(Ops[1].getNode()))
@@ -17853,7 +17854,7 @@ PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
}
bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 404b3c8ae59fd..2c55b5427297a 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -1014,9 +1014,9 @@ namespace llvm {
return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
}
- void CollectTargetIntrinsicOperands(const CallBase &I,
- SmallVectorImpl<SDValue> &Ops,
- SelectionDAG &DAG) const override;
+ void CollectTargetIntrinsicOperands(const CallInst &I,
+ SmallVectorImpl<SDValue> &Ops,
+ SelectionDAG &DAG) const override;
/// isLegalAddressingMode - Return true if the addressing mode represented
/// by AM is legal for this target, for a load/store of the specified type.
@@ -1075,7 +1075,8 @@ namespace llvm {
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 45893a879afc8..0a849f49116ee 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1709,7 +1709,7 @@ bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
}
bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 8c1a629864bdd..78f2044ba83a7 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -35,7 +35,7 @@ class RISCVTargetLowering : public TargetLowering {
const RISCVSubtarget &getSubtarget() const { return Subtarget; }
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 36ae2e3eccc03..8a873426e78d8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -91,7 +91,7 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
}
bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const Callase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
unsigned AlignIdx = 3;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index d6e10d611704a..9025e6eb0842e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -49,7 +49,7 @@ class SPIRVTargetLowering : public TargetLowering {
EVT VT) const override;
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
EVT VT) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index c77a9375fbce8..aac3473311192 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1034,7 +1034,7 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
}
bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index f8f23c932177b..72401a7a259c0 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -72,7 +72,7 @@ class WebAssemblyTargetLowering final : public TargetLowering {
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
EVT VT) const override;
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e1971d12818cb..92e980574a187 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3105,7 +3105,7 @@ static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
}
bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
- const CallBase &I,
+ const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
Info.flags = MachineMemOperand::MONone;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 95ddde8b17e90..359f24768b3da 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1474,7 +1474,7 @@ namespace llvm {
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and stores the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
- bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
+ bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const override;
>From 17e680d28a82bb2ee5d1f02ae6c1bb49e3e04a89 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Sun, 6 Apr 2025 14:58:41 -0500
Subject: [PATCH 06/13] abstract parts of call code; disallow operand bundles
for callbr in verifier
---
.../llvm/CodeGen/GlobalISel/IRTranslator.h | 5 +
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 158 +++++++-------
.../SelectionDAG/SelectionDAGBuilder.cpp | 192 +++++++++++-------
.../SelectionDAG/SelectionDAGBuilder.h | 14 +-
llvm/lib/IR/Verifier.cpp | 4 +-
llvm/test/Verifier/callbr.ll | 8 +
6 files changed, 237 insertions(+), 144 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 6fd05c8fddd5f..ba2aeac8dffe1 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -297,6 +297,10 @@ class IRTranslator : public MachineFunctionPass {
/// \pre \p U is a call instruction.
bool translateCall(const User &U, MachineIRBuilder &MIRBuilder);
+ bool translateTargetIntrinsic(
+ const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
+ TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo = nullptr);
+
/// When an invoke or a cleanupret unwinds to the next EH pad, there are
/// many places it could ultimately go. In the IR, we have a single unwind
/// destination, but in the machine CFG, we enumerate all the possible blocks.
@@ -313,6 +317,7 @@ class IRTranslator : public MachineFunctionPass {
bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
+ bool translateCallBrIntrinsic(const CallBrInst &I, MachineIRBuilder &MIRBuilder);
bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 3b33d3784c1f9..144203edc0ad7 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2750,59 +2750,27 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
return Success;
}
-bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
- if (containsBF16Type(U))
- return false;
-
- const CallInst &CI = cast<CallInst>(U);
- const Function *F = CI.getCalledFunction();
-
- // FIXME: support Windows dllimport function calls and calls through
- // weak symbols.
- if (F && (F->hasDLLImportStorageClass() ||
- (MF->getTarget().getTargetTriple().isOSWindows() &&
- F->hasExternalWeakLinkage())))
- return false;
-
- // FIXME: support control flow guard targets.
- if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
- return false;
-
- // FIXME: support statepoints and related.
- if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
- return false;
-
- if (CI.isInlineAsm())
- return translateInlineAsm(CI, MIRBuilder);
-
- diagnoseDontCall(CI);
-
- Intrinsic::ID ID = Intrinsic::not_intrinsic;
- if (F && F->isIntrinsic())
- ID = F->getIntrinsicID();
-
- if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
- return translateCallBase(CI, MIRBuilder);
-
- assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
-
- if (translateKnownIntrinsic(CI, ID, MIRBuilder))
- return true;
-
+/// Translate a call or callbr to a target intrinsic.
+/// Depending on whether TLI->getTgtMemIntrinsic() is true, TgtMemIntrinsicInfo
+/// is a pointer to the correspondingly populated IntrinsicInfo object.
+/// Otherwise, this pointer is null.
+bool IRTranslator::translateTargetIntrinsic(
+ const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
+ TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
ArrayRef<Register> ResultRegs;
- if (!CI.getType()->isVoidTy())
- ResultRegs = getOrCreateVRegs(CI);
+ if (!CB.getType()->isVoidTy())
+ ResultRegs = getOrCreateVRegs(CB);
// Ignore the callsite attributes. Backend code is most likely not expecting
// an intrinsic to sometimes have side effects and sometimes not.
MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
- if (isa<FPMathOperator>(CI))
- MIB->copyIRFlags(CI);
+ if (isa<FPMathOperator>(CB))
+ MIB->copyIRFlags(CB);
- for (const auto &Arg : enumerate(CI.args())) {
+ for (const auto &Arg : enumerate(CB.args())) {
// If this is required to be an immediate, don't materialize it in a
// register.
- if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
+ if (CB.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
// imm arguments are more convenient than cimm (and realistically
// probably sufficient), so use them.
@@ -2831,29 +2799,31 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
}
// Add a MachineMemOperand if it is a target mem intrinsic.
- TargetLowering::IntrinsicInfo Info;
- // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
- if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) {
- Align Alignment = Info.align.value_or(
- DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
- LLT MemTy = Info.memVT.isSimple()
- ? getLLTForMVT(Info.memVT.getSimpleVT())
- : LLT::scalar(Info.memVT.getStoreSizeInBits());
+ if (TgtMemIntrinsicInfo) {
+ const Function *F = CB.getCalledFunction();
+
+ Align Alignment = TgtMemIntrinsicInfo->align.value_or(DL->getABITypeAlign(
+ TgtMemIntrinsicInfo->memVT.getTypeForEVT(F->getContext())));
+ LLT MemTy =
+ TgtMemIntrinsicInfo->memVT.isSimple()
+ ? getLLTForMVT(TgtMemIntrinsicInfo->memVT.getSimpleVT())
+ : LLT::scalar(TgtMemIntrinsicInfo->memVT.getStoreSizeInBits());
// TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
// didn't yield anything useful.
MachinePointerInfo MPI;
- if (Info.ptrVal)
- MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
- else if (Info.fallbackAddressSpace)
- MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
+ if (TgtMemIntrinsicInfo->ptrVal)
+ MPI = MachinePointerInfo(TgtMemIntrinsicInfo->ptrVal,
+ TgtMemIntrinsicInfo->offset);
+ else if (TgtMemIntrinsicInfo->fallbackAddressSpace)
+ MPI = MachinePointerInfo(*TgtMemIntrinsicInfo->fallbackAddressSpace);
MIB.addMemOperand(MF->getMachineMemOperand(
- MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata(),
- /*Ranges=*/nullptr, Info.ssid, Info.order, Info.failureOrder));
+ MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CI.getAAMetadata(),
+ /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid, TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder));
}
- if (CI.isConvergent()) {
- if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_convergencectrl)) {
+ if (CB.isConvergent()) {
+ if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
auto *Token = Bundle->Inputs[0].get();
Register TokenReg = getOrCreateVReg(*Token);
MIB.addUse(TokenReg, RegState::Implicit);
@@ -2863,6 +2833,53 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
return true;
}
+bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
+ if (containsBF16Type(U))
+ return false;
+
+ const CallInst &CI = cast<CallInst>(U);
+ const Function *F = CI.getCalledFunction();
+
+ // FIXME: support Windows dllimport function calls and calls through
+ // weak symbols.
+ if (F && (F->hasDLLImportStorageClass() ||
+ (MF->getTarget().getTargetTriple().isOSWindows() &&
+ F->hasExternalWeakLinkage())))
+ return false;
+
+ // FIXME: support control flow guard targets.
+ if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
+ return false;
+
+ // FIXME: support statepoints and related.
+ if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
+ return false;
+
+ if (CI.isInlineAsm())
+ return translateInlineAsm(CI, MIRBuilder);
+
+ diagnoseDontCall(CI);
+
+ Intrinsic::ID ID = Intrinsic::not_intrinsic;
+ if (F && F->isIntrinsic())
+ ID = F->getIntrinsicID();
+
+ if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
+ return translateCallBase(CI, MIRBuilder);
+
+ assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
+
+ if (translateKnownIntrinsic(CI, ID, MIRBuilder))
+ return true;
+
+ TargetLowering::IntrinsicInfo Info;
+ // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
+ bool IsTgtMemIntrinsic = TLI->getTgtMemIntrinsic(Info, CI, *MF, ID);
+
+ return translateTargetIntrinsic(CI, ID, MIRBuilder,
+ IsTgtMemIntrinsic ? &Info : nullptr);
+}
+
bool IRTranslator::findUnwindDestinations(
const BasicBlock *EHPadBB,
BranchProbability Prob,
@@ -3006,15 +3023,16 @@ bool IRTranslator::translateInvoke(const User &U,
return true;
}
+/// The intrinsics currently supported by callbr are implicit control flow
+/// intrinsics such as amdgcn.kill.
bool IRTranslator::translateCallBr(const User &U,
MachineIRBuilder &MIRBuilder) {
+ if (containsBF16Type(U))
+ return false; // see translateCall
+
const CallBrInst &I = cast<CallBrInst>(U);
MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
- // TODO: operand bundles (see SelDAG implementation of callbr)?
- assert(!I.hasOperandBundles() &&
- "Cannot lower callbrs with operand bundles yet");
-
if (I.isInlineAsm()) {
// FIXME: inline asm not yet supported
if (!translateInlineAsm(I, MIRBuilder))
@@ -3024,15 +3042,7 @@ bool IRTranslator::translateCallBr(const User &U,
default:
report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
- if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
- CallInst *CI =
- CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
- SmallVector<Value *, 1>(I.args()));
- bool Success = translateCall(*CI, MIRBuilder);
- CI->deleteValue();
- if (!Success)
+ if (!translateTargetIntrinsic(I, Intrinsic::amdgcn_kill, MIRBuilder))
return false;
break;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 68f6a0445b1af..97e28d226b81d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3375,15 +3375,28 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
DAG.getBasicBlock(Return)));
}
+/// The intrinsics currently supported by callbr are implicit control flow
+/// intrinsics such as amdgcn.kill.
+/// - they should be called (no "dontcall-" attributes)
+/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
+/// - they do not need custom argument handling (no TLI.CollectTargetIntrinsicOperands())
+void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
+ auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
+
+ SmallVector<SDValue, 8> Ops =
+ getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
+ SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
+
+ // Create the node.
+ SDValue Result = getTargetNonMemIntrinsicNode(I, HasChain, Ops, VTs);
+ Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
+
+ setValue(&I, Result);
+}
+
void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
- // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
- // have to do anything here to lower funclet bundles.
- assert(!I.hasOperandBundlesOtherThan(
- {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
- "Cannot lower callbrs with arbitrary operand bundles yet!");
-
if (I.isInlineAsm()) {
visitInlineAsm(I);
} else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
@@ -3391,14 +3404,7 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
default:
report_fatal_error("Unsupported intrinsic for callbr");
case Intrinsic::amdgcn_kill:
- if (I.getNumIndirectDests() != 1)
- report_fatal_error(
- "amdgcn.kill supportes exactly one indirect destination");
- CallInst *CI =
- CallInst::Create(I.getFunctionType(), I.getCalledFunction(),
- SmallVector<Value *, 1>(I.args()));
- visitCall(*CI);
- CI->deleteValue();
+ visitCallBrIntrinsic(I);
break;
}
} else {
@@ -5234,18 +5240,25 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
DAG.setRoot(OutChain);
}
-/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
-/// node.
-void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
- unsigned Intrinsic) {
- // Ignore the callsite's attributes. A specific call site may be marked with
- // readnone, but the lowering code will expect the chain based on the
- // definition.
+/// Check if this intrinsic call depends on the chain (1st return value)
+/// and if it only *loads* memory.
+/// Ignore the callsite's attributes. A specific call site may be marked with
+/// readnone, but the lowering code will expect the chain based on the
+/// definition.
+std::pair<bool, bool> SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase& I) {
const Function *F = I.getCalledFunction();
bool HasChain = !F->doesNotAccessMemory();
bool OnlyLoad =
HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
+ return {HasChain, OnlyLoad};
+}
+
+SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
+ const CallBase &I, bool HasChain, bool OnlyLoad,
+ TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
// Build the operand list.
SmallVector<SDValue, 8> Ops;
if (HasChain) { // If this intrinsic has side-effects, chainify it.
@@ -5257,17 +5270,10 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
}
}
- // Info is set by getTgtMemIntrinsic
- TargetLowering::IntrinsicInfo Info;
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
- DAG.getMachineFunction(),
- Intrinsic);
-
// Add the intrinsic ID as an integer operand if it's not a target intrinsic.
- if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
- Info.opc == ISD::INTRINSIC_W_CHAIN)
- Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
+ if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
+ TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
+ Ops.push_back(DAG.getTargetConstant(I.getIntrinsicID(), getCurSDLoc(),
TLI.getPointerTy(DAG.getDataLayout())));
// Add all operands of the call to the operand list.
@@ -5290,13 +5296,96 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
}
}
+ if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
+ auto *Token = Bundle->Inputs[0].get();
+ SDValue ConvControlToken = getValue(Token);
+ assert(Ops.back().getValueType() != MVT::Glue &&
+ "Did not expected another glue node here.");
+ ConvControlToken =
+ DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
+ Ops.push_back(ConvControlToken);
+ }
+
+ return Ops;
+}
+
+SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
+ bool HasChain) {
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
SmallVector<EVT, 4> ValueVTs;
ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
if (HasChain)
ValueVTs.push_back(MVT::Other);
- SDVTList VTs = DAG.getVTList(ValueVTs);
+ return DAG.getVTList(ValueVTs);
+}
+
+/// Get an INTRINSIC node for a target intrinsic which does not touch touch memory.
+SDValue
+SelectionDAGBuilder::getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain,
+ SmallVector<SDValue, 8> &Ops,
+ SDVTList &VTs) {
+ SDValue Result;
+
+ if (!HasChain) {
+ Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
+ } else if (!I.getType()->isVoidTy()) {
+ Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
+ } else {
+ Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
+ }
+
+ return Result;
+}
+
+/// Set root, convert return type if necessaey and check alignment.
+SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
+ bool HasChain,
+ bool OnlyLoad,
+ SDValue Result) {
+ if (HasChain) {
+ SDValue Chain = Result.getValue(Result.getNode()->getNumValues() - 1);
+ if (OnlyLoad)
+ PendingLoads.push_back(Chain);
+ else
+ DAG.setRoot(Chain);
+ }
+
+ if (I.getType()->isVoidTy())
+ return Result;
+
+ if (!isa<VectorType>(I.getType()))
+ Result = lowerRangeToAssertZExt(DAG, I, Result);
+
+ MaybeAlign Alignment = I.getRetAlign();
+
+ // Insert `assertalign` node if there's an alignment.
+ if (InsertAssertAlign && Alignment) {
+ Result =
+ DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
+ }
+
+ return Result;
+}
+
+/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
+/// node.
+void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
+ unsigned Intrinsic) {
+ auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
+
+ // Info is set by getTgtMemIntrinsic
+ TargetLowering::IntrinsicInfo Info;
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ bool IsTgtMemIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
+ DAG.getMachineFunction(),
+ Intrinsic);
+
+ SmallVector<SDValue, 8> Ops = getTargetIntrinsicOperands(
+ I, HasChain, OnlyLoad, IsTgtMemIntrinsic ? &Info : nullptr);
+ SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
// Propagate fast-math-flags from IR to node(s).
SDNodeFlags Flags;
@@ -5307,19 +5396,9 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
// Create the node.
SDValue Result;
- if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
- auto *Token = Bundle->Inputs[0].get();
- SDValue ConvControlToken = getValue(Token);
- assert(Ops.back().getValueType() != MVT::Glue &&
- "Did not expected another glue node here.");
- ConvControlToken =
- DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
- Ops.push_back(ConvControlToken);
- }
-
// In some cases, custom collection of operands from CallInst I may be needed.
TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
- if (IsTgtIntrinsic) {
+ if (IsTgtMemIntrinsic) {
// This is target intrinsic that touches memory
//
// TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
@@ -5339,34 +5418,11 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
Info.ssid, Info.order, Info.failureOrder);
Result =
DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, MemVT, MMO);
- } else if (!HasChain) {
- Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
- } else if (!I.getType()->isVoidTy()) {
- Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
} else {
- Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
+ Result = getTargetNonMemIntrinsicNode(I, HasChain, Ops, VTs);
}
- if (HasChain) {
- SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
- if (OnlyLoad)
- PendingLoads.push_back(Chain);
- else
- DAG.setRoot(Chain);
- }
-
- if (!I.getType()->isVoidTy()) {
- if (!isa<VectorType>(I.getType()))
- Result = lowerRangeToAssertZExt(DAG, I, Result);
-
- MaybeAlign Alignment = I.getRetAlign();
-
- // Insert `assertalign` node if there's an alignment.
- if (InsertAssertAlign && Alignment) {
- Result =
- DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
- }
- }
+ Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
setValue(&I, Result);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 35c15bc269d4b..71f67a7f1eff6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -536,10 +536,12 @@ class SelectionDAGBuilder {
private:
// These all get lowered before this pass.
void visitInvoke(const InvokeInst &I);
- void visitCallBr(const CallBrInst &I);
void visitCallBrLandingPad(const CallInst &I);
void visitResume(const ResumeInst &I);
+ void visitCallBr(const CallBrInst &I);
+ void visitCallBrIntrinsic(const CallBrInst &I);
+
void visitUnary(const User &I, unsigned Opcode);
void visitFNeg(const User &I) { visitUnary(I, ISD::FNEG); }
@@ -709,6 +711,16 @@ class SelectionDAGBuilder {
MCSymbol *&BeginLabel);
SDValue lowerEndEH(SDValue Chain, const InvokeInst *II,
const BasicBlock *EHPadBB, MCSymbol *BeginLabel);
+
+ std::pair<bool, bool> getTargetIntrinsicCallProperties(const CallBase& I);
+ SmallVector<SDValue, 8> getTargetIntrinsicOperands(
+ const CallBase &I, bool HasChain, bool OnlyLoad,
+ TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo = nullptr);
+ SDVTList getTargetIntrinsicVTList(const CallBase &I, bool HasChain);
+ SDValue getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain,
+ SmallVector<SDValue, 8> &Ops, SDVTList &VTs);
+ SDValue handleTargetIntrinsicRet(const CallBase &I, bool HasChain,
+ bool OnlyLoad, SDValue Result);
};
/// This struct represents the registers (physical or virtual)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 2811c6c712b3b..4dd1ca412a99c 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3279,6 +3279,9 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) {
void Verifier::visitCallBrInst(CallBrInst &CBI) {
if (!CBI.isInlineAsm()) {
+ Check(CBI.getCalledFunction(), "Callbr: indirect function / invalid signature");
+ Check(!CBI.hasOperandBundles(), "Callbr currently doesn't support operand bundles");
+
switch (CBI.getIntrinsicID()) {
case Intrinsic::amdgcn_kill: {
Check(CBI.getNumIndirectDests() == 1,
@@ -3288,7 +3291,6 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) {
Check(Unreachable || (Call && Call->getIntrinsicID() ==
Intrinsic::amdgcn_unreachable),
"Callbr amdgcn_kill indirect dest needs to be unreachable");
- visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI);
break;
}
default:
diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll
index 8e125e723e6fc..29bd3397b8980 100644
--- a/llvm/test/Verifier/callbr.ll
+++ b/llvm/test/Verifier/callbr.ll
@@ -159,3 +159,11 @@ define void @test_callbr_intrinsic_unsupported() {
cont:
ret void
}
+
+; CHECK-NEXT: Callbr: indirect function / invalid signature
+define void @test_callbr_intrinsic_wrong_signature(ptr %ptr) {
+ %func = load ptr, ptr %ptr, align 8
+ callbr void %func() to label %cont []
+cont:
+ ret void
+}
>From b46aacd2329860b34179a16a0f83ca793ce5f347 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Sun, 6 Apr 2025 14:59:29 -0500
Subject: [PATCH 07/13] fix formatting
---
.../llvm/CodeGen/GlobalISel/IRTranslator.h | 3 ++-
.../SelectionDAG/SelectionDAGBuilder.cpp | 24 +++++++++----------
.../SelectionDAG/SelectionDAGBuilder.h | 5 ++--
llvm/lib/IR/Verifier.cpp | 6 +++--
4 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index ba2aeac8dffe1..941750510f1e1 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -317,7 +317,8 @@ class IRTranslator : public MachineFunctionPass {
bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
- bool translateCallBrIntrinsic(const CallBrInst &I, MachineIRBuilder &MIRBuilder);
+ bool translateCallBrIntrinsic(const CallBrInst &I,
+ MachineIRBuilder &MIRBuilder);
bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 97e28d226b81d..96a04cddb533c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3379,7 +3379,8 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
/// intrinsics such as amdgcn.kill.
/// - they should be called (no "dontcall-" attributes)
/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
-/// - they do not need custom argument handling (no TLI.CollectTargetIntrinsicOperands())
+/// - they do not need custom argument handling (no
+/// TLI.CollectTargetIntrinsicOperands())
void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
@@ -5245,7 +5246,8 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
/// Ignore the callsite's attributes. A specific call site may be marked with
/// readnone, but the lowering code will expect the chain based on the
/// definition.
-std::pair<bool, bool> SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase& I) {
+std::pair<bool, bool>
+SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
const Function *F = I.getCalledFunction();
bool HasChain = !F->doesNotAccessMemory();
bool OnlyLoad =
@@ -5322,11 +5324,11 @@ SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
return DAG.getVTList(ValueVTs);
}
-/// Get an INTRINSIC node for a target intrinsic which does not touch touch memory.
-SDValue
-SelectionDAGBuilder::getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain,
- SmallVector<SDValue, 8> &Ops,
- SDVTList &VTs) {
+/// Get an INTRINSIC node for a target intrinsic which does not touch touch
+/// memory.
+SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
+ const CallBase &I, bool HasChain, SmallVector<SDValue, 8> &Ops,
+ SDVTList &VTs) {
SDValue Result;
if (!HasChain) {
@@ -5363,8 +5365,7 @@ SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
// Insert `assertalign` node if there's an alignment.
if (InsertAssertAlign && Alignment) {
- Result =
- DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
+ Result = DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
}
return Result;
@@ -5379,9 +5380,8 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
// Info is set by getTgtMemIntrinsic
TargetLowering::IntrinsicInfo Info;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- bool IsTgtMemIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
- DAG.getMachineFunction(),
- Intrinsic);
+ bool IsTgtMemIntrinsic =
+ TLI.getTgtMemIntrinsic(Info, I, DAG.getMachineFunction(), Intrinsic);
SmallVector<SDValue, 8> Ops = getTargetIntrinsicOperands(
I, HasChain, OnlyLoad, IsTgtMemIntrinsic ? &Info : nullptr);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 71f67a7f1eff6..c1cf2c4035103 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -712,13 +712,14 @@ class SelectionDAGBuilder {
SDValue lowerEndEH(SDValue Chain, const InvokeInst *II,
const BasicBlock *EHPadBB, MCSymbol *BeginLabel);
- std::pair<bool, bool> getTargetIntrinsicCallProperties(const CallBase& I);
+ std::pair<bool, bool> getTargetIntrinsicCallProperties(const CallBase &I);
SmallVector<SDValue, 8> getTargetIntrinsicOperands(
const CallBase &I, bool HasChain, bool OnlyLoad,
TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo = nullptr);
SDVTList getTargetIntrinsicVTList(const CallBase &I, bool HasChain);
SDValue getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain,
- SmallVector<SDValue, 8> &Ops, SDVTList &VTs);
+ SmallVector<SDValue, 8> &Ops,
+ SDVTList &VTs);
SDValue handleTargetIntrinsicRet(const CallBase &I, bool HasChain,
bool OnlyLoad, SDValue Result);
};
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 4dd1ca412a99c..b84a09c98fe39 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3279,8 +3279,10 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) {
void Verifier::visitCallBrInst(CallBrInst &CBI) {
if (!CBI.isInlineAsm()) {
- Check(CBI.getCalledFunction(), "Callbr: indirect function / invalid signature");
- Check(!CBI.hasOperandBundles(), "Callbr currently doesn't support operand bundles");
+ Check(CBI.getCalledFunction(),
+ "Callbr: indirect function / invalid signature");
+ Check(!CBI.hasOperandBundles(),
+ "Callbr currently doesn't support operand bundles");
switch (CBI.getIntrinsicID()) {
case Intrinsic::amdgcn_kill: {
>From 8d2b926280a0ac8c2bbed9351fe05e9784ed7f19 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 9 Apr 2025 09:53:45 -0500
Subject: [PATCH 08/13] add callbr test with different succ order
---
llvm/test/CodeGen/AMDGPU/callbr.ll | 50 ++++++++++++++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll
index 33c75f6cf5aab..52856b872ee2a 100644
--- a/llvm/test/CodeGen/AMDGPU/callbr.ll
+++ b/llvm/test/CodeGen/AMDGPU/callbr.ll
@@ -50,3 +50,53 @@ cont:
store i32 %a, ptr %dst, align 4
ret void
}
+
+; SELDAG-LABEL: test_kill_block_order:
+; SELDAG-NEXT: ; %bb.0:
+; SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: flat_load_dword v0, v[0:1]
+; SELDAG-NEXT: v_and_b32_e32 v1, 1, v4
+; SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; SELDAG-NEXT: s_mov_b64 s[4:5], exec
+; SELDAG-NEXT: s_andn2_b64 s[6:7], exec, vcc
+; SELDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; SELDAG-NEXT: s_cbranch_scc0 .LBB1_2
+; SELDAG-NEXT: ; %bb.1:
+; SELDAG-NEXT: s_and_b64 exec, exec, s[4:5]
+; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: flat_store_dword v[2:3], v0
+; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; SELDAG-NEXT: s_setpc_b64 s[30:31]
+; SELDAG-NEXT: .LBB1_2:
+; SELDAG-NEXT: s_mov_b64 exec, 0
+; SELDAG-NEXT: s_endpgm
+
+; GISEL-LABEL: test_kill_block_order:
+; GISEL-NEXT: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: flat_load_dword v0, v[0:1]
+; GISEL-NEXT: v_and_b32_e32 v1, 1, v4
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: s_andn2_b64 s[6:7], exec, vcc
+; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GISEL-NEXT: s_cbranch_scc0 .LBB1_2
+; GISEL-NEXT: ; %bb.1:
+; GISEL-NEXT: s_and_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT: flat_store_dword v[2:3], v0
+; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB1_2:
+; GISEL-NEXT: s_mov_b64 exec, 0
+; GISEL-NEXT: s_endpgm
+
+define void @test_kill_block_order(ptr %src, ptr %dst, i1 %c) {
+ %a = load i32, ptr %src, align 4
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+cont:
+ store i32 %a, ptr %dst, align 4
+ ret void
+kill:
+ unreachable
+}
>From 47dc4f14bdc39315c21864d3f689e15c4b0abf37 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 14 Apr 2025 03:32:41 -0500
Subject: [PATCH 09/13] implement feedback
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 12 +++---------
.../lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 10 ++--------
2 files changed, 5 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 144203edc0ad7..248d9287efdf4 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3038,16 +3038,10 @@ bool IRTranslator::translateCallBr(const User &U,
if (!translateInlineAsm(I, MIRBuilder))
return false;
} else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
- switch (I.getIntrinsicID()) {
- default:
- report_fatal_error("Unsupported intrinsic for callbr");
- case Intrinsic::amdgcn_kill:
- if (!translateTargetIntrinsic(I, Intrinsic::amdgcn_kill, MIRBuilder))
- return false;
- break;
- }
+ if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder))
+ return false;
} else {
- report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ return false;
}
// Retrieve successors.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 96a04cddb533c..5747638a1a4da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3401,15 +3401,9 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
if (I.isInlineAsm()) {
visitInlineAsm(I);
} else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
- switch (I.getIntrinsicID()) {
- default:
- report_fatal_error("Unsupported intrinsic for callbr");
- case Intrinsic::amdgcn_kill:
- visitCallBrIntrinsic(I);
- break;
- }
+ visitCallBrIntrinsic(I);
} else {
- report_fatal_error("Only know how to handle inlineasm/intrinsic callbr");
+ report_fatal_error("only know how to handle inlineasm/intrinsic callbr");
}
CopyToExportRegsIfNeeded(&I);
>From 681cb92c50d18bae09fcc784acb63862e0cfd1bf Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 12 May 2025 03:57:49 -0500
Subject: [PATCH 10/13] update LangRef
---
llvm/docs/LangRef.rst | 28 ++++++++++++++++++++++------
1 file changed, 22 insertions(+), 6 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 7fbac14112af3..8dd9feb3126b1 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -9585,8 +9585,12 @@ The '``callbr``' instruction causes control to transfer to a specified
function, with the possibility of control flow transfer to either the
'``fallthrough``' label or one of the '``indirect``' labels.
-This instruction should only be used to implement the "goto" feature of gcc
-style inline assembly. Any other usage is an error in the IR verifier.
+This instruction can currently only be used
+
+#. to implement the "goto" feature of gcc style inline assembly or
+#. to call selected intrinsics.
+
+Any other usage is an error in the IR verifier.
Note that in order to support outputs along indirect edges, LLVM may need to
split critical edges, which may require synthesizing a replacement block for
@@ -9627,7 +9631,8 @@ This instruction requires several arguments:
indicates the function accepts a variable number of arguments, the
extra arguments can be specified.
#. '``fallthrough label``': the label reached when the inline assembly's
- execution exits the bottom.
+ execution exits the bottom. In case of an intrinsic call, the semantic
+ depends on the semantic of the intrinsic.
#. '``indirect labels``': the labels reached when a callee transfers control
to a location other than the '``fallthrough label``'. Label constraints
refer to these destinations.
@@ -9645,9 +9650,12 @@ flow goes after the call.
The output values of a '``callbr``' instruction are available both in the
the '``fallthrough``' block, and any '``indirect``' blocks(s).
-The only use of this today is to implement the "goto" feature of gcc inline
-assembly where additional labels can be provided as locations for the inline
-assembly to jump to.
+The only uses of this today are:
+
+#. implement the "goto" feature of gcc inline assembly where additional
+ labels can be provided as locations for the inline assembly to jump to.
+#. support selected intrinsics which manipulate control flow and should
+ be chained to specific terminators, such as '``unreachable``'.
Example:
""""""""
@@ -9662,6 +9670,14 @@ Example:
<result> = callbr i32 asm "", "=r,r,!i"(i32 %x)
to label %fallthrough [label %indirect]
+ ; intrinsic which should be followed by unreachable (the order of the
+ ; blocks after the callbr instruction doesn't matter)
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+ cont:
+ ...
+ kill:
+ unreachable
+
.. _i_resume:
'``resume``' Instruction
>From 9ffe1f528ce042fed8999e003765cbc7fc5ee578 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 12 May 2025 04:46:22 -0500
Subject: [PATCH 11/13] don't start supporting inline-asm for callbr in
GlobalISel with this patch
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 248d9287efdf4..ec0919c4046d7 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3033,16 +3033,13 @@ bool IRTranslator::translateCallBr(const User &U,
const CallBrInst &I = cast<CallBrInst>(U);
MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
- if (I.isInlineAsm()) {
- // FIXME: inline asm not yet supported
- if (!translateInlineAsm(I, MIRBuilder))
- return false;
- } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) {
- if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder))
- return false;
- } else {
+ // FIXME: inline asm not yet supported
+ if (I.isInlineAsm())
+ return false;
+ if (I.getIntrinsicID() == Intrinsic::not_intrinsic)
+ return false;
+ if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder))
return false;
- }
// Retrieve successors.
SmallPtrSet<BasicBlock *, 8> Dests;
>From f486ac46e6dc123ac01f5d23fabe52b08d75f349 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 12 May 2025 10:22:01 -0500
Subject: [PATCH 12/13] add comments on callbr region handling and fix polly
test
---
llvm/include/llvm/Analysis/RegionInfoImpl.h | 3 +++
polly/test/ScopDetect/callbr.ll | 9 +++------
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h
index 759e9c47bebb8..3d507f5467037 100644
--- a/llvm/include/llvm/Analysis/RegionInfoImpl.h
+++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h
@@ -553,6 +553,9 @@ bool RegionInfoBase<Tr>::isRegion(BlockT *entry, BlockT *exit) const {
using DST = typename DomFrontierT::DomSetType;
+ // Make sure that a region involving a callbr contains every successor
+ // blocks up to the ones that postdominate the callbr block. Otherwise,
+ // StructurizeCFG will tear the callbr apart.
// TODO? post domination frontier?
if constexpr (std::is_same_v<BlockT, BasicBlock>) {
if (DomTreeNodeT *PDTNode = PDT->getNode(exit); PDTNode) {
diff --git a/polly/test/ScopDetect/callbr.ll b/polly/test/ScopDetect/callbr.ll
index 4182974693678..75f676afd79c4 100644
--- a/polly/test/ScopDetect/callbr.ll
+++ b/polly/test/ScopDetect/callbr.ll
@@ -1,10 +1,7 @@
-; RUN: opt %loadNPMPolly '-passes=print<polly-detect>' -polly-detect-track-failures -disable-output -pass-remarks-missed=polly-detect < %s 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt %loadNPMPolly '-passes=print<polly-detect>' -polly-detect-track-failures -disable-output -stats < %s 2>&1 | FileCheck %s --check-prefix=STAT
-; REQUIRES: asserts
+; RUN: opt %loadNPMPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; REMARK: Branch from indirect terminator.
-
-; STAT: 1 polly-detect - Number of rejected regions: Branch from indirect terminator
+; CHECK-LABEL: func
+; CHECK-NOT: Valid
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
>From 4670b7af4a8ccc8dc0f96bb16e1f7be66eeac7f5 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 26 May 2025 04:40:37 -0500
Subject: [PATCH 13/13] fix typo and formatting
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index ec0919c4046d7..60b55303ad490 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2818,8 +2818,9 @@ bool IRTranslator::translateTargetIntrinsic(
else if (TgtMemIntrinsicInfo->fallbackAddressSpace)
MPI = MachinePointerInfo(*TgtMemIntrinsicInfo->fallbackAddressSpace);
MIB.addMemOperand(MF->getMachineMemOperand(
- MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CI.getAAMetadata(),
- /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid, TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder));
+ MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CB.getAAMetadata(),
+ /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid,
+ TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder));
}
if (CB.isConvergent()) {
More information about the llvm-commits
mailing list