[clang] [llvm] [WPD] Enable speculative devirtualization with LTO (PR #233380)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Oct 19 02:20:04 PDT 2026
https://github.com/KRM7 created https://github.com/llvm/llvm-project/pull/233380
Enable speculative single-target devirtualization with LTO (disabled by default, can be enabled using -fdevirtualize-speculatively).
The DevirtSpeculatively member variable is also removed from WPD, so that the optimization is controlled entirely by the -devirtualize-speculatively flag. The -enable-devirtualize-speculatively flag is now only used to add the WPD pass to the non-LTO pipeline.
>From 4be403204bd31c7d32502f9fe57817a2884625fe Mon Sep 17 00:00:00 2001
From: Krisztian Rugasi <krisztian.rugasi at arm.com>
Date: Thu, 15 Oct 2026 10:03:19 +0000
Subject: [PATCH] [WPD] Enable speculative devirtualization with LTO
Enable speculative single-target devirtualization with LTO (disabled
by default, can be enabled using -fdevirtualize-speculatively).
The DevirtSpeculatively member variable is also removed from WPD,
so that the optimization is controlled entirely by the
-devirtualize-speculatively flag. The -enable-devirtualize-speculatively
flag is now only used to add the WPD pass to the non-LTO pipeline.
---
clang/docs/UsersManual.md | 2 -
clang/lib/Driver/ToolChains/CommonArgs.cpp | 8 +
clang/test/Driver/lto.c | 8 +
llvm/include/llvm/AsmParser/LLToken.h | 1 +
llvm/include/llvm/IR/ModuleSummaryIndex.h | 1 +
llvm/include/llvm/IR/ModuleSummaryIndexYAML.h | 2 +
.../llvm/Transforms/IPO/WholeProgramDevirt.h | 7 +-
llvm/lib/AsmParser/LLLexer.cpp | 1 +
llvm/lib/AsmParser/LLParser.cpp | 3 +
llvm/lib/IR/AsmWriter.cpp | 5 +-
llvm/lib/Passes/PassBuilderPipelines.cpp | 3 +-
llvm/lib/Passes/PassesOptions.td | 2 +-
.../lib/Transforms/IPO/WholeProgramDevirt.cpp | 143 +++++++++---------
llvm/test/Assembler/thinlto-summary.ll | 4 +-
llvm/test/ThinLTO/X86/devirt_speculative.ll | 71 +++++++++
.../X86/devirt_speculative_mixed_vis.ll | 105 +++++++++++++
.../devirt-single-impl-check-cfi.ll | 68 +++++++++
.../speculative-devirt-cfi.ll | 68 +++++++++
.../speculative-devirt-mixed-vis.ll | 77 ++++++++++
.../speculative-devirt-single-impl.ll | 14 +-
20 files changed, 505 insertions(+), 88 deletions(-)
create mode 100644 llvm/test/ThinLTO/X86/devirt_speculative.ll
create mode 100644 llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll
create mode 100644 llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll
create mode 100644 llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll
create mode 100644 llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll
diff --git a/clang/docs/UsersManual.md b/clang/docs/UsersManual.md
index 96d1c4e009c71a..2c2c131b656e03 100644
--- a/clang/docs/UsersManual.md
+++ b/clang/docs/UsersManual.md
@@ -2589,8 +2589,6 @@ if.end.icp: ; preds = %if.false.orig_indir
ret void
```
-This feature is temporarily ignored at the LLVM side when LTO is enabled.
-TODO: Update the comment when the LLVM side supports this feature for LTO.
This feature is turned off by default.
:::
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 7c832ef405fa51..329f5317ea47a9 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1222,6 +1222,14 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
(A->getOption().matches(options::OPT_fglobal_isel) ? "1" : "0")));
}
+ if (Arg *A = Args.getLastArg(options::OPT_fdevirtualize_speculatively,
+ options::OPT_fno_devirtualize_speculatively))
+ CmdArgs.push_back(Args.MakeArgString(
+ Twine(PluginOptPrefix) + "-devirtualize-speculatively=" +
+ (A->getOption().matches(options::OPT_fdevirtualize_speculatively)
+ ? "1"
+ : "0")));
+
// If an explicit debugger tuning argument appeared, pass it along.
if (Arg *A =
Args.getLastArg(options::OPT_gTune_Group, options::OPT_ggdbN_Group)) {
diff --git a/clang/test/Driver/lto.c b/clang/test/Driver/lto.c
index 838eb740863752..be18231f5d335c 100644
--- a/clang/test/Driver/lto.c
+++ b/clang/test/Driver/lto.c
@@ -127,6 +127,14 @@
// CHECK-NO-SLP: "-plugin-opt=-vectorize-slp=0"
// CHECK-SLP: "-plugin-opt=-vectorize-slp=1"
+// RUN: %clang --target=x86_64-unknown-linux -### %s -flto -fno-devirtualize-speculatively 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DEVIRTUALIZE-SPECULATIVELY < %t %s
+// RUN: %clang --target=x86_64-unknown-linux -### %s -flto -fdevirtualize-speculatively 2> %t
+// RUN: FileCheck --check-prefix=CHECK-DEVIRTUALIZE-SPECULATIVELY < %t %s
+//
+// CHECK-NO-DEVIRTUALIZE-SPECULATIVELY: "-plugin-opt=-devirtualize-speculatively=0"
+// CHECK-DEVIRTUALIZE-SPECULATIVELY: "-plugin-opt=-devirtualize-speculatively=1"
+
// -flto passes -time-passes when -ftime-report is passed
// RUN: %clang --target=x86_64-unknown-linux-gnu -### %s -flto -ftime-report 2> %t
// RUN: FileCheck --check-prefix=CHECK-TIME-REPORT < %t %s
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index 2679a630c2dd95..e5816ca301527e 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -467,6 +467,7 @@ enum Kind {
kw_indir,
kw_singleImpl,
kw_branchFunnel,
+ kw_speculative,
kw_singleImplName,
kw_resByArg,
kw_byArg,
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 4f68582b65c4eb..f9d6640f50852a 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -1349,6 +1349,7 @@ struct WholeProgramDevirtResolution {
BranchFunnel, ///< When retpoline mitigation is enabled, use a branch funnel
///< that is defined in the merged module. Otherwise same as
///< Indir.
+ Speculative, ///< Speculative devirtualization
} TheKind = Indir;
std::string SingleImplName;
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
index 158c03e77306c9..441433eb381adc 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
@@ -102,6 +102,8 @@ template <> struct ScalarEnumerationTraits<WholeProgramDevirtResolution::Kind> {
io.enumCase(value, "SingleImpl", WholeProgramDevirtResolution::SingleImpl);
io.enumCase(value, "BranchFunnel",
WholeProgramDevirtResolution::BranchFunnel);
+ io.enumCase(value, "Speculative",
+ WholeProgramDevirtResolution::Speculative);
}
};
diff --git a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
index c3a394ab4fb720..0dd61bbd41a4ab 100644
--- a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
+++ b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
@@ -227,14 +227,11 @@ struct WholeProgramDevirtPass
ModuleSummaryIndex *ExportSummary;
const ModuleSummaryIndex *ImportSummary;
bool UseCommandLine = false;
- bool DevirtSpeculatively = false;
WholeProgramDevirtPass()
: ExportSummary(nullptr), ImportSummary(nullptr), UseCommandLine(true) {}
WholeProgramDevirtPass(ModuleSummaryIndex *ExportSummary,
- const ModuleSummaryIndex *ImportSummary,
- bool DevirtSpeculatively = false)
- : ExportSummary(ExportSummary), ImportSummary(ImportSummary),
- DevirtSpeculatively(DevirtSpeculatively) {
+ const ModuleSummaryIndex *ImportSummary)
+ : ExportSummary(ExportSummary), ImportSummary(ImportSummary) {
assert(!(ExportSummary && ImportSummary));
}
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 5e52dc8dcb2478..20212fc26451aa 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -884,6 +884,7 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(indir);
KEYWORD(singleImpl);
KEYWORD(branchFunnel);
+ KEYWORD(speculative);
KEYWORD(singleImplName);
KEYWORD(resByArg);
KEYWORD(byArg);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 60b7824342aa38..03a266931632cf 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -10047,6 +10047,9 @@ bool LLParser::parseWpdRes(WholeProgramDevirtResolution &WPDRes) {
case lltok::kw_branchFunnel:
WPDRes.TheKind = WholeProgramDevirtResolution::BranchFunnel;
break;
+ case lltok::kw_speculative:
+ WPDRes.TheKind = WholeProgramDevirtResolution::Speculative;
+ break;
default:
return error(Lex.getLoc(), "unexpected WholeProgramDevirtResolution kind");
}
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 1cbcfb195764b3..a30ff85e33160b 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -3402,6 +3402,8 @@ getWholeProgDevirtResKindName(WholeProgramDevirtResolution::Kind K) {
return "singleImpl";
case WholeProgramDevirtResolution::BranchFunnel:
return "branchFunnel";
+ case WholeProgramDevirtResolution::Speculative:
+ return "speculative";
}
llvm_unreachable("invalid WholeProgramDevirtResolution kind");
}
@@ -3496,7 +3498,8 @@ void AssemblyWriter::printWPDRes(const WholeProgramDevirtResolution &WPDRes) {
Out << "wpdRes: (kind: ";
Out << getWholeProgDevirtResKindName(WPDRes.TheKind);
- if (WPDRes.TheKind == WholeProgramDevirtResolution::SingleImpl)
+ if (WPDRes.TheKind == WholeProgramDevirtResolution::SingleImpl ||
+ WPDRes.TheKind == WholeProgramDevirtResolution::Speculative)
Out << ", singleImplName: \"" << WPDRes.SingleImplName << "\"";
if (!WPDRes.ResByArg.empty()) {
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 125c8e442f27b6..b4ce758e314e94 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1620,8 +1620,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
MPM.addPass(AssignGUIDPass());
MPM.addPass(WholeProgramDevirtPass(
/*ExportSummary*/ nullptr,
- /*ImportSummary*/ nullptr,
- /*DevirtSpeculatively*/ PTO.DevirtualizeSpeculatively));
+ /*ImportSummary*/ nullptr));
MPM.addPass(DropTypeTestsPass());
// Given that the devirtualization creates more opportunities for inlining,
// we run the Inliner again here to maximize the optimization gain we
diff --git a/llvm/lib/Passes/PassesOptions.td b/llvm/lib/Passes/PassesOptions.td
index 4398d5c06bff2b..d37930c7ca39d8 100644
--- a/llvm/lib/Passes/PassesOptions.td
+++ b/llvm/lib/Passes/PassesOptions.td
@@ -38,7 +38,7 @@ defm : BoolField<"enable-constraint-elimination", "true",
// TODO: There is a similar flag in WPD pass, we should consolidate them by
// parsing the option only once in PassBuilder and share it across both places.
defm : BoolField<"enable-devirtualize-speculatively", "false",
- "Enable speculative devirtualization optimization">;
+ "Enable speculative devirtualization without LTO">;
defm : BoolField<"enable-dfa-jump-thread", "true",
"Enable DFA jump threading">;
defm : BoolField<"enable-global-analyses", "true",
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 88c57d2195bbda..ba0718882d77e0 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -160,8 +160,6 @@ static cl::opt<std::string> ClWriteSummary(
"bitcode, otherwise YAML"),
cl::Hidden);
-// TODO: This option eventually should support any public visibility vtables
-// with/out LTO.
static cl::opt<bool> ClDevirtualizeSpeculatively(
"devirtualize-speculatively",
cl::desc("Enable speculative devirtualization optimization"),
@@ -611,11 +609,9 @@ struct DevirtModule {
std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;
PatternList FunctionsToSkip;
- const bool DevirtSpeculatively;
DevirtModule(Module &M, ModuleAnalysisManager &MAM,
ModuleSummaryIndex *ExportSummary,
- const ModuleSummaryIndex *ImportSummary,
- bool DevirtSpeculatively)
+ const ModuleSummaryIndex *ImportSummary)
: M(M), MAM(MAM),
FAM(MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()),
ExportSummary(ExportSummary), ImportSummary(ImportSummary),
@@ -628,8 +624,7 @@ struct DevirtModule {
RemarksEnabled(areRemarksEnabled()),
OREGetter([&](Function &F) -> OptimizationRemarkEmitter & {
return FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
- }),
- DevirtSpeculatively(DevirtSpeculatively) {
+ }) {
assert(!(ExportSummary && ImportSummary));
FunctionsToSkip.init(SkipFunctionNames);
}
@@ -645,18 +640,18 @@ struct DevirtModule {
std::vector<VTableBits> &Bits,
DenseMap<Metadata *, std::set<TypeMemberInfo>> &TypeIdMap);
- bool
- tryFindVirtualCallTargets(std::vector<VirtualCallTarget> &TargetsForSlot,
- const std::set<TypeMemberInfo> &TypeMemberInfos,
- uint64_t ByteOffset,
- ModuleSummaryIndex *ExportSummary);
+ bool tryFindVirtualCallTargets(
+ std::vector<VirtualCallTarget> &TargetsForSlot,
+ const std::set<TypeMemberInfo> &TypeMemberInfos, uint64_t ByteOffset,
+ ModuleSummaryIndex *ExportSummary, bool &HasPublicTarget);
void applySingleImplDevirt(VTableSlotInfo &SlotInfo, Constant *TheFn,
- bool &IsExported);
+ bool HasPublicTarget, bool &IsExported);
bool trySingleImplDevirt(ModuleSummaryIndex *ExportSummary,
MutableArrayRef<VirtualCallTarget> TargetsForSlot,
VTableSlotInfo &SlotInfo,
- WholeProgramDevirtResolution *Res);
+ WholeProgramDevirtResolution *Res,
+ bool HasPublicTarget);
void applyICallBranchFunnel(VTableSlotInfo &SlotInfo, Function &JT,
bool &IsExported);
@@ -743,8 +738,7 @@ struct DevirtModule {
// Lower the module using the action and summary passed as command line
// arguments. For testing purposes only.
- static bool runForTesting(Module &M, ModuleAnalysisManager &MAM,
- bool DevirtSpeculatively);
+ static bool runForTesting(Module &M, ModuleAnalysisManager &MAM);
};
struct DevirtIndex {
@@ -779,13 +773,14 @@ struct DevirtIndex {
bool tryFindVirtualCallTargets(std::vector<ValueInfo> &TargetsForSlot,
const TypeIdCompatibleVtableInfo TIdInfo,
- uint64_t ByteOffset);
+ uint64_t ByteOffset, bool &HasPublicTarget);
bool trySingleImplDevirt(MutableArrayRef<ValueInfo> TargetsForSlot,
VTableSlotSummary &SlotSummary,
VTableSlotInfo &SlotInfo,
WholeProgramDevirtResolution *Res,
- std::set<ValueInfo> &DevirtTargets);
+ std::set<ValueInfo> &DevirtTargets,
+ bool HasPublicTarget);
void run();
};
@@ -794,22 +789,17 @@ struct DevirtIndex {
PreservedAnalyses WholeProgramDevirtPass::run(Module &M,
ModuleAnalysisManager &MAM) {
if (UseCommandLine) {
- if (!DevirtModule::runForTesting(M, MAM, ClDevirtualizeSpeculatively))
+ if (!DevirtModule::runForTesting(M, MAM))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
}
std::optional<ModuleSummaryIndex> Index;
- if (!ExportSummary && !ImportSummary && DevirtSpeculatively) {
- // Build the ExportSummary from the module.
- assert(!ExportSummary &&
- "ExportSummary is expected to be empty in non-LTO mode");
+ if (!ExportSummary && !ImportSummary && ClDevirtualizeSpeculatively) {
ProfileSummaryInfo PSI(M);
- Index.emplace(buildModuleSummaryIndex(M, nullptr, &PSI));
- ExportSummary = Index.has_value() ? &Index.value() : nullptr;
+ ExportSummary = &Index.emplace(buildModuleSummaryIndex(M, nullptr, &PSI));
}
- if (!DevirtModule(M, MAM, ExportSummary, ImportSummary, DevirtSpeculatively)
- .run())
+ if (!DevirtModule(M, MAM, ExportSummary, ImportSummary).run())
return PreservedAnalyses::all();
return PreservedAnalyses::none();
}
@@ -907,9 +897,7 @@ void llvm::updatePublicTypeTestCalls(Module &M,
CI->replaceAllUsesWith(NewCI);
CI->eraseFromParent();
}
- } else {
- // TODO: Don't replace public type tests when speculative devirtualization
- // gets enabled in LTO mode.
+ } else if (!ClDevirtualizeSpeculatively) {
auto *True = ConstantInt::getTrue(M.getContext());
for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) {
auto *CI = cast<CallInst>(U.getUser());
@@ -1014,8 +1002,7 @@ static Error checkCombinedSummaryForTesting(ModuleSummaryIndex *Summary) {
return ErrorSuccess();
}
-bool DevirtModule::runForTesting(Module &M, ModuleAnalysisManager &MAM,
- bool DevirtSpeculatively) {
+bool DevirtModule::runForTesting(Module &M, ModuleAnalysisManager &MAM) {
std::unique_ptr<ModuleSummaryIndex> Summary =
std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
@@ -1044,8 +1031,7 @@ bool DevirtModule::runForTesting(Module &M, ModuleAnalysisManager &MAM,
ClSummaryAction == PassSummaryAction::Export ? Summary.get()
: nullptr,
ClSummaryAction == PassSummaryAction::Import ? Summary.get()
- : nullptr,
- DevirtSpeculatively)
+ : nullptr)
.run();
if (!ClWriteSummary.empty()) {
@@ -1104,16 +1090,19 @@ void DevirtModule::buildTypeIdentifierMap(
bool DevirtModule::tryFindVirtualCallTargets(
std::vector<VirtualCallTarget> &TargetsForSlot,
const std::set<TypeMemberInfo> &TypeMemberInfos, uint64_t ByteOffset,
- ModuleSummaryIndex *ExportSummary) {
+ ModuleSummaryIndex *ExportSummary, bool &HasPublicTarget) {
+ HasPublicTarget = false;
+
for (const TypeMemberInfo &TM : TypeMemberInfos) {
if (!TM.Bits->GV->isConstant())
return false;
- // Without DevirtSpeculatively, we cannot perform whole program
- // devirtualization analysis on a vtable with public LTO visibility.
- if (!DevirtSpeculatively && TM.Bits->GV->getVCallVisibility() ==
- GlobalObject::VCallVisibilityPublic)
- return false;
+ if (TM.Bits->GV->getVCallVisibility() ==
+ GlobalObject::VCallVisibilityPublic) {
+ HasPublicTarget = true;
+ if (!ClDevirtualizeSpeculatively)
+ return false;
+ }
Function *Fn = nullptr;
Constant *C = nullptr;
@@ -1131,12 +1120,6 @@ bool DevirtModule::tryFindVirtualCallTargets(
if (Fn->getName() == "__cxa_pure_virtual")
continue;
- // In most cases empty functions will be overridden by the
- // implementation of the derived class, so we can skip them.
- if (DevirtSpeculatively && Fn->getReturnType()->isVoidTy() &&
- Fn->getInstructionCount() <= 1)
- continue;
-
// We can disregard unreachable functions as possible call targets, as
// unreachable functions shouldn't be called.
if (mustBeUnreachableFunction(Fn, ExportSummary))
@@ -1158,7 +1141,10 @@ bool DevirtModule::tryFindVirtualCallTargets(
bool DevirtIndex::tryFindVirtualCallTargets(
std::vector<ValueInfo> &TargetsForSlot,
- const TypeIdCompatibleVtableInfo TIdInfo, uint64_t ByteOffset) {
+ const TypeIdCompatibleVtableInfo TIdInfo, uint64_t ByteOffset,
+ bool &HasPublicTarget) {
+ HasPublicTarget = false;
+
for (const TypeIdOffsetVtableInfo &P : TIdInfo) {
// Find a representative copy of the vtable initializer.
// We can have multiple available_externally, linkonce_odr and weak_odr
@@ -1187,8 +1173,11 @@ bool DevirtIndex::tryFindVirtualCallTargets(
VS = CurVS;
// We cannot perform whole program devirtualization analysis on a vtable
// with public LTO visibility.
- if (VS->getVCallVisibility() == GlobalObject::VCallVisibilityPublic)
- return false;
+ if (VS->getVCallVisibility() == GlobalObject::VCallVisibilityPublic) {
+ HasPublicTarget = true;
+ if (!ClDevirtualizeSpeculatively)
+ return false;
+ }
break;
}
}
@@ -1214,7 +1203,8 @@ bool DevirtIndex::tryFindVirtualCallTargets(
}
void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
- Constant *TheFn, bool &IsExported) {
+ Constant *TheFn, bool HasPublicTarget,
+ bool &IsExported) {
// Don't devirtualize function if we're told to skip it
// in -wholeprogramdevirt-skip.
if (FunctionsToSkip.match(TheFn->stripPointerCasts()->getName()))
@@ -1257,7 +1247,7 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
// add support to compare the virtual function pointer to the
// devirtualized target. In case of a mismatch, fall back to indirect
// call.
- if (DevirtCheckMode == WPDCheckMode::Fallback || DevirtSpeculatively) {
+ if (DevirtCheckMode == WPDCheckMode::Fallback || HasPublicTarget) {
MDNode *Weights = MDBuilder(M.getContext()).createLikelyBranchWeights();
// Version the indirect call site. If the called value is equal to the
// given callee, 'NewInst' will be executed, otherwise the original call
@@ -1278,6 +1268,7 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
// In either trapping or non-checking mode, devirtualize original call.
else {
// Devirtualize unconditionally.
+ assert(!HasPublicTarget);
CB.setCalledOperand(Callee);
// Since the call site is now direct, we must clear metadata that
// is only appropriate for indirect calls. This includes !prof and
@@ -1295,7 +1286,7 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
}
// This use is no longer unsafe.
- if (VCallSite.NumUnsafeUses)
+ if (!HasPublicTarget && VCallSite.NumUnsafeUses)
--*VCallSite.NumUnsafeUses;
}
if (CSInfo.isExported())
@@ -1338,7 +1329,7 @@ static bool addCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee) {
bool DevirtModule::trySingleImplDevirt(
ModuleSummaryIndex *ExportSummary,
MutableArrayRef<VirtualCallTarget> TargetsForSlot, VTableSlotInfo &SlotInfo,
- WholeProgramDevirtResolution *Res) {
+ WholeProgramDevirtResolution *Res, bool HasPublicTarget) {
// See if the program contains a single implementation of this virtual
// function.
auto *TheFn = TargetsForSlot[0].Fn;
@@ -1351,7 +1342,7 @@ bool DevirtModule::trySingleImplDevirt(
TargetsForSlot[0].WasDevirt = true;
bool IsExported = false;
- applySingleImplDevirt(SlotInfo, TheFn, IsExported);
+ applySingleImplDevirt(SlotInfo, TheFn, HasPublicTarget, IsExported);
if (!IsExported)
return false;
@@ -1383,7 +1374,11 @@ bool DevirtModule::trySingleImplDevirt(
// LTO unit split, so we can ignore return value of AddCalls.
addCalls(SlotInfo, TheFnVI);
- Res->TheKind = WholeProgramDevirtResolution::SingleImpl;
+ if (HasPublicTarget)
+ Res->TheKind = WholeProgramDevirtResolution::Speculative;
+ else
+ Res->TheKind = WholeProgramDevirtResolution::SingleImpl;
+
Res->SingleImplName = std::string(TheFn->getName());
return true;
@@ -1393,7 +1388,8 @@ bool DevirtIndex::trySingleImplDevirt(MutableArrayRef<ValueInfo> TargetsForSlot,
VTableSlotSummary &SlotSummary,
VTableSlotInfo &SlotInfo,
WholeProgramDevirtResolution *Res,
- std::set<ValueInfo> &DevirtTargets) {
+ std::set<ValueInfo> &DevirtTargets,
+ bool HasPublicTarget) {
// See if the program contains a single implementation of this virtual
// function.
auto TheFn = TargetsForSlot[0];
@@ -1427,7 +1423,11 @@ bool DevirtIndex::trySingleImplDevirt(MutableArrayRef<ValueInfo> TargetsForSlot,
// Record in summary for use in devirtualization during the ThinLTO import
// step.
- Res->TheKind = WholeProgramDevirtResolution::SingleImpl;
+ if (HasPublicTarget)
+ Res->TheKind = WholeProgramDevirtResolution::Speculative;
+ else
+ Res->TheKind = WholeProgramDevirtResolution::SingleImpl;
+
if (GlobalValue::isLocalLinkage(S->linkage())) {
if (IsExported) {
// If target is a local function and we are exporting it by
@@ -2318,7 +2318,8 @@ void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
return;
const WholeProgramDevirtResolution &Res = ResI->second;
- if (Res.TheKind == WholeProgramDevirtResolution::SingleImpl) {
+ if (Res.TheKind == WholeProgramDevirtResolution::SingleImpl ||
+ Res.TheKind == WholeProgramDevirtResolution::Speculative) {
assert(!Res.SingleImplName.empty());
// The type of the function in the declaration is irrelevant because every
// call site will cast it to the correct type.
@@ -2333,7 +2334,9 @@ void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
// This is the import phase so we should not be exporting anything.
bool IsExported = false;
- applySingleImplDevirt(SlotInfo, SingleImpl, IsExported);
+ bool IsSpeculative =
+ Res.TheKind == WholeProgramDevirtResolution::Speculative;
+ applySingleImplDevirt(SlotInfo, SingleImpl, IsSpeculative, IsExported);
assert(!IsExported);
}
@@ -2445,7 +2448,7 @@ bool DevirtModule::run() {
Function *PublicTypeTestFunc = nullptr;
// If we are in speculative devirtualization mode, we can work on the public
// type test intrinsics.
- if (DevirtSpeculatively)
+ if (ClDevirtualizeSpeculatively)
PublicTypeTestFunc =
Intrinsic::getDeclarationIfExists(&M, Intrinsic::public_type_test);
Function *TypeTestFunc =
@@ -2574,14 +2577,14 @@ bool DevirtModule::run() {
->getOrInsertTypeIdSummary(
cast<MDString>(S.first.TypeID)->getString())
.WPDRes[S.first.ByteOffset];
+
+ bool HasPublicTarget = true;
if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,
- S.first.ByteOffset, ExportSummary)) {
- bool SingleImplDevirt =
- trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res);
- // Out of speculative devirtualization mode, Try to apply virtual constant
- // propagation or branch funneling.
- // TODO: This should eventually be enabled for non-public type tests.
- if (!SingleImplDevirt && !DevirtSpeculatively) {
+ S.first.ByteOffset, ExportSummary,
+ HasPublicTarget)) {
+ bool SingleImplDevirt = trySingleImplDevirt(
+ ExportSummary, TargetsForSlot, S.second, Res, HasPublicTarget);
+ if (!SingleImplDevirt && !HasPublicTarget) {
DidVirtualConstProp |=
tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first);
@@ -2725,11 +2728,13 @@ void DevirtIndex::run() {
WholeProgramDevirtResolution *Res =
&ExportSummary.getTypeIdSummary(S.first.TypeID)
->WPDRes[S.first.ByteOffset];
+
+ bool HasPublicTarget = true;
if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,
- S.first.ByteOffset)) {
+ S.first.ByteOffset, HasPublicTarget)) {
if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,
- DevirtTargets))
+ DevirtTargets, HasPublicTarget))
continue;
}
}
diff --git a/llvm/test/Assembler/thinlto-summary.ll b/llvm/test/Assembler/thinlto-summary.ll
index 5b907f9a06039a..31c2aa25024e11 100644
--- a/llvm/test/Assembler/thinlto-summary.ll
+++ b/llvm/test/Assembler/thinlto-summary.ll
@@ -70,7 +70,7 @@
^28 = typeid: (name: "_ZTS1B", summary: (typeTestRes: (kind: inline, sizeM1BitWidth: 0, alignLog2: 1, sizeM1: 2, bitMask: 3, inlineBits: 4)))
; Test the AllOnes resolution, and all kinds of WholeProgramDevirtResolution
; types, including all optional resolution by argument kinds.
-^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp)))))))
+^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: speculative, singleImplName: "_ZN1A1mEi")), (offset: 24, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp)))))))
; Test the other kinds of type test resoultions
^30 = typeid: (name: "_ZTS1D", summary: (typeTestRes: (kind: byteArray, sizeM1BitWidth: 0)))
^31 = typeid: (name: "_ZTS1E", summary: (typeTestRes: (kind: unsat, sizeM1BitWidth: 0)))
@@ -109,7 +109,7 @@
; CHECK: ^26 = gv: (guid: 9123456789101112131, summaries: (function: (module: ^0, flags: (linkage: internal, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition, noRenameOnPromotion: 0), insts: 1)))
; CHECK: ^27 = typeid: (name: "_ZTS1C", summary: (typeTestRes: (kind: single, sizeM1BitWidth: 0))) ; guid = 1884921850105019584
; CHECK: ^28 = typeid: (name: "_ZTS1B", summary: (typeTestRes: (kind: inline, sizeM1BitWidth: 0, alignLog2: 1, sizeM1: 2, bitMask: 3, inlineBits: 4))) ; guid = 6203814149063363976
-; CHECK: ^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp))))))) ; guid = 7004155349499253778
+; CHECK: ^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: speculative, singleImplName: "_ZN1A1mEi")), (offset: 24, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp))))))) ; guid = 7004155349499253778
; CHECK: ^30 = typeid: (name: "_ZTS1D", summary: (typeTestRes: (kind: byteArray, sizeM1BitWidth: 0))) ; guid = 9614786172484273522
; CHECK: ^31 = typeid: (name: "_ZTS1E", summary: (typeTestRes: (kind: unsat, sizeM1BitWidth: 0))) ; guid = 17437243864166745132
; CHECK: ^32 = flags: 8
diff --git a/llvm/test/ThinLTO/X86/devirt_speculative.ll b/llvm/test/ThinLTO/X86/devirt_speculative.ll
new file mode 100644
index 00000000000000..2edd0deb924a92
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_speculative.ll
@@ -0,0 +1,71 @@
+; REQUIRES: x86-registered-target
+
+; Test that speculative devirtualization works with LTO
+
+; Index based WPD
+; RUN: opt -passes=assign-guid -thinlto-bc -o %t2.o %s
+; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \
+; RUN: -o %t3 \
+; RUN: -r=%t2.o,test,px \
+; RUN: -r=%t2.o,_ZN1A1fEv,p \
+; RUN: -r=%t2.o,_ZTV1A,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s
+
+; Hybrid WPD
+; RUN: opt -passes=assign-guid -thinlto-bc -thinlto-split-lto-unit -o %t2.o %s
+; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \
+; RUN: -o %t3 \
+; RUN: -r=%t2.o,test,px \
+; RUN: -r=%t2.o,_ZN1A1fEv \
+; RUN: -r=%t2.o,_ZN1A1fEv,p \
+; RUN: -r=%t2.o,_ZTV1A \
+; RUN: -r=%t2.o,_ZTV1A,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s
+
+; Regular LTO WPD
+; RUN: opt -passes=assign-guid -o %t4.o %s
+; RUN: llvm-lto2 run %t4.o -devirtualize-speculatively -save-temps -pass-remarks=. \
+; RUN: -o %t5 \
+; RUN: -r=%t4.o,test,px \
+; RUN: -r=%t4.o,_ZN1A1fEv,p \
+; RUN: -r=%t4.o,_ZTV1A,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t5.0.4.opt.bc -o - | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+; REMARK: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv
+
+%struct.A = type { ptr }
+
+ at _ZTV1A = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !1
+
+define i32 @test(ptr %p) {
+entry:
+ ; CHECK-LABEL: define i32 @test(
+ ; CHECK-NEXT: [[ENTRY:.*]]:
+ ; CHECK: [[ICMP:%.*]] = icmp eq ptr [[FPTR:%.*]], @_ZN1A1fEv
+ ; CHECK-NEXT: br i1 [[ICMP]], label %[[IF_END:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]],
+ ; CHECK: [[IF_FALSE_ORIG_INDIRECT]]:
+ ; CHECK-NEXT: [[TMP0:%.*]] = tail call i32 [[FPTR]]
+ ; CHECK-NEXT: br label %[[IF_END]]
+ ; CHECK: [[IF_END]]:
+ ; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ 12, %[[ENTRY]] ]
+ ; CHECK-NEXT: ret i32 [[TMP1]]
+ %0 = load ptr, ptr %p
+ %1 = call i1 @llvm.public.type.test(ptr %0, metadata !"_ZTS1A")
+ call void @llvm.assume(i1 %1)
+ %2 = load ptr, ptr %0
+ %3 = call i32 %2(ptr %p)
+ ret i32 %3
+}
+
+define i32 @_ZN1A1fEv(ptr %this) {
+entry:
+ ret i32 12
+}
+
+declare i1 @llvm.public.type.test(ptr, metadata)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 0}
diff --git a/llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll b/llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll
new file mode 100644
index 00000000000000..bbb237043180b9
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll
@@ -0,0 +1,105 @@
+; REQUIRES: x86-registered-target
+
+; Test that speculative and non-speculative single-impl devirtualization work
+; correctly together based on vcall_visibilities.
+
+; Index based WPD
+; RUN: opt -passes=assign-guid -thinlto-bc -o %t2.o %s
+; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \
+; RUN: -o %t3 \
+; RUN: -r=%t2.o,test_speculative,px \
+; RUN: -r=%t2.o,test_nonspeculative,px \
+; RUN: -r=%t2.o,_ZN1A1fEv,p \
+; RUN: -r=%t2.o,_ZN1A1gEv,p \
+; RUN: -r=%t2.o,_ZTV1A,px \
+; RUN: -r=%t2.o,_ZTV1B,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s
+
+; Hybrid WPD
+; RUN: opt -passes=assign-guid -thinlto-bc -thinlto-split-lto-unit -o %t2.o %s
+; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \
+; RUN: -o %t3 \
+; RUN: -r=%t2.o,test_speculative,px \
+; RUN: -r=%t2.o,test_nonspeculative,px \
+; RUN: -r=%t2.o,_ZN1A1fEv \
+; RUN: -r=%t2.o,_ZN1A1gEv \
+; RUN: -r=%t2.o,_ZN1A1fEv,p \
+; RUN: -r=%t2.o,_ZN1A1gEv,p \
+; RUN: -r=%t2.o,_ZTV1A \
+; RUN: -r=%t2.o,_ZTV1B \
+; RUN: -r=%t2.o,_ZTV1A,px \
+; RUN: -r=%t2.o,_ZTV1B,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s
+
+; Regular LTO WPD
+; RUN: opt -passes=assign-guid -o %t4.o %s
+; RUN: llvm-lto2 run %t4.o -devirtualize-speculatively -save-temps -pass-remarks=. \
+; RUN: -o %t5 \
+; RUN: -r=%t4.o,test_speculative,px \
+; RUN: -r=%t4.o,test_nonspeculative,px \
+; RUN: -r=%t4.o,_ZN1A1fEv,p \
+; RUN: -r=%t4.o,_ZN1A1gEv,p \
+; RUN: -r=%t4.o,_ZTV1A,px \
+; RUN: -r=%t4.o,_ZTV1B,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t5.0.4.opt.bc -o - | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+; REMARK-DAG: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv
+; REMARK-DAG: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1gEv
+
+%struct.A = type { ptr }
+%struct.B = type { ptr }
+
+ at _ZTV1A = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !2
+ at _ZTV1B = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1gEv] }, !type !1, !vcall_visibility !3
+
+define i32 @test_speculative(ptr %p) {
+entry:
+ ; CHECK-LABEL: define i32 @test_speculative(
+ ; CHECK-NEXT: [[ENTRY1:.*]]:
+ ; CHECK: [[ICMP:%.*]] = icmp eq ptr [[FPTR:%.*]], @_ZN1A1fEv
+ ; CHECK-NEXT: br i1 [[ICMP]], label %[[IF_END:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]],
+ ; CHECK: [[IF_FALSE_ORIG_INDIRECT]]:
+ ; CHECK-NEXT: [[TMP0:%.*]] = tail call i32 [[FPTR]]
+ ; CHECK-NEXT: br label %[[IF_END]]
+ ; CHECK: [[IF_END]]:
+ ; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ 12, %[[ENTRY1]] ]
+ ; CHECK-NEXT: ret i32 [[TMP1]]
+ %0 = load ptr, ptr %p
+ %1 = call i1 @llvm.public.type.test(ptr %0, metadata !"_ZTS1A")
+ call void @llvm.assume(i1 %1)
+ %2 = load ptr, ptr %0
+ %3 = call i32 %2(ptr %p)
+ ret i32 %3
+}
+
+define i32 @test_nonspeculative(ptr %p) {
+entry:
+ ; CHECK-LABEL: define {{.*}} i32 @test_nonspeculative(
+ ; CHECK: ret i32 23
+ %0 = load ptr, ptr %p
+ %1 = call i1 @llvm.public.type.test(ptr %0, metadata !"_ZTS1B")
+ call void @llvm.assume(i1 %1)
+ %2 = load ptr, ptr %0
+ %3 = call i32 %2(ptr %p)
+ ret i32 %3
+}
+
+define i32 @_ZN1A1fEv(ptr %this) {
+entry:
+ ret i32 12
+}
+
+define i32 @_ZN1A1gEv(ptr %this) {
+entry:
+ ret i32 23
+}
+
+declare i1 @llvm.public.type.test(ptr, metadata)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
+!2 = !{i64 0}
+!3 = !{i64 1}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll
new file mode 100644
index 00000000000000..80081d145d1aa0
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=wholeprogramdevirt -wholeprogramdevirt-check=fallback -whole-program-visibility -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+; Check whether the type.test intrinsic is removed after devirtualization when
+; the fallback mode is used for WPD.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv
+
+%struct.A = type { ptr }
+
+ at _ZTV1A = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !1
+
+define i32 @test(ptr %p) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr [[P:%.*]]) !guid [[META2:![0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT: br i1 true, label %[[CONT:.*]], label %[[TRAP:.*]]
+; CHECK: [[TRAP]]:
+; CHECK-NEXT: call void @llvm.ubsantrap(i8 2)
+; CHECK-NEXT: unreachable
+; CHECK: [[CONT]]:
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[VTABLE]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq ptr [[TMP2]], @_ZN1A1fEv
+; CHECK-NEXT: br i1 [[TMP4]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF3:![0-9]+]]
+; CHECK: [[IF_TRUE_DIRECT_TARG]]:
+; CHECK-NEXT: [[CALL:%.*]] = call i32 @_ZN1A1fEv(ptr [[TMP0]])
+; CHECK-NEXT: br label %[[IF_END_ICP:.*]]
+; CHECK: [[IF_FALSE_ORIG_INDIRECT]]:
+; CHECK-NEXT: [[CALL1:%.*]] = call i32 [[TMP2]](ptr [[TMP0]])
+; CHECK-NEXT: br label %[[IF_END_ICP]]
+; CHECK: [[IF_END_ICP]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[CALL1]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ [[CALL]], %[[IF_TRUE_DIRECT_TARG]] ]
+; CHECK-NEXT: ret i32 [[TMP6]]
+;
+entry:
+ %0 = load ptr, ptr %p
+ %vtable = load ptr, ptr %0
+ %1 = call { ptr, i1 } @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"_ZTS1A")
+ %2 = extractvalue { ptr, i1 } %1, 1
+ br i1 %2, label %cont, label %trap
+
+trap:
+ call void @llvm.ubsantrap(i8 2)
+ unreachable
+
+cont:
+ %3 = extractvalue { ptr, i1 } %1, 0
+ %call = call i32 %3(ptr %0)
+ ret i32 %call
+}
+
+declare i32 @_ZN1A1fEv(ptr %this)
+
+declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
+declare void @llvm.ubsantrap(i8 immarg)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 1}
+;.
+; CHECK: [[META2]] = !{i64 8346051122425466633}
+; CHECK: [[PROF3]] = !{!"branch_weights", i32 1048575, i32 1}
+;.
diff --git a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll
new file mode 100644
index 00000000000000..bc13a884f5a033
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=wholeprogramdevirt -devirtualize-speculatively -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+; Check that the type.test intrinsic is not removed after speculative
+; devirtualization, since the indirect call will not be eliminated in this case.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv
+
+%struct.A = type { ptr }
+
+ at _ZTV1A = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !1
+
+define i32 @test(ptr %p) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"_ZTS1A")
+; CHECK-NEXT: br i1 [[TMP3]], label %[[CONT:.*]], label %[[TRAP:.*]]
+; CHECK: [[TRAP]]:
+; CHECK-NEXT: call void @llvm.ubsantrap(i8 2)
+; CHECK-NEXT: unreachable
+; CHECK: [[CONT]]:
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[VTABLE]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq ptr [[TMP2]], @_ZN1A1fEv
+; CHECK-NEXT: br i1 [[TMP4]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK: [[IF_TRUE_DIRECT_TARG]]:
+; CHECK-NEXT: [[CALL:%.*]] = call i32 @_ZN1A1fEv(ptr [[TMP0]])
+; CHECK-NEXT: br label %[[IF_END_ICP:.*]]
+; CHECK: [[IF_FALSE_ORIG_INDIRECT]]:
+; CHECK-NEXT: [[CALL1:%.*]] = call i32 [[TMP2]](ptr [[TMP0]])
+; CHECK-NEXT: br label %[[IF_END_ICP]]
+; CHECK: [[IF_END_ICP]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[CALL1]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ [[CALL]], %[[IF_TRUE_DIRECT_TARG]] ]
+; CHECK-NEXT: ret i32 [[TMP6]]
+;
+entry:
+ %vptr = load ptr, ptr %p, align 8
+ %vtable = load ptr, ptr %vptr, align 8
+ %1 = call { ptr, i1 } @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"_ZTS1A")
+ %2 = extractvalue { ptr, i1 } %1, 1
+ br i1 %2, label %cont, label %trap
+
+trap:
+ call void @llvm.ubsantrap(i8 2)
+ unreachable
+
+cont:
+ %3 = extractvalue { ptr, i1 } %1, 0
+ %call = call i32 %3(ptr %vptr)
+ ret i32 %call
+}
+
+declare i32 @_ZN1A1fEv(ptr %this)
+
+declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
+declare void @llvm.ubsantrap(i8 immarg)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 0}
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1048575, i32 1}
+;.
diff --git a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll
new file mode 100644
index 00000000000000..c84922800d7810
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=wholeprogramdevirt -devirtualize-speculatively -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+; Check that speculative and non-speculative single-impl devirtualization
+; work correctly together based on vcall_visibilities.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1B1fEv
+
+%struct.A = type { ptr }
+%struct.B = type { ptr }
+
+ at _ZTV1A = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !2
+ at _ZTV1B = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1B1fEv] }, !type !1, !vcall_visibility !3
+
+define i32 @test_speculative(ptr %p) {
+; CHECK-LABEL: define i32 @test_speculative(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.public.type.test(ptr [[TMP0]], metadata !"_ZTS1A")
+; CHECK-NEXT: call void @llvm.assume(i1 [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq ptr [[TMP2]], @_ZN1A1fEv
+; CHECK-NEXT: br i1 [[TMP3]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF2:![0-9]+]]
+; CHECK: [[IF_TRUE_DIRECT_TARG]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @_ZN1A1fEv(ptr [[P]])
+; CHECK-NEXT: br label %[[IF_END_ICP:.*]]
+; CHECK: [[IF_FALSE_ORIG_INDIRECT]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 [[TMP2]](ptr [[P]])
+; CHECK-NEXT: br label %[[IF_END_ICP]]
+; CHECK: [[IF_END_ICP]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[TMP5]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ [[TMP4]], %[[IF_TRUE_DIRECT_TARG]] ]
+; CHECK-NEXT: ret i32 [[TMP6]]
+;
+entry:
+ %2 = load ptr, ptr %p
+ %3 = call i1 @llvm.public.type.test(ptr %2, metadata !"_ZTS1A") ; public vcall_visibility
+ call void @llvm.assume(i1 %3)
+ %4 = load ptr, ptr %2
+ %5 = call i32 %4(ptr %p)
+ ret i32 %5
+}
+
+define i32 @test_non_speculative(ptr %p) {
+; CHECK-LABEL: define i32 @test_non_speculative(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.public.type.test(ptr [[TMP0]], metadata !"_ZTS1B")
+; CHECK-NEXT: call void @llvm.assume(i1 [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @_ZN1B1fEv(ptr [[P]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %2 = load ptr, ptr %p
+ %3 = call i1 @llvm.public.type.test(ptr %2, metadata !"_ZTS1B") ; hidden vcall_visibility
+ call void @llvm.assume(i1 %3)
+ %4 = load ptr, ptr %2
+ %5 = call i32 %4(ptr %p)
+ ret i32 %5
+}
+
+declare i32 @_ZN1A1fEv(ptr %this)
+declare i32 @_ZN1B1fEv(ptr %this)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
+!2 = !{i64 0}
+!3 = !{i64 1}
+;.
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 1048575, i32 1}
+;.
diff --git a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll
index 0d30d09ccbb742..d9af1753294cbe 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll
@@ -11,10 +11,11 @@ target datalayout = "e-p:64:64"
target triple = "x86_64-unknown-linux-gnu"
; CHECK: remark: devirt-single.cc:30:32: single-impl: devirtualized a call to vf
+; CHECK: remark: devirt-single.cc:35:32: single-impl: devirtualized a call to vf_empty
; CHECK: remark: devirt-single.cc:41:32: single-impl: devirtualized a call to vf
; CHECK: remark: devirt-single.cc:51:32: single-impl: devirtualized a call to vf
; CHECK: remark: devirt-single.cc:13:0: devirtualized vf
-; CHECK-NOT: devirtualized
+; CHECK: remark: devirt-single.cc:23:0: devirtualized vf_empty
@vt1 = constant [1 x ptr] [ptr @vf], !type !8
@vt2 = constant [1 x ptr] [ptr @vf_empty], !type !12
@@ -23,8 +24,6 @@ define i1 @vf(ptr %this) #0 !dbg !7 {
ret i1 true
}
-; This should NOT be devirtualized because during non-lto empty functions
-; are skipped.
define void @vf_empty(ptr %this) !dbg !11 {
ret void
}
@@ -50,7 +49,10 @@ define void @call1(ptr %obj) #1 !dbg !9 {
%p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid1")
call void @llvm.assume(i1 %p)
%fptr = load ptr, ptr %vtable, align 8
- ; CHECK: call i1 %fptr
+ ; CHECK: if.true.direct_targ:
+ ; CHECK: call i1 @vf_empty(
+ ; CHECK: if.false.orig_indirect:
+ ; CHECK: call i1 %fptr(
%1 = call i1 %fptr(ptr %obj), !dbg !10
ret void
}
@@ -130,5 +132,5 @@ declare void @llvm.assume(i1)
-; CHECK: 1 wholeprogramdevirt - Number of whole program devirtualization targets
-; CHECK: 3 wholeprogramdevirt - Number of single implementation devirtualizations
+; CHECK: 2 wholeprogramdevirt - Number of whole program devirtualization targets
+; CHECK: 4 wholeprogramdevirt - Number of single implementation devirtualizations
More information about the cfe-commits
mailing list