[llvm] [WholeProgramDevirt] Add check for AvailableExternal and give up devirt (PR #143468)

[Tianle Liu via llvm-commits]

https://github.com/tianleliu created https://github.com/llvm/llvm-project/pull/143468

…devirtualization.

When a customer class inherits from a libc++ class, and is built with
"-flto  -fwhole-program-vtables -static-libstdc++ \
 -Wl,-plugin-opt=-whole-program-visibility", the libc++ class's vtable
is available_externally, meanwhile the customer class vtable is private. 
And both of them are !vcall_visibility == Linkage Unit. In this case, icall.branch.funnel might be generated.

But the icall.branch.funnel would cause crash in LowerTypeTests
because available_externally Global_Object is skipped to save and leads to a NULL GlobalTypeMember.
Even walking around the crash in LowerTypeTests, it still crashes in SelectionDAGBuilder or VerifierPass,
because they ask operands of icall.branch.funnel must be the same GlobalValue.

This patch only fix fullLTO mode.

>From 49b51cf548222550f15b0e8fa63ec56fdfe95642 Mon Sep 17 00:00:00 2001
From: tianleli <tianle.l.liu at intel.com>
Date: Mon, 9 Jun 2025 15:51:23 +0800
Subject: [PATCH] [WholeProgramDevirt] Add checking for AvailableExternal and
 gives up devirtualization.

When a customer class inherits from a libc++ class, and is built with
"-flto  -fwhole-program-vtables -static-libstdc++ \
 -Wl,-plugin-opt=-whole-program-visibility", the libc++ class's vtable
is available_externally, meanwhile the customer class vtable is private.
And both of them are !vcall_visibility == Linkage Unit.
In this case, icall.branch.funnel might be generated.

But the icall.branch.funnel would cause crash in LowerTypeTests because
available_externally Global_Object is skipped to save and
leads to a NULL GlobalTypeMember.
Even walking around the crash in LowerTypeTests, it still crashes in
SelectionDAGBuilder or VerifierPass, because they ask operands of
icall.branch.funnel must be the same GlobalValue.

This patch only fix fullLTO mode.
---
 .../lib/Transforms/IPO/WholeProgramDevirt.cpp | 11 ++++
 .../availableexternal-check.ll                | 57 +++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll

diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index a7d9f3ba24b24..c0ddb50ea83ed 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -1093,6 +1093,7 @@ bool DevirtModule::tryFindVirtualCallTargets(
     std::vector<VirtualCallTarget> &TargetsForSlot,
     const std::set<TypeMemberInfo> &TypeMemberInfos, uint64_t ByteOffset,
     ModuleSummaryIndex *ExportSummary) {
+  bool hasAvailableExternally = false;
   for (const TypeMemberInfo &TM : TypeMemberInfos) {
     if (!TM.Bits->GV->isConstant())
       return false;
@@ -1103,6 +1104,16 @@ bool DevirtModule::tryFindVirtualCallTargets(
         GlobalObject::VCallVisibilityPublic)
       return false;
 
+    // Record if the first GV is AvailableExternally
+    if (TargetsForSlot.empty())
+      hasAvailableExternally = TM.Bits->GV->hasAvailableExternallyLinkage();
+
+    // When the first GV is AvailableExternally, check if all other GVs are
+    // also AvailableExternally. If they are not the same, return false.
+    if (!TargetsForSlot.empty() && hasAvailableExternally &&
+        !TM.Bits->GV->hasAvailableExternallyLinkage())
+      return false;
+
     Function *Fn = nullptr;
     Constant *C = nullptr;
     std::tie(Fn, C) =
diff --git a/llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll b/llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll
new file mode 100644
index 0000000000000..d72075cdc1bb3
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll
@@ -0,0 +1,57 @@
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility %s | FileCheck %s
+
+; This test is reduced from C++ code like this:
+; class A :public std::exception {
+; public:
+;   A() {};
+;   const char* what () const throw () {return "A";}
+; };
+; long test(std::exception *p) {
+;   const char* ch = p->what();
+;   return std::strlen(ch);
+; }
+;
+; Build command is "clang++ -O2 -target x86_64-unknown-linux -flto=thin \
+; -fwhole-program-vtables -static-libstdc++  -Wl,-plugin-opt=-whole-program-visibility"
+;
+; _ZTVSt9exception's visibility is 1 (Linkage Unit), and available_externally.
+; But another vtable _ZTV1A.0 is not available_externally.
+; They should not do devirtualization because they are in different linkage type.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+ at _ZTVSt9exception = available_externally constant { [5 x ptr] } { [5 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr @_ZNKSt9exception4whatEv] }, !type !0, !type !1, !vcall_visibility !2
+ at _ZTV1A.0 = constant [5 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr @_ZNK1A4whatEv], !type !3, !type !4, !type !5, !type !6, !vcall_visibility !2
+
+declare ptr @_ZNKSt9exception4whatEv()
+
+define i64 @_Z4testPSt9exception() {
+  %1 = call i1 @llvm.type.test(ptr null, metadata !"_ZTSSt9exception")
+  tail call void @llvm.assume(i1 %1)
+  %2 = getelementptr i8, ptr null, i64 16
+  %3 = load ptr, ptr %2, align 8
+  %4 = tail call ptr %3(ptr null)
+  ret i64 0
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
+declare void @llvm.assume(i1 noundef) #0
+
+declare ptr @_ZNK1A4whatEv()
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i1 @llvm.type.test(ptr, metadata) #1
+
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+
+attributes #0 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!0 = !{i64 16, !"_ZTSSt9exception"}
+!1 = !{i64 32, !"_ZTSMSt9exceptionKDoFPKcvE.virtual"}
+!2 = !{i64 1}
+!3 = !{i32 16, !"_ZTS1A"}
+!4 = !{i32 32, !"_ZTSM1AKDoFPKcvE.virtual"}
+!5 = !{i32 16, !"_ZTSSt9exception"}
+!6 = !{i32 32, !"_ZTSMSt9exceptionKDoFPKcvE.virtual"}