[llvm] [LLVM] Add flatten function attribute to LLVM IR and implement recursive inlining in AlwaysInliner (PR #174899)

[Arthur Eubanks via llvm-commits]

================
@@ -99,11 +110,77 @@ bool AlwaysInlineImpl(
     }
   }
 
+  // Flatten functions with the flatten attribute using a local worklist.
+  for (Function *F : NeedFlattening) {
+    SmallVector<std::pair<CallBase *, int>, 16> Worklist;
+    SmallVector<std::pair<Function *, int>, 16> InlineHistory;
+    SmallVector<CallBase *> NewCallSites;
+    OptimizationRemarkEmitter ORE(F);
+
+    // Collect initial calls.
+    for (BasicBlock &BB : *F)
+      for (Instruction &I : BB)
+        if (auto *CB = dyn_cast<CallBase>(&I)) {
+          if (CB->getAttributes().hasFnAttr(Attribute::NoInline))
+            continue;
+          Function *Callee = CB->getCalledFunction();
+          if (!Callee || Callee->isDeclaration())
+            continue;
+          Worklist.push_back({CB, -1});
+        }
+
+    while (!Worklist.empty()) {
+      auto Item = Worklist.pop_back_val();
+      CallBase *CB = Item.first;
+      int InlineHistoryID = Item.second;
+      Function *Callee = CB->getCalledFunction();
+      if (!Callee)
+        continue;
+
+      // Detect recursion.
+      if (Callee == F ||
+          inlineHistoryIncludes(Callee, InlineHistoryID, InlineHistory)) {
+        ORE.emit([&]() {
+          return OptimizationRemarkMissed("inline", "NotInlined",
+                                          CB->getDebugLoc(), CB->getParent())
+                 << "'" << ore::NV("Callee", Callee)
+                 << "' is not inlined into '"
+                 << ore::NV("Caller", CB->getCaller())
+                 << "': recursive call during flattening";
+        });
+        continue;
+      }
+
+      if (Callee->isPresplitCoroutine() || Callee->isDeclaration() ||
+          !isInlineViable(*Callee).isSuccess())
+        continue;
+
+      // Check TTI for target-specific inlining restrictions (e.g., SME ABI).
+      TargetTransformInfo &TTI = GetTTI(*Callee);
+      if (!TTI.areInlineCompatible(F, Callee))
----------------
aeubanks wrote:

there may be other frontends that don't check, it would be nice to check here as well, but that's a separate issue

https://github.com/llvm/llvm-project/pull/174899