[llvm] [readtapi] Add support for stubify-ing directories (PR #76885)

[Samuel Marks via llvm-commits]

================
@@ -167,26 +197,237 @@ static bool handleMergeAction(const Context &Ctx) {
   return handleWriteAction(Ctx, std::move(Out));
 }
 
+static void stubifyImpl(std::unique_ptr<InterfaceFile> IF, Context &Ctx) {
+  // TODO: Add inlining and magic merge support.
+  if (Ctx.OutStream == nullptr) {
+    std::error_code EC;
+    SmallString<PATH_MAX> OutputLoc = IF->getPath();
+    replace_extension(OutputLoc, ".tbd");
+    Ctx.OutStream = std::make_unique<llvm::raw_fd_stream>(OutputLoc, EC);
+    if (EC)
+      reportError("opening file '" + OutputLoc + ": " + EC.message());
+  }
+
+  handleWriteAction(Ctx, std::move(IF));
+  // Clear out output stream after file has been written incase more files are
+  // stubifed.
+  Ctx.OutStream = nullptr;
+}
+
+static void stubifyDirectory(const StringRef InputPath, Context &Ctx) {
+  assert(InputPath.back() != '/' && "Unexpected / at end of input path.");
+  StringMap<std::vector<SymLink>> SymLinks;
+  StringMap<std::unique_ptr<InterfaceFile>> Dylibs;
+  StringMap<std::string> OriginalNames;
+  std::set<std::pair<std::string, bool>> LibsToDelete;
+
+  std::error_code EC;
+  for (sys::fs::recursive_directory_iterator IT(InputPath, EC), IE; IT != IE;
+       IT.increment(EC)) {
+    if (EC == std::errc::no_such_file_or_directory) {
+      reportWarning(IT->path() + ": " + EC.message());
+      continue;
+    }
+    if (EC)
+      reportError(IT->path() + ": " + EC.message());
+
+    // Skip header directories (include/Headers/PrivateHeaders) and module
+    // files.
+    StringRef Path = IT->path();
+    if (Path.ends_with("/include") || Path.ends_with("/Headers") ||
+        Path.ends_with("/PrivateHeaders") || Path.ends_with("/Modules") ||
+        Path.ends_with(".map") || Path.ends_with(".modulemap")) {
+      IT.no_push();
+      continue;
+    }
+
+    // Check if the entry is a symlink. We don't follow symlinks but we record
+    // their content.
+    bool IsSymLink;
+    if (auto EC = sys::fs::is_symlink_file(Path, IsSymLink))
+      reportError(Path + ": " + EC.message());
+
+    if (IsSymLink) {
+      IT.no_push();
+
+      bool ShouldSkip;
+      auto SymLinkEC = shouldSkipSymLink(Path, ShouldSkip);
+
+      // If symlink is broken, for some reason, we should continue
+      // trying to repair it before quitting.
+      if (!SymLinkEC && ShouldSkip)
+        continue;
+
+      if (Ctx.StubOpt.DeletePrivate &&
+          isPrivateLibrary(Path.drop_front(InputPath.size()), true)) {
+        LibsToDelete.emplace(Path, false);
+        continue;
+      }
+
+      SmallString<PATH_MAX> SymPath;
+      if (auto EC = read_link(Path, SymPath))
+        reportError("cannot read '" + Path + "' :" + EC.message());
+
+      // Sometimes there are broken symlinks that are absolute paths, which are
+      // invalid during build time, but would be correct during runtime. In the
+      // case of an absolute path we should check first if the path exists with
+      // the known locations as prefix.
+      SmallString<PATH_MAX> LinkSrc = Path;
+      SmallString<PATH_MAX> LinkTarget;
+      if (sys::path::is_absolute(SymPath)) {
+        LinkTarget = InputPath;
+        sys::path::append(LinkTarget, SymPath);
+
+        // TODO: Investigate supporting a file manager for file system accesses.
+        if (sys::fs::exists(LinkTarget)) {
+          // Convert the absolute path to an relative path.
+          if (auto ec = MachO::make_relative(LinkSrc, LinkTarget, SymPath))
+            reportError(LinkTarget + ": " + EC.message());
+        } else if (!sys::fs::exists(SymPath)) {
+          reportWarning("ignoring broken symlink: " + Path);
+          continue;
+        } else {
+          LinkTarget = SymPath;
+        }
+      } else {
+        LinkTarget = LinkSrc;
+        sys::path::remove_filename(LinkTarget);
+        sys::path::append(LinkTarget, SymPath);
+      }
+
+      // For Apple SDKs, the symlink src is guaranteed to be a canonical path
+      // because we don't follow symlinks when scanning. The symlink target is
+      // constructed from the symlink path and needs to be canonicalized.
+      if (auto ec = sys::fs::real_path(Twine(LinkTarget), LinkTarget)) {
+        reportWarning(LinkTarget + ": " + ec.message());
+        continue;
+      }
+
+      auto itr = SymLinks.insert({LinkTarget.c_str(), std::vector<SymLink>()});
+      itr.first->second.emplace_back(LinkSrc.str(), std::string(SymPath.str()));
+
+      continue;
+    }
+
+    bool IsDirectory = false;
+    if (auto EC = sys::fs::is_directory(Path, IsDirectory))
+      reportError(Path + ": " + EC.message());
+    if (IsDirectory)
+      continue;
+
+    if (Ctx.StubOpt.DeletePrivate &&
+        isPrivateLibrary(Path.drop_front(InputPath.size()))) {
+      IT.no_push();
+      LibsToDelete.emplace(Path, false);
+      continue;
+    }
+    auto IF = getInterfaceFile(Path);
+    if (Ctx.StubOpt.TraceLibs)
+      errs() << Path << "\n";
+
+    // Normalize path for map lookup by removing the extension.
+    SmallString<PATH_MAX> NormalizedPath(Path);
+    replace_extension(NormalizedPath, "");
+
+    if ((IF->getFileType() == FileType::MachO_DynamicLibrary) ||
+        (IF->getFileType() == FileType::MachO_DynamicLibrary_Stub)) {
+      OriginalNames[NormalizedPath.c_str()] = IF->getPath();
+
+      // Don't add this MachO dynamic library because we already have a
+      // text-based stub recorded for this path.
+      if (Dylibs.count(NormalizedPath.c_str()))
+        continue;
+    }
+
+    Dylibs[NormalizedPath.c_str()] = std::move(IF);
+  }
+
+  for (auto &Lib : Dylibs) {
+    auto &Dylib = Lib.second;
+    // Get the original file name.
+    SmallString<PATH_MAX> NormalizedPath(Dylib->getPath());
----------------
SamuelMarks wrote:

@cyndyishida It can technically only be known at runtime on Windows and can be > 32,766 https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation. Furthermore on POSIX you might want to actually query the file - https://pubs.opengroup.org/onlinepubs/9699919799/functions/pathconf.html

[`SmallVector`](https://llvm.org/doxygen/SmallVector_8h.html); which [`SmallString`](https://llvm.org/doxygen/classllvm_1_1SmallString.html) uses internally; exposes a [`reserve`](https://llvm.org/doxygen/classllvm_1_1SmallVectorImpl.html#a499ea32ca1b8d16cedfe01d1e5b08f29) method which you can make `PATH_MAX`, `MAX_PATH`; so that it has reasonable default allocations but can grow.

https://github.com/llvm/llvm-project/pull/76885