[clang-tools-extra] [compiler-rt] [clang] [llvm] [Profile] Allow profile merging with multiple correlate files. (PR #75957)

[Ellis Hoag via cfe-commits]

================
@@ -481,3 +509,49 @@ Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
   this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize);
   return Error::success();
 }
+
+llvm::Expected<std::unique_ptr<InstrProfCorrelators>> InstrProfCorrelators::get(
+    ArrayRef<std::pair<StringRef, InstrProfCorrelator::ProfCorrelatorKind>>
+        CorrelateInputs,
+    uint32_t MaxWarnings) {
+  StringMap<std::unique_ptr<InstrProfCorrelator>> CorrelatorMap;
+  StringMap<StringRef> FileMap;
+  auto WarnCounter =
+      std::make_unique<InstrProfCorrelator::AtomicWarningCounter>(MaxWarnings);
+  std::unique_ptr<std::mutex> CorrelateLock = std::make_unique<std::mutex>();
+  std::unique_ptr<std::mutex> WarnLock = std::make_unique<std::mutex>();
+  for (const auto &Input : CorrelateInputs) {
+    std::unique_ptr<InstrProfCorrelator> Correlator;
+    if (auto Err = InstrProfCorrelator::get(Input.first, Input.second,
+                                            *CorrelateLock.get(),
+                                            *WarnLock.get(), WarnCounter.get())
+                       .moveInto(Correlator))
+      return Err;
+    std::string BuildID = toHex(Correlator->getBuildID());
+    FileMap.try_emplace(BuildID, Input.first);
+    bool Inserted =
+        CorrelatorMap.try_emplace(BuildID, std::move(Correlator)).second;
+    if (!Inserted && WarnCounter->shouldEmitWarning()) {
+      std::lock_guard<std::mutex> Guard(*WarnLock);
+      WithColor::warning() << format(
+          "Duplicate build id (%s) found for %s and %s\n", BuildID.c_str(),
+          FileMap[BuildID].str().c_str(), Input.first.str().c_str());
+    }
+  }
+  return std::make_unique<InstrProfCorrelators>(
+      std::move(CorrelatorMap), std::move(CorrelateLock), std::move(WarnLock),
+      std::move(WarnCounter));
+}
+
+llvm::Expected<const InstrProfCorrelator *>
+InstrProfCorrelators::getCorrelator(object::BuildIDRef BuildID) const {
+  std::string BuildIDStr = toHex(BuildID);
+  auto I = CorrelatorMap.find(BuildIDStr);
+  if (I == CorrelatorMap.end())
+    return make_error<InstrProfError>(
+        instrprof_error::unable_to_correlate_profile,
+        "missing correlator file with build id " + BuildIDStr + "\n");
+  if (auto Err = I->getValue()->correlateProfileData())
----------------
ellishg wrote:

We can still process correlation files in parallel without processing them lazily. In `llvm-profdata.cpp` we would just process all the correlation files before processing any raw profiles. I guess we would still need an atomic counter to make sure we don't emit too many warnings.

https://github.com/llvm/llvm-project/pull/75957