[compiler-rt] [clang] [llvm] [clang-tools-extra] [Profile] Allow profile merging with multiple correlate files. (PR #75957)

[Zequan Wu via cfe-commits]

================
@@ -481,3 +509,49 @@ Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
   this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize);
   return Error::success();
 }
+
+llvm::Expected<std::unique_ptr<InstrProfCorrelators>> InstrProfCorrelators::get(
+    ArrayRef<std::pair<StringRef, InstrProfCorrelator::ProfCorrelatorKind>>
+        CorrelateInputs,
+    uint32_t MaxWarnings) {
+  StringMap<std::unique_ptr<InstrProfCorrelator>> CorrelatorMap;
+  StringMap<StringRef> FileMap;
+  auto WarnCounter =
+      std::make_unique<InstrProfCorrelator::AtomicWarningCounter>(MaxWarnings);
+  std::unique_ptr<std::mutex> CorrelateLock = std::make_unique<std::mutex>();
+  std::unique_ptr<std::mutex> WarnLock = std::make_unique<std::mutex>();
+  for (const auto &Input : CorrelateInputs) {
+    std::unique_ptr<InstrProfCorrelator> Correlator;
+    if (auto Err = InstrProfCorrelator::get(Input.first, Input.second,
+                                            *CorrelateLock.get(),
+                                            *WarnLock.get(), WarnCounter.get())
+                       .moveInto(Correlator))
+      return Err;
+    std::string BuildID = toHex(Correlator->getBuildID());
+    FileMap.try_emplace(BuildID, Input.first);
+    bool Inserted =
+        CorrelatorMap.try_emplace(BuildID, std::move(Correlator)).second;
+    if (!Inserted && WarnCounter->shouldEmitWarning()) {
+      std::lock_guard<std::mutex> Guard(*WarnLock);
+      WithColor::warning() << format(
+          "Duplicate build id (%s) found for %s and %s\n", BuildID.c_str(),
+          FileMap[BuildID].str().c_str(), Input.first.str().c_str());
+    }
+  }
+  return std::make_unique<InstrProfCorrelators>(
+      std::move(CorrelatorMap), std::move(CorrelateLock), std::move(WarnLock),
+      std::move(WarnCounter));
+}
+
+llvm::Expected<const InstrProfCorrelator *>
+InstrProfCorrelators::getCorrelator(object::BuildIDRef BuildID) const {
+  std::string BuildIDStr = toHex(BuildID);
+  auto I = CorrelatorMap.find(BuildIDStr);
+  if (I == CorrelatorMap.end())
+    return make_error<InstrProfError>(
+        instrprof_error::unable_to_correlate_profile,
+        "missing correlator file with build id " + BuildIDStr + "\n");
+  if (auto Err = I->getValue()->correlateProfileData())
----------------
ZequanWu wrote:

Right now, the parsing of correlation files is done before the parallelization starts. For each raw profiles, it parallelly associate parsed correlation info with lightweight raw profiles (which basically returns pointers to constant data and constant names)

> I guess we would still need an atomic counter to make sure we don't emit too many warnings.

The warnings are only emitted at the time when parsing the correlation files which is before the parallelization starts. So, at least for now, we don't need atomic counter. If we want to parallelly parse correlation files, then we need lock or atomic counter to handle warning emissions.

https://github.com/llvm/llvm-project/pull/75957