[Lldb-commits] [lldb] a758205 - [trace][intelpt] Support system-wide tracing [9] - Collect and return context switch traces

Walter Erquinigo via lldb-commits lldb-commits at lists.llvm.org
Wed Jun 15 12:08:10 PDT 2022


Author: Walter Erquinigo
Date: 2022-06-15T12:07:59-07:00
New Revision: a758205951d3a1eb60bdb65146ab23e8567f30a7

URL: https://github.com/llvm/llvm-project/commit/a758205951d3a1eb60bdb65146ab23e8567f30a7
DIFF: https://github.com/llvm/llvm-project/commit/a758205951d3a1eb60bdb65146ab23e8567f30a7.diff

LOG: [trace][intelpt] Support system-wide tracing [9] - Collect and return context switch traces

- Add collection of context switches per cpu grouped with the per-cpu intel pt traces.
- Move the state handling from the interl pt trace class to the PerfEvent one.
- Add support for stopping and enabling perf event groups.
- Return context switch entries as part of the jLLDBTraceGetState response.
- Move the triggers of whenever the process stopped or resumed. Now the will-resume notification is in a better location, which will ensure that we'll capture the instructions that will be executed.
- Remove IntelPTSingleBufferTraceUP. The unique pointer was useless.
- Add unit tests

Differential Revision: https://reviews.llvm.org/D125897

Added: 
    

Modified: 
    lldb/docs/lldb-gdb-remote.txt
    lldb/include/lldb/Host/common/NativeProcessProtocol.h
    lldb/include/lldb/Utility/TraceIntelPTGDBRemotePackets.h
    lldb/source/Host/common/NativeProcessProtocol.cpp
    lldb/source/Plugins/Process/Linux/IntelPTCollector.cpp
    lldb/source/Plugins/Process/Linux/IntelPTCollector.h
    lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.cpp
    lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.h
    lldb/source/Plugins/Process/Linux/IntelPTProcessTrace.h
    lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp
    lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.h
    lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.cpp
    lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.h
    lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
    lldb/source/Plugins/Process/Linux/NativeProcessLinux.h
    lldb/source/Plugins/Process/Linux/Perf.cpp
    lldb/source/Plugins/Process/Linux/Perf.h
    lldb/source/Utility/TraceIntelPTGDBRemotePackets.cpp
    lldb/test/API/commands/trace/multiple-threads/TestTraceStartStopMultipleThreads.py

Removed: 
    


################################################################################
diff  --git a/lldb/docs/lldb-gdb-remote.txt b/lldb/docs/lldb-gdb-remote.txt
index a7b1803a92f02..1c9f29f50e299 100644
--- a/lldb/docs/lldb-gdb-remote.txt
+++ b/lldb/docs/lldb-gdb-remote.txt
@@ -516,6 +516,8 @@ read packet: OK/E<error code>;AAAAAAAAA
 //
 //  Binary data kinds:
 //    - traceBuffer: trace buffer for a thread or a core.
+//    - perfContextSwitchTrace: context switch trace for a core generated by
+//                              perf_event_open.
 //    - procfsCpuInfo: contents of the /proc/cpuinfo file.
 //
 //  Counter info kinds:

diff  --git a/lldb/include/lldb/Host/common/NativeProcessProtocol.h b/lldb/include/lldb/Host/common/NativeProcessProtocol.h
index 7cb353d477a5c..63dbb3a62ea5f 100644
--- a/lldb/include/lldb/Host/common/NativeProcessProtocol.h
+++ b/lldb/include/lldb/Host/common/NativeProcessProtocol.h
@@ -310,6 +310,12 @@ class NativeProcessProtocol {
     virtual Extension GetSupportedExtensions() const { return {}; }
   };
 
+  /// Notify tracers that the target process will resume
+  virtual void NotifyTracersProcessWillResume() {}
+
+  /// Notify tracers that the target process just stopped
+  virtual void NotifyTracersProcessDidStop() {}
+
   /// Start tracing a process or its threads.
   ///
   /// \param[in] json_params
@@ -461,9 +467,6 @@ class NativeProcessProtocol {
 
   NativeThreadProtocol *GetThreadByIDUnlocked(lldb::tid_t tid);
 
-  /// Notify tracers that the state of the target process has changed.
-  virtual void NotifyTracersProcessStateChanged(lldb::StateType state) {}
-
 private:
   void SynchronouslyNotifyProcessStateChanged(lldb::StateType state);
   llvm::Expected<SoftwareBreakpoint>

diff  --git a/lldb/include/lldb/Utility/TraceIntelPTGDBRemotePackets.h b/lldb/include/lldb/Utility/TraceIntelPTGDBRemotePackets.h
index 788d8c7809de4..abe67ff9c89b7 100644
--- a/lldb/include/lldb/Utility/TraceIntelPTGDBRemotePackets.h
+++ b/lldb/include/lldb/Utility/TraceIntelPTGDBRemotePackets.h
@@ -25,6 +25,7 @@ namespace lldb_private {
 struct IntelPTDataKinds {
   static const char *kProcFsCpuInfo;
   static const char *kTraceBuffer;
+  static const char *kPerfContextSwitchTrace;
 };
 
 /// jLLDBTraceStart gdb-remote packet

diff  --git a/lldb/source/Host/common/NativeProcessProtocol.cpp b/lldb/source/Host/common/NativeProcessProtocol.cpp
index 31fc00538b5dd..be521a31cb377 100644
--- a/lldb/source/Host/common/NativeProcessProtocol.cpp
+++ b/lldb/source/Host/common/NativeProcessProtocol.cpp
@@ -312,7 +312,16 @@ void NativeProcessProtocol::SynchronouslyNotifyProcessStateChanged(
   Log *log = GetLog(LLDBLog::Process);
 
   m_delegate.ProcessStateChanged(this, state);
-  NotifyTracersProcessStateChanged(state);
+
+  switch (state) {
+  case eStateStopped:
+  case eStateExited:
+  case eStateCrashed:
+    NotifyTracersProcessDidStop();
+    break;
+  default:
+    break;
+  }
 
   LLDB_LOG(log, "sent state notification [{0}] from process {1}", state,
            GetID());

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTCollector.cpp b/lldb/source/Plugins/Process/Linux/IntelPTCollector.cpp
index 08fa6924121cb..1016488a8d66b 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTCollector.cpp
+++ b/lldb/source/Plugins/Process/Linux/IntelPTCollector.cpp
@@ -116,9 +116,14 @@ Error IntelPTCollector::TraceStart(const TraceIntelPTStartRequest &request) {
   }
 }
 
-void IntelPTCollector::OnProcessStateChanged(lldb::StateType state) {
+void IntelPTCollector::ProcessWillResume() {
   if (m_process_trace_up)
-    m_process_trace_up->OnProcessStateChanged(state);
+    m_process_trace_up->ProcessWillResume();
+}
+
+void IntelPTCollector::ProcessDidStop() {
+  if (m_process_trace_up)
+    m_process_trace_up->ProcessDidStop();
 }
 
 Error IntelPTCollector::OnThreadCreated(lldb::tid_t tid) {

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTCollector.h b/lldb/source/Plugins/Process/Linux/IntelPTCollector.h
index 5173ef4cc04fe..19f8d7bb06abf 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTCollector.h
+++ b/lldb/source/Plugins/Process/Linux/IntelPTCollector.h
@@ -37,8 +37,12 @@ class IntelPTCollector {
 
   static bool IsSupported();
 
-  /// To be invoked whenever the state of the target process has changed.
-  void OnProcessStateChanged(lldb::StateType state);
+  /// To be invoked as soon as we know the process stopped.
+  void ProcessDidStop();
+
+  /// To be invoked before the process will resume, so that we can capture the
+  /// first instructions after the resume.
+  void ProcessWillResume();
 
   /// If "process tracing" is enabled, then trace the given thread.
   llvm::Error OnThreadCreated(lldb::tid_t tid);

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.cpp b/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.cpp
index c32a1b9d5e550..ce0bb485cae98 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.cpp
+++ b/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.cpp
@@ -33,6 +33,54 @@ static Error IncludePerfEventParanoidMessageInError(Error &&error) {
       toString(std::move(error)).c_str());
 }
 
+static Expected<PerfEvent> CreateContextSwitchTracePerfEvent(
+    bool disabled, lldb::core_id_t core_id,
+    IntelPTSingleBufferTrace &intelpt_core_trace) {
+  Log *log = GetLog(POSIXLog::Trace);
+#ifndef PERF_ATTR_SIZE_VER5
+  return createStringError(inconvertibleErrorCode(),
+                           "Intel PT Linux perf event not supported");
+#else
+  perf_event_attr attr;
+  memset(&attr, 0, sizeof(attr));
+  attr.size = sizeof(attr);
+  attr.sample_period = 0;
+  attr.sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME;
+  attr.type = PERF_TYPE_SOFTWARE;
+  attr.context_switch = 1;
+  attr.exclude_kernel = 1;
+  attr.sample_id_all = 1;
+  attr.exclude_hv = 1;
+  attr.disabled = disabled;
+
+  // The given perf configuration will product context switch records of 32
+  // bytes each. Assuming that every context switch will be emitted twice (one
+  // for context switch ins and another one for context switch outs), and that a
+  // context switch will happen at least every half a millisecond per core, we
+  // need 500 * 32 bytes (~16 KB) for a trace of one second, which is much more
+  // than what a regular intel pt trace can get. Pessimistically we pick as
+  // 32KiB for the size of our context switch trace.
+
+  uint64_t data_buffer_size = 32768;
+  uint64_t data_buffer_numpages = data_buffer_size / getpagesize();
+
+  LLDB_LOG(log, "Will create context switch trace buffer of size {0}",
+           data_buffer_size);
+
+  if (Expected<PerfEvent> perf_event = PerfEvent::Init(
+          attr, /*pid=*/None, core_id,
+          intelpt_core_trace.GetPerfEvent().GetFd(), /*flags=*/0)) {
+    if (Error mmap_err =
+            perf_event->MmapMetadataAndBuffers(data_buffer_numpages, 0)) {
+      return std::move(mmap_err);
+    }
+    return perf_event;
+  } else {
+    return perf_event.takeError();
+  }
+#endif
+}
+
 Expected<IntelPTProcessTraceUP>
 IntelPTMultiCoreTrace::StartOnAllCores(const TraceIntelPTStartRequest &request,
                                        NativeProcessProtocol &process) {
@@ -46,55 +94,63 @@ IntelPTMultiCoreTrace::StartOnAllCores(const TraceIntelPTStartRequest &request,
         "The process can't be traced because the process trace size limit "
         "has been reached. Consider retracing with a higher limit.");
 
-  llvm::DenseMap<core_id_t, IntelPTSingleBufferTraceUP> buffers;
+  DenseMap<core_id_t, std::pair<IntelPTSingleBufferTrace, ContextSwitchTrace>>
+      traces;
+
   for (core_id_t core_id : *core_ids) {
-    if (Expected<IntelPTSingleBufferTraceUP> core_trace =
-            IntelPTSingleBufferTrace::Start(request, /*tid=*/None, core_id,
-                                            TraceCollectionState::Paused))
-      buffers.try_emplace(core_id, std::move(*core_trace));
-    else
+    Expected<IntelPTSingleBufferTrace> core_trace =
+        IntelPTSingleBufferTrace::Start(request, /*tid=*/None, core_id,
+                                        /*disabled=*/true);
+    if (!core_trace)
       return IncludePerfEventParanoidMessageInError(core_trace.takeError());
+
+    if (Expected<PerfEvent> context_switch_trace =
+            CreateContextSwitchTracePerfEvent(/*disabled=*/true, core_id,
+                                              core_trace.get())) {
+      traces.try_emplace(core_id,
+                         std::make_pair(std::move(*core_trace),
+                                        std::move(*context_switch_trace)));
+    } else {
+      return context_switch_trace.takeError();
+    }
   }
 
   return IntelPTProcessTraceUP(
-      new IntelPTMultiCoreTrace(std::move(buffers), process));
+      new IntelPTMultiCoreTrace(std::move(traces), process));
 }
 
 void IntelPTMultiCoreTrace::ForEachCore(
     std::function<void(core_id_t core_id, IntelPTSingleBufferTrace &core_trace)>
         callback) {
   for (auto &it : m_traces_per_core)
-    callback(it.first, *it.second);
+    callback(it.first, it.second.first);
 }
 
-void IntelPTMultiCoreTrace::OnProcessStateChanged(lldb::StateType state) {
-  if (m_process_state == state)
-    return;
-  switch (state) {
-  case eStateStopped:
-  case eStateExited: {
-    ForEachCore([](core_id_t core_id, IntelPTSingleBufferTrace &core_trace) {
-      if (Error err =
-              core_trace.ChangeCollectionState(TraceCollectionState::Paused)) {
-        LLDB_LOG_ERROR(GetLog(POSIXLog::Trace), std::move(err),
-                       "Unable to pause the core trace for core {0}", core_id);
-      }
-    });
-    break;
-  }
-  case eStateRunning: {
-    ForEachCore([](core_id_t core_id, IntelPTSingleBufferTrace &core_trace) {
-      if (Error err =
-              core_trace.ChangeCollectionState(TraceCollectionState::Running)) {
-        LLDB_LOG_ERROR(GetLog(POSIXLog::Trace), std::move(err),
-                       "Unable to resume the core trace for core {0}", core_id);
-      }
-    });
-    break;
-  }
-  default:
-    break;
-  }
+void IntelPTMultiCoreTrace::ForEachCore(
+    std::function<void(core_id_t core_id,
+                       IntelPTSingleBufferTrace &intelpt_trace,
+                       PerfEvent &context_switch_trace)>
+        callback) {
+  for (auto &it : m_traces_per_core)
+    callback(it.first, it.second.first, it.second.second);
+}
+
+void IntelPTMultiCoreTrace::ProcessDidStop() {
+  ForEachCore([](core_id_t core_id, IntelPTSingleBufferTrace &core_trace) {
+    if (Error err = core_trace.Pause()) {
+      LLDB_LOG_ERROR(GetLog(POSIXLog::Trace), std::move(err),
+                     "Unable to pause the core trace for core {0}", core_id);
+    }
+  });
+}
+
+void IntelPTMultiCoreTrace::ProcessWillResume() {
+  ForEachCore([](core_id_t core_id, IntelPTSingleBufferTrace &core_trace) {
+    if (Error err = core_trace.Resume()) {
+      LLDB_LOG_ERROR(GetLog(POSIXLog::Trace), std::move(err),
+                     "Unable to resume the core trace for core {0}", core_id);
+    }
+  });
 }
 
 TraceGetStateResponse IntelPTMultiCoreTrace::GetState() {
@@ -106,10 +162,13 @@ TraceGetStateResponse IntelPTMultiCoreTrace::GetState() {
 
   state.cores.emplace();
   ForEachCore([&](lldb::core_id_t core_id,
-                  const IntelPTSingleBufferTrace &core_trace) {
+                  const IntelPTSingleBufferTrace &core_trace,
+                  const PerfEvent &context_switch_trace) {
     state.cores->push_back(
         {core_id,
-         {{IntelPTDataKinds::kTraceBuffer, core_trace.GetTraceBufferSize()}}});
+         {{IntelPTDataKinds::kTraceBuffer, core_trace.GetTraceBufferSize()},
+          {IntelPTDataKinds::kPerfContextSwitchTrace,
+           context_switch_trace.GetEffectiveDataBufferSize()}}});
   });
 
   return state;

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.h b/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.h
index e0ef795481c16..5c73067f6c5ba 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.h
+++ b/lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.h
@@ -24,6 +24,8 @@ namespace lldb_private {
 namespace process_linux {
 
 class IntelPTMultiCoreTrace : public IntelPTProcessTrace {
+  using ContextSwitchTrace = PerfEvent;
+
 public:
   /// Start tracing all CPU cores.
   ///
@@ -51,7 +53,24 @@ class IntelPTMultiCoreTrace : public IntelPTProcessTrace {
                                       IntelPTSingleBufferTrace &core_trace)>
                        callback);
 
-  void OnProcessStateChanged(lldb::StateType state) override;
+  /// Execute the provided callback on each core that is being traced.
+  ///
+  /// \param[in] callback.core_id
+  ///   The core id that is being traced.
+  ///
+  /// \param[in] callback.intelpt_trace
+  ///   The single-buffer intel pt trace instance for the given core.
+  ///
+  /// \param[in] callback.context_switch_trace
+  ///   The perf event collecting context switches for the given core.
+  void ForEachCore(std::function<void(lldb::core_id_t core_id,
+                                      IntelPTSingleBufferTrace &intelpt_trace,
+                                      PerfEvent &context_switch_trace)>
+                       callback);
+
+  void ProcessDidStop() override;
+
+  void ProcessWillResume() override;
 
   TraceGetStateResponse GetState() override;
 
@@ -65,17 +84,18 @@ class IntelPTMultiCoreTrace : public IntelPTProcessTrace {
   GetBinaryData(const TraceGetBinaryDataRequest &request) override;
 
 private:
+  /// This assumes that all underlying perf_events for each core are part of the
+  /// same perf event group.
   IntelPTMultiCoreTrace(
-      llvm::DenseMap<lldb::core_id_t, IntelPTSingleBufferTraceUP>
+      llvm::DenseMap<lldb::core_id_t,
+                     std::pair<IntelPTSingleBufferTrace, ContextSwitchTrace>>
           &&traces_per_core,
       NativeProcessProtocol &process)
       : m_traces_per_core(std::move(traces_per_core)), m_process(process) {}
 
-  llvm::DenseMap<lldb::core_id_t, IntelPTSingleBufferTraceUP> m_traces_per_core;
-
-  /// The initial state is stopped because tracing can only start when the
-  /// process is paused.
-  lldb::StateType m_process_state = lldb::StateType::eStateStopped;
+  llvm::DenseMap<lldb::core_id_t,
+                 std::pair<IntelPTSingleBufferTrace, ContextSwitchTrace>>
+      m_traces_per_core;
 
   /// The target process.
   NativeProcessProtocol &m_process;

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTProcessTrace.h b/lldb/source/Plugins/Process/Linux/IntelPTProcessTrace.h
index ec4e42602d3c9..c893866df37a5 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTProcessTrace.h
+++ b/lldb/source/Plugins/Process/Linux/IntelPTProcessTrace.h
@@ -21,19 +21,9 @@ class IntelPTProcessTrace {
 public:
   virtual ~IntelPTProcessTrace() = default;
 
-  /// This method should be invoked as early as possible whenever the process
-  /// resumes or stops so that intel-pt collection is not enabled when
-  /// the process is not running. A case in which this is useful in when
-  /// tracing is done per-core. In this case we want to prevent polluting the
-  /// core traces with executions of unrelated processes, which increases the
-  /// data loss of the target process, given that core traces don't filter by
-  /// process.
-  /// A possible way to avoid this is to use CR3 filtering, which is equivalent
-  /// to process filtering, but the perf_event API doesn't support it.
-  ///
-  /// \param[in] state
-  ///     The new state of the target process.
-  virtual void OnProcessStateChanged(lldb::StateType state){};
+  virtual void ProcessDidStop() {}
+
+  virtual void ProcessWillResume() {}
 
   /// Construct a minimal jLLDBTraceGetState response for this process trace.
   virtual TraceGetStateResponse GetState() = 0;

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp b/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp
index a587b5826af23..2d3aad52dbb12 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp
+++ b/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp
@@ -205,23 +205,12 @@ size_t IntelPTSingleBufferTrace::GetTraceBufferSize() const {
   return m_perf_event.GetAuxBuffer().size();
 }
 
-Error IntelPTSingleBufferTrace::ChangeCollectionState(
-    TraceCollectionState new_state) {
-  if (new_state == m_collection_state)
-    return Error::success();
+Error IntelPTSingleBufferTrace::Pause() {
+  return m_perf_event.DisableWithIoctl();
+}
 
-  switch (new_state) {
-  case TraceCollectionState::Paused:
-    if (Error err = m_perf_event.DisableWithIoctl())
-      return err;
-    break;
-  case TraceCollectionState::Running:
-    if (Error err = m_perf_event.EnableWithIoctl())
-      return err;
-    break;
-  }
-  m_collection_state = new_state;
-  return Error::success();
+Error IntelPTSingleBufferTrace::Resume() {
+  return m_perf_event.EnableWithIoctl();
 }
 
 Expected<std::vector<uint8_t>>
@@ -240,42 +229,13 @@ IntelPTSingleBufferTrace::GetTraceBuffer(size_t offset, size_t size) {
   //
   // This is achieved by the PERF_EVENT_IOC_DISABLE ioctl request, as
   // mentioned in the man page of perf_event_open.
-  TraceCollectionState previous_state = m_collection_state;
-  if (Error err = ChangeCollectionState(TraceCollectionState::Paused))
-    return std::move(err);
-
-  std::vector<uint8_t> data(size, 0);
-  perf_event_mmap_page &mmap_metadata = m_perf_event.GetMetadataPage();
-  Log *log = GetLog(POSIXLog::Trace);
-  uint64_t head = mmap_metadata.aux_head;
-
-  LLDB_LOG(log, "Aux size -{0} , Head - {1}", mmap_metadata.aux_size, head);
-
-  /**
-   * When configured as ring buffer, the aux buffer keeps wrapping around
-   * the buffer and its not possible to detect how many times the buffer
-   * wrapped. Initially the buffer is filled with zeros,as shown below
-   * so in order to get complete buffer we first copy firstpartsize, followed
-   * by any left over part from beginning to aux_head
-   *
-   * aux_offset [d,d,d,d,d,d,d,d,0,0,0,0,0,0,0,0,0,0,0] aux_size
-   *                 aux_head->||<- firstpartsize  ->|
-   *
-   * */
-
-  MutableArrayRef<uint8_t> buffer(data);
-  ReadCyclicBuffer(buffer, m_perf_event.GetAuxBuffer(),
-                   static_cast<size_t>(head), offset);
-
-  if (Error err = ChangeCollectionState(previous_state))
-    return std::move(err);
-
-  return data;
+  return m_perf_event.ReadFlushedOutAuxCyclicBuffer(offset, size);
 }
 
-Expected<IntelPTSingleBufferTraceUP> IntelPTSingleBufferTrace::Start(
-    const TraceIntelPTStartRequest &request, Optional<lldb::tid_t> tid,
-    Optional<core_id_t> core_id, TraceCollectionState initial_state) {
+Expected<IntelPTSingleBufferTrace>
+IntelPTSingleBufferTrace::Start(const TraceIntelPTStartRequest &request,
+                                Optional<lldb::tid_t> tid,
+                                Optional<core_id_t> core_id, bool disabled) {
 #ifndef PERF_ATTR_SIZE_VER5
   return createStringError(inconvertibleErrorCode(),
                            "Intel PT Linux perf event not supported");
@@ -303,7 +263,7 @@ Expected<IntelPTSingleBufferTraceUP> IntelPTSingleBufferTrace::Start(
       }));
   if (!attr)
     return attr.takeError();
-  attr->disabled = initial_state == TraceCollectionState::Paused;
+  attr->disabled = disabled;
 
   LLDB_LOG(log, "Will create trace buffer of size {0}",
            request.trace_buffer_size);
@@ -313,11 +273,13 @@ Expected<IntelPTSingleBufferTraceUP> IntelPTSingleBufferTrace::Start(
             /*num_data_pages=*/0, aux_buffer_numpages)) {
       return std::move(mmap_err);
     }
-    IntelPTSingleBufferTraceUP trace_up(
-        new IntelPTSingleBufferTrace(std::move(*perf_event), initial_state));
-    return std::move(trace_up);
+    return IntelPTSingleBufferTrace(std::move(*perf_event));
   } else {
     return perf_event.takeError();
   }
 #endif
 }
+
+const PerfEvent &IntelPTSingleBufferTrace::GetPerfEvent() const {
+  return m_perf_event;
+}

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.h b/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.h
index c22e26e31519a..99e7e416e02f7 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.h
+++ b/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.h
@@ -23,15 +23,6 @@ namespace process_linux {
 
 llvm::Expected<uint32_t> GetIntelPTOSEventType();
 
-class IntelPTSingleBufferTrace;
-
-using IntelPTSingleBufferTraceUP = std::unique_ptr<IntelPTSingleBufferTrace>;
-
-enum class TraceCollectionState {
-  Running,
-  Paused,
-};
-
 /// This class wraps a single perf event collecting intel pt data in a single
 /// buffer.
 class IntelPTSingleBufferTrace {
@@ -48,17 +39,17 @@ class IntelPTSingleBufferTrace {
   /// \param[in] core_id
   ///     The CPU core id where to trace. If \b None, then this traces all CPUs.
   ///
-  /// \param[in] initial_state
-  ///     The initial trace collection state.
+  /// \param[in] disabled
+  ///     Whether to start the tracing paused.
   ///
   /// \return
   ///   A \a IntelPTSingleBufferTrace instance if tracing was successful, or
   ///   an \a llvm::Error otherwise.
-  static llvm::Expected<IntelPTSingleBufferTraceUP>
+  static llvm::Expected<IntelPTSingleBufferTrace>
   Start(const TraceIntelPTStartRequest &request,
         llvm::Optional<lldb::tid_t> tid,
-        llvm::Optional<lldb::core_id_t> core_id,
-        TraceCollectionState initial_state);
+        llvm::Optional<lldb::core_id_t> core_id = llvm::None,
+        bool disabled = false);
 
   /// \return
   ///    The bytes requested by a jLLDBTraceGetBinaryData packet that was routed
@@ -89,16 +80,23 @@ class IntelPTSingleBufferTrace {
   ///     trace instance.
   size_t GetTraceBufferSize() const;
 
-  /// Change the collection state for this trace.
-  ///
-  /// This is a no-op if \p state is the same as the current state.
+  /// Resume the collection of this trace.
   ///
-  /// \param[in] state
-  ///     The new state.
+  /// \return
+  ///     An error if the trace couldn't be resumed. If the trace is already
+  ///     running, this returns \a Error::success().
+  llvm::Error Resume();
+
+  /// Pause the collection of this trace.
   ///
   /// \return
-  ///     An error if the state couldn't be changed.
-  llvm::Error ChangeCollectionState(TraceCollectionState state);
+  ///     An error if the trace couldn't be paused. If the trace is already
+  ///     paused, this returns \a Error::success().
+  llvm::Error Pause();
+
+  /// \return
+  ///     The underlying PerfEvent for this trace.
+  const PerfEvent &GetPerfEvent() const;
 
 private:
   /// Construct new \a IntelPTSingleBufferThreadTrace. Users are supposed to
@@ -110,17 +108,11 @@ class IntelPTSingleBufferTrace {
   ///
   /// \param[in] collection_state
   ///   The initial collection state for the provided perf_event.
-  IntelPTSingleBufferTrace(PerfEvent &&perf_event,
-                           TraceCollectionState collection_state)
-      : m_perf_event(std::move(perf_event)),
-        m_collection_state(collection_state) {}
+  IntelPTSingleBufferTrace(PerfEvent &&perf_event)
+      : m_perf_event(std::move(perf_event)) {}
 
   /// perf event configured for IntelPT.
   PerfEvent m_perf_event;
-
-  /// The initial state is stopped because tracing can only start when the
-  /// process is paused.
-  TraceCollectionState m_collection_state;
 };
 
 } // namespace process_linux

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.cpp b/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.cpp
index 2ae179ce795f5..d7206b193e8f6 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.cpp
+++ b/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.cpp
@@ -22,7 +22,7 @@ Error IntelPTThreadTraceCollection::TraceStop(lldb::tid_t tid) {
   if (it == m_thread_traces.end())
     return createStringError(inconvertibleErrorCode(),
                              "Thread %" PRIu64 " not currently traced", tid);
-  m_total_buffer_size -= it->second->GetTraceBufferSize();
+  m_total_buffer_size -= it->second.GetTraceBufferSize();
   m_thread_traces.erase(tid);
   return Error::success();
 }
@@ -33,14 +33,13 @@ Error IntelPTThreadTraceCollection::TraceStart(
     return createStringError(inconvertibleErrorCode(),
                              "Thread %" PRIu64 " already traced", tid);
 
-  Expected<IntelPTSingleBufferTraceUP> trace_up =
-      IntelPTSingleBufferTrace::Start(request, tid, /*core_id=*/None,
-                                      TraceCollectionState::Running);
-  if (!trace_up)
-    return trace_up.takeError();
+  Expected<IntelPTSingleBufferTrace> trace =
+      IntelPTSingleBufferTrace::Start(request, tid);
+  if (!trace)
+    return trace.takeError();
 
-  m_total_buffer_size += (*trace_up)->GetTraceBufferSize();
-  m_thread_traces.try_emplace(tid, std::move(*trace_up));
+  m_total_buffer_size += trace->GetTraceBufferSize();
+  m_thread_traces.try_emplace(tid, std::move(*trace));
   return Error::success();
 }
 
@@ -52,7 +51,7 @@ void IntelPTThreadTraceCollection::ForEachThread(
     std::function<void(lldb::tid_t tid, IntelPTSingleBufferTrace &thread_trace)>
         callback) {
   for (auto &it : m_thread_traces)
-    callback(it.first, *it.second);
+    callback(it.first, it.second);
 }
 
 Expected<IntelPTSingleBufferTrace &>
@@ -61,7 +60,7 @@ IntelPTThreadTraceCollection::GetTracedThread(lldb::tid_t tid) {
   if (it == m_thread_traces.end())
     return createStringError(inconvertibleErrorCode(),
                              "Thread %" PRIu64 " not currently traced", tid);
-  return *it->second.get();
+  return it->second;
 }
 
 void IntelPTThreadTraceCollection::Clear() {

diff  --git a/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.h b/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.h
index 3dfbc2ee34d1b..f343e436d692a 100644
--- a/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.h
+++ b/lldb/source/Plugins/Process/Linux/IntelPTThreadTraceCollection.h
@@ -60,7 +60,7 @@ class IntelPTThreadTraceCollection {
   size_t GetTracedThreadsCount() const;
 
 private:
-  llvm::DenseMap<lldb::tid_t, IntelPTSingleBufferTraceUP> m_thread_traces;
+  llvm::DenseMap<lldb::tid_t, IntelPTSingleBufferTrace> m_thread_traces;
   /// Total actual thread buffer size in bytes
   size_t m_total_buffer_size = 0;
 };

diff  --git a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
index a7869d2241aec..429c68a9dc78e 100644
--- a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
@@ -898,6 +898,8 @@ Status NativeProcessLinux::Resume(const ResumeActionList &resume_actions) {
   Log *log = GetLog(POSIXLog::Process);
   LLDB_LOG(log, "pid {0}", GetID());
 
+  NotifyTracersProcessWillResume();
+
   bool software_single_step = !SupportHardwareSingleStepping();
 
   if (software_single_step) {
@@ -1665,9 +1667,12 @@ void NativeProcessLinux::StopTrackingThread(NativeThreadLinux &thread) {
   SignalIfAllThreadsStopped();
 }
 
-void NativeProcessLinux::NotifyTracersProcessStateChanged(
-    lldb::StateType state) {
-  m_intel_pt_collector.OnProcessStateChanged(state);
+void NativeProcessLinux::NotifyTracersProcessDidStop() {
+  m_intel_pt_collector.ProcessDidStop();
+}
+
+void NativeProcessLinux::NotifyTracersProcessWillResume() {
+  m_intel_pt_collector.ProcessWillResume();
 }
 
 Status NativeProcessLinux::NotifyTracersOfNewThread(lldb::tid_t tid) {

diff  --git a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.h b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.h
index d8afc48c650ea..65b8d5d6b37c0 100644
--- a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.h
+++ b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.h
@@ -209,7 +209,9 @@ class NativeProcessLinux : public NativeProcessELF,
   /// stopping for threads being destroyed.
   Status NotifyTracersOfThreadDestroyed(lldb::tid_t tid);
 
-  void NotifyTracersProcessStateChanged(lldb::StateType state) override;
+  void NotifyTracersProcessWillResume() override;
+
+  void NotifyTracersProcessDidStop() override;
 
   /// Writes the raw event message code (vis-a-vis PTRACE_GETEVENTMSG)
   /// corresponding to the given thread ID to the memory pointed to by @p

diff  --git a/lldb/source/Plugins/Process/Linux/Perf.cpp b/lldb/source/Plugins/Process/Linux/Perf.cpp
index c1ae9d26dcd0b..d11a54e702592 100644
--- a/lldb/source/Plugins/Process/Linux/Perf.cpp
+++ b/lldb/source/Plugins/Process/Linux/Perf.cpp
@@ -130,7 +130,8 @@ llvm::Expected<PerfEvent> PerfEvent::Init(perf_event_attr &attr,
         llvm::formatv("perf event syscall failed: {0}", std::strerror(errno));
     return llvm::createStringError(llvm::inconvertibleErrorCode(), err_msg);
   }
-  return PerfEvent{fd};
+  return PerfEvent(fd, attr.disabled ? CollectionState::Disabled
+                                     : CollectionState::Enabled);
 }
 
 llvm::Expected<PerfEvent> PerfEvent::Init(perf_event_attr &attr,
@@ -143,7 +144,7 @@ llvm::Expected<resource_handle::MmapUP>
 PerfEvent::DoMmap(void *addr, size_t length, int prot, int flags,
                   long int offset, llvm::StringRef buffer_name) {
   errno = 0;
-  auto mmap_result = ::mmap(nullptr, length, prot, flags, GetFd(), offset);
+  auto mmap_result = ::mmap(addr, length, prot, flags, GetFd(), offset);
 
   if (mmap_result == MAP_FAILED) {
     std::string err_msg =
@@ -157,7 +158,7 @@ PerfEvent::DoMmap(void *addr, size_t length, int prot, int flags,
 llvm::Error PerfEvent::MmapMetadataAndDataBuffer(size_t num_data_pages) {
   size_t mmap_size = (num_data_pages + 1) * getpagesize();
   if (Expected<resource_handle::MmapUP> mmap_metadata_data =
-          DoMmap(nullptr, mmap_size, PROT_WRITE, MAP_SHARED, 0,
+          DoMmap(nullptr, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, 0,
                  "metadata and data buffer")) {
     m_metadata_data_base = std::move(mmap_metadata_data.get());
     return Error::success();
@@ -221,18 +222,72 @@ ArrayRef<uint8_t> PerfEvent::GetAuxBuffer() const {
            static_cast<size_t>(mmap_metadata.aux_size)};
 }
 
-Error PerfEvent::DisableWithIoctl() const {
-  if (ioctl(*m_fd, PERF_EVENT_IOC_DISABLE) < 0)
+Expected<std::vector<uint8_t>>
+PerfEvent::ReadFlushedOutAuxCyclicBuffer(size_t offset, size_t size) {
+  CollectionState previous_state = m_collection_state;
+  if (Error err = DisableWithIoctl())
+    return std::move(err);
+
+  std::vector<uint8_t> data(size, 0);
+  perf_event_mmap_page &mmap_metadata = GetMetadataPage();
+  Log *log = GetLog(POSIXLog::Trace);
+  uint64_t head = mmap_metadata.aux_head;
+
+  LLDB_LOG(log, "Aux size -{0} , Head - {1}", mmap_metadata.aux_size, head);
+
+  /**
+   * When configured as ring buffer, the aux buffer keeps wrapping around
+   * the buffer and its not possible to detect how many times the buffer
+   * wrapped. Initially the buffer is filled with zeros,as shown below
+   * so in order to get complete buffer we first copy firstpartsize, followed
+   * by any left over part from beginning to aux_head
+   *
+   * aux_offset [d,d,d,d,d,d,d,d,0,0,0,0,0,0,0,0,0,0,0] aux_size
+   *                 aux_head->||<- firstpartsize  ->|
+   *
+   * */
+
+  MutableArrayRef<uint8_t> buffer(data);
+  ReadCyclicBuffer(buffer, GetAuxBuffer(), static_cast<size_t>(head), offset);
+
+  if (previous_state == CollectionState::Enabled) {
+    if (Error err = EnableWithIoctl())
+      return std::move(err);
+  }
+
+  return data;
+}
+
+Error PerfEvent::DisableWithIoctl() {
+  if (m_collection_state == CollectionState::Disabled)
+    return Error::success();
+
+  if (ioctl(*m_fd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP) < 0)
     return createStringError(inconvertibleErrorCode(),
                              "Can't disable perf event. %s",
                              std::strerror(errno));
+
+  m_collection_state = CollectionState::Disabled;
   return Error::success();
 }
 
-Error PerfEvent::EnableWithIoctl() const {
-  if (ioctl(*m_fd, PERF_EVENT_IOC_ENABLE) < 0)
+Error PerfEvent::EnableWithIoctl() {
+  if (m_collection_state == CollectionState::Enabled)
+    return Error::success();
+
+  if (ioctl(*m_fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP) < 0)
     return createStringError(inconvertibleErrorCode(),
-                             "Can't disable perf event. %s",
+                             "Can't enable perf event. %s",
                              std::strerror(errno));
+
+  m_collection_state = CollectionState::Enabled;
   return Error::success();
 }
+
+size_t PerfEvent::GetEffectiveDataBufferSize() const {
+  perf_event_mmap_page &mmap_metadata = GetMetadataPage();
+  if (mmap_metadata.data_head < mmap_metadata.data_size)
+    return mmap_metadata.data_head;
+  else
+    return mmap_metadata.data_size; // The buffer has wrapped.
+}

diff  --git a/lldb/source/Plugins/Process/Linux/Perf.h b/lldb/source/Plugins/Process/Linux/Perf.h
index b249ab9858cf6..6bb943a30b891 100644
--- a/lldb/source/Plugins/Process/Linux/Perf.h
+++ b/lldb/source/Plugins/Process/Linux/Perf.h
@@ -98,6 +98,11 @@ void ReadCyclicBuffer(llvm::MutableArrayRef<uint8_t> &dst,
 /// Handles the management of the event's file descriptor and mmap'ed
 /// regions.
 class PerfEvent {
+  enum class CollectionState {
+    Enabled,
+    Disabled,
+  };
+
 public:
   /// Create a new performance monitoring event via the perf_event_open syscall.
   ///
@@ -213,27 +218,59 @@ class PerfEvent {
   ///   \a ArrayRef<uint8_t> extending \a aux_size bytes from \a aux_offset.
   llvm::ArrayRef<uint8_t> GetAuxBuffer() const;
 
-  /// Use the ioctl API to disable the perf event. This doesn't terminate the
-  /// perf event.
+  /// Read the aux buffer managed by this perf event. To ensure that the
+  /// data is up-to-date and is not corrupted by read-write race conditions, the
+  /// underlying perf_event is paused during read, and later it's returned to
+  /// its initial state. The returned data will be linear, i.e. it will fix the
+  /// circular wrapping the might exist int he buffer.
+  ///
+  /// \param[in] offset
+  ///     Offset of the data to read.
+  ///
+  /// \param[in] size
+  ///     Number of bytes to read.
+  ///
+  /// \return
+  ///     A vector with the requested binary data. The vector will have the
+  ///     size of the requested \a size. Non-available positions will be
+  ///     filled with zeroes.
+  llvm::Expected<std::vector<uint8_t>>
+  ReadFlushedOutAuxCyclicBuffer(size_t offset, size_t size);
+
+  /// Use the ioctl API to disable the perf event and all the events in its
+  /// group. This doesn't terminate the perf event.
+  ///
+  /// This is no-op if the perf event is already disabled.
   ///
   /// \return
   ///   An Error if the perf event couldn't be disabled.
-  llvm::Error DisableWithIoctl() const;
+  llvm::Error DisableWithIoctl();
 
-  /// Use the ioctl API to enable the perf event.
+  /// Use the ioctl API to enable the perf event and all the events in its
+  /// group.
+  ///
+  /// This is no-op if the perf event is already enabled.
   ///
   /// \return
   ///   An Error if the perf event couldn't be enabled.
-  llvm::Error EnableWithIoctl() const;
+  llvm::Error EnableWithIoctl();
+
+  /// \return
+  ///   The size in bytes of the section of the data buffer that has effective
+  ///   data.
+  size_t GetEffectiveDataBufferSize() const;
 
 private:
   /// Create new \a PerfEvent.
   ///
   /// \param[in] fd
   ///   File descriptor of the perf event.
-  PerfEvent(long fd)
+  ///
+  /// \param[in] initial_state
+  ///   Initial collection state configured for this perf_event.
+  PerfEvent(long fd, CollectionState initial_state)
       : m_fd(new long(fd), resource_handle::FileDescriptorDeleter()),
-        m_metadata_data_base(), m_aux_base() {}
+        m_collection_state(initial_state) {}
 
   /// Wrapper for \a mmap to provide custom error messages.
   ///
@@ -270,6 +307,8 @@ class PerfEvent {
   /// AUX buffer is a separate region for high-bandwidth data streams
   /// such as IntelPT.
   resource_handle::MmapUP m_aux_base;
+  /// The state of the underlying perf_event.
+  CollectionState m_collection_state;
 };
 
 /// Load \a PerfTscConversionParameters from \a perf_event_mmap_page, if

diff  --git a/lldb/source/Utility/TraceIntelPTGDBRemotePackets.cpp b/lldb/source/Utility/TraceIntelPTGDBRemotePackets.cpp
index aff77fddc4242..f4658f34cb0f9 100644
--- a/lldb/source/Utility/TraceIntelPTGDBRemotePackets.cpp
+++ b/lldb/source/Utility/TraceIntelPTGDBRemotePackets.cpp
@@ -15,6 +15,8 @@ namespace lldb_private {
 
 const char *IntelPTDataKinds::kProcFsCpuInfo = "procfsCpuInfo";
 const char *IntelPTDataKinds::kTraceBuffer = "traceBuffer";
+const char *IntelPTDataKinds::kPerfContextSwitchTrace =
+    "perfContextSwitchTrace";
 
 bool TraceIntelPTStartRequest::IsPerCoreTracing() const {
   return per_core_tracing.getValueOr(false);

diff  --git a/lldb/test/API/commands/trace/multiple-threads/TestTraceStartStopMultipleThreads.py b/lldb/test/API/commands/trace/multiple-threads/TestTraceStartStopMultipleThreads.py
index b76c0559ff776..859f486150482 100644
--- a/lldb/test/API/commands/trace/multiple-threads/TestTraceStartStopMultipleThreads.py
+++ b/lldb/test/API/commands/trace/multiple-threads/TestTraceStartStopMultipleThreads.py
@@ -1,4 +1,5 @@
 import lldb
+import json
 from intelpt_testcase import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
@@ -195,8 +196,40 @@ def testStartPerCoreSession(self):
         self.traceStopThread(error="True",
             substrs=["Can't stop tracing an individual thread when per-core process tracing is enabled"])
 
-        # The GetState packet should return trace buffers per core and at least one traced thread
-        self.expect("""process plugin packet send 'jLLDBTraceGetState:{"type":"intel-pt"}]'""",
-            substrs=['''[{"kind":"traceBuffer","size":4096}],"coreId":''', '"tid":'])
+        # We move forward a little bit to collect some data
+        self.expect("b 19")
+        self.expect("c")
+
+        # We will assert that the trace state will contain valid context switch and trace buffer entries
+
+        # We first parse the json response from the custom packet
+        self.runCmd("""process plugin packet send 'jLLDBTraceGetState:{"type":"intel-pt"}]'""")
+        response_header = 'response: '
+        output = None
+        for line in self.res.GetOutput().splitlines():
+            if line.find(response_header) != -1:
+                response = line[line.find(response_header) + len(response_header):].strip()
+                output = json.loads(response)
+
+        self.assertTrue(output is not None)
+        self.assertIn("cores", output)
+        found_non_empty_context_switch = False
+
+        for core in output["cores"]:
+            context_switch_size = None
+            trace_buffer_size = None
+            for binary_data in core["binaryData"]:
+                if binary_data["kind"] == "traceBuffer":
+                    trace_buffer_size = binary_data["size"]
+                elif binary_data["kind"] == "perfContextSwitchTrace":
+                    context_switch_size = binary_data["size"]
+            self.assertTrue(context_switch_size is not None)
+            self.assertTrue(trace_buffer_size is not None)
+            if context_switch_size > 0:
+                found_non_empty_context_switch = True
+
+        # We must have captured the context switch of when the target resumed
+        self.assertTrue(found_non_empty_context_switch)
+
 
         self.traceStopProcess()


        


More information about the lldb-commits mailing list