[Lldb-commits] [lldb] [lldb][Windows] use pipes when no terminal dimensions are sent (PR #203562)

[Charles Zablit via lldb-commits]

https://github.com/charles-zablit updated https://github.com/llvm/llvm-project/pull/203562

>From 00d88746f78d289201ddf4e0fa0d2c0d4174bc5c Mon Sep 17 00:00:00 2001
From: Charles Zablit <c_zablit at apple.com>
Date: Mon, 15 Jun 2026 16:38:25 +0100
Subject: [PATCH] track explicit size

---
 lldb/include/lldb/Host/ProcessLaunchInfo.h          |  8 ++++++++
 lldb/source/Host/windows/PseudoConsole.cpp          |  2 ++
 .../GDBRemoteCommunicationServerCommon.cpp          |  5 ++++-
 .../gdb-remote/GDBRemoteCommunicationServerLLGS.cpp | 13 +++++++++++++
 .../Plugins/Process/gdb-remote/ProcessGDBRemote.cpp |  8 ++++++--
 5 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/lldb/include/lldb/Host/ProcessLaunchInfo.h b/lldb/include/lldb/Host/ProcessLaunchInfo.h
index 99f4d48aa4f27..096d51357d857 100644
--- a/lldb/include/lldb/Host/ProcessLaunchInfo.h
+++ b/lldb/include/lldb/Host/ProcessLaunchInfo.h
@@ -184,8 +184,15 @@ class ProcessLaunchInfo : public ProcessInfo {
   void SetSTDIOWindowSize(uint16_t cols, uint16_t rows) {
     m_stdio_window_size.cols = cols;
     m_stdio_window_size.rows = rows;
+    m_stdio_window_size_explicit = true;
   }
 
+  bool IsSTDIOWindowSizeExplicit() const {
+    return m_stdio_window_size_explicit;
+  }
+
+  STDIOWindowSize GetSTDIOWindowSize() const { return m_stdio_window_size; }
+
 protected:
   FileSpec m_working_dir;
   std::string m_plugin_name;
@@ -199,6 +206,7 @@ class ProcessLaunchInfo : public ProcessInfo {
   std::string m_event_data; // A string passed to the plugin launch, having no
                             // meaning to the upper levels of lldb.
   STDIOWindowSize m_stdio_window_size;
+  bool m_stdio_window_size_explicit = false;
 };
 }
 
diff --git a/lldb/source/Host/windows/PseudoConsole.cpp b/lldb/source/Host/windows/PseudoConsole.cpp
index 2b8293393bfdf..3a28adacfea53 100644
--- a/lldb/source/Host/windows/PseudoConsole.cpp
+++ b/lldb/source/Host/windows/PseudoConsole.cpp
@@ -188,6 +188,8 @@ void PseudoConsole::Close() {
   std::unique_lock<std::mutex> guard(m_mutex);
   if (m_conpty_handle != INVALID_HANDLE_VALUE)
     kernel32.ClosePseudoConsole(m_conpty_handle);
+  if (m_mode == Mode::Pipe && m_conpty_output != INVALID_HANDLE_VALUE)
+    CancelIoEx(m_conpty_output, nullptr);
   m_conpty_handle = INVALID_HANDLE_VALUE;
   SetStopping(false);
   m_cv.notify_all();
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index d676699ef3176..d710dfec95873 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -991,7 +991,10 @@ GDBRemoteCommunicationServerCommon::Handle_QSetSTDIOWindowSize(
       continue;
     *dest = static_cast<uint16_t>(parsed);
   }
-  if (cols == 0 || rows == 0)
+  // 0x0 is a valid request: it signals "no terminal" and a redirection
+  // backend that supports an alternative path (anonymous pipes on Windows
+  // ConPTY) can switch on it. Reject only the malformed cases.
+  if ((cols == 0) != (rows == 0))
     return SendErrorResponse(28);
 
   m_process_launch_info.SetSTDIOWindowSize(cols, rows);
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
index 0f14231cadbd5..f265fcb71be4e 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
@@ -291,8 +291,21 @@ Status GDBRemoteCommunicationServerLLGS::LaunchProcess() {
   m_process_launch_info.GetFlags().Set(eLaunchFlagDebug);
 
   if (should_forward_stdio) {
+#if defined(_WIN32)
+    ProcessLaunchInfo::STDIOWindowSize win_size =
+        m_process_launch_info.GetSTDIOWindowSize();
+    if (m_process_launch_info.IsSTDIOWindowSizeExplicit() &&
+        win_size.cols == 0 && win_size.rows == 0) {
+      if (llvm::Error Err = m_process_launch_info.SetUpPipeRedirection())
+        return Status::FromError(std::move(Err));
+    } else {
+      if (llvm::Error Err = m_process_launch_info.SetUpPtyRedirection())
+        return Status::FromError(std::move(Err));
+    }
+#else
     if (llvm::Error Err = m_process_launch_info.SetUpPtyRedirection())
       return Status::FromError(std::move(Err));
+#endif
   }
 
   {
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 2fc6dbb546f79..5f59cbeb2dda0 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -843,8 +843,12 @@ Status ProcessGDBRemote::DoLaunch(lldb_private::Module *exe_module,
     if (stderr_file_spec)
       m_gdb_comm.SetSTDERR(stderr_file_spec);
 
-    auto [terminal_cols, terminal_rows] = GetClientTerminalSize();
-    m_gdb_comm.SetSTDIOWindowSize(terminal_cols, terminal_rows);
+    if (launch_flags & eLaunchFlagUsePipes) {
+      m_gdb_comm.SetSTDIOWindowSize(0, 0);
+    } else {
+      auto [terminal_cols, terminal_rows] = GetClientTerminalSize();
+      m_gdb_comm.SetSTDIOWindowSize(terminal_cols, terminal_rows);
+    }
 
     m_gdb_comm.SetDisableASLR(launch_flags & eLaunchFlagDisableASLR);
     m_gdb_comm.SetDetachOnError(launch_flags & eLaunchFlagDetachOnError);