[Lldb-commits] [lldb] [lldb][Windows] use pipes when no terminal dimensions are sent (PR #203562)

[Charles Zablit via lldb-commits]

https://github.com/charles-zablit created https://github.com/llvm/llvm-project/pull/203562

Plumb `eLaunchFlagUsePipes` from the lldb-dap client through the gdb-remote protocol to lldb-server so the server's LaunchProcess can choose between ConPTY and anonymous pipes for inferior stdio.

This is needed for LLDB DAP in `internalConsole` mode.

Fixes `TestDAP_launch_args.py`, `TestDAP_launch_basic.py`, and `TestDAP_launch_shellExpandArguments_disabled.py` on Windows under `LLDB_USE_LLDB_SERVER=1`.

>From 88b366b38f5a4b0b3ff5f85b7899bce24d8f8d82 Mon Sep 17 00:00:00 2001
From: Charles Zablit <c_zablit at apple.com>
Date: Fri, 12 Jun 2026 16:09:14 +0100
Subject: [PATCH] [lldb][Windows] use pipes when no terminal dimensions are
 sent

---
 lldb/include/lldb/Host/ProcessLaunchInfo.h           |  2 ++
 lldb/source/Host/windows/PseudoConsole.cpp           |  2 ++
 .../GDBRemoteCommunicationServerCommon.cpp           |  5 ++++-
 .../gdb-remote/GDBRemoteCommunicationServerLLGS.cpp  | 12 ++++++++++++
 .../Plugins/Process/gdb-remote/ProcessGDBRemote.cpp  |  8 ++++++--
 5 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/lldb/include/lldb/Host/ProcessLaunchInfo.h b/lldb/include/lldb/Host/ProcessLaunchInfo.h
index 99f4d48aa4f27..70e93e95d93cb 100644
--- a/lldb/include/lldb/Host/ProcessLaunchInfo.h
+++ b/lldb/include/lldb/Host/ProcessLaunchInfo.h
@@ -186,6 +186,8 @@ class ProcessLaunchInfo : public ProcessInfo {
     m_stdio_window_size.rows = rows;
   }
 
+  STDIOWindowSize GetSTDIOWindowSize() const { return m_stdio_window_size; }
+
 protected:
   FileSpec m_working_dir;
   std::string m_plugin_name;
diff --git a/lldb/source/Host/windows/PseudoConsole.cpp b/lldb/source/Host/windows/PseudoConsole.cpp
index 2b8293393bfdf..c9ecf3a97bb99 100644
--- a/lldb/source/Host/windows/PseudoConsole.cpp
+++ b/lldb/source/Host/windows/PseudoConsole.cpp
@@ -189,6 +189,8 @@ void PseudoConsole::Close() {
   if (m_conpty_handle != INVALID_HANDLE_VALUE)
     kernel32.ClosePseudoConsole(m_conpty_handle);
   m_conpty_handle = INVALID_HANDLE_VALUE;
+  if (m_mode == Mode::Pipe && m_conpty_output != INVALID_HANDLE_VALUE)
+    CancelIoEx(m_conpty_output, nullptr);
   SetStopping(false);
   m_cv.notify_all();
 }
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index d676699ef3176..d710dfec95873 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -991,7 +991,10 @@ GDBRemoteCommunicationServerCommon::Handle_QSetSTDIOWindowSize(
       continue;
     *dest = static_cast<uint16_t>(parsed);
   }
-  if (cols == 0 || rows == 0)
+  // 0x0 is a valid request: it signals "no terminal" and a redirection
+  // backend that supports an alternative path (anonymous pipes on Windows
+  // ConPTY) can switch on it. Reject only the malformed cases.
+  if ((cols == 0) != (rows == 0))
     return SendErrorResponse(28);
 
   m_process_launch_info.SetSTDIOWindowSize(cols, rows);
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
index 0f14231cadbd5..00113f693aa19 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
@@ -291,8 +291,20 @@ Status GDBRemoteCommunicationServerLLGS::LaunchProcess() {
   m_process_launch_info.GetFlags().Set(eLaunchFlagDebug);
 
   if (should_forward_stdio) {
+#if defined(_WIN32)
+    ProcessLaunchInfo::STDIOWindowSize win_size =
+        m_process_launch_info.GetSTDIOWindowSize();
+    if (win_size.cols == 0 && win_size.rows == 0) {
+      if (llvm::Error Err = m_process_launch_info.SetUpPipeRedirection())
+        return Status::FromError(std::move(Err));
+    } else {
+      if (llvm::Error Err = m_process_launch_info.SetUpPtyRedirection())
+        return Status::FromError(std::move(Err));
+    }
+#else
     if (llvm::Error Err = m_process_launch_info.SetUpPtyRedirection())
       return Status::FromError(std::move(Err));
+#endif
   }
 
   {
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 2fc6dbb546f79..5f59cbeb2dda0 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -843,8 +843,12 @@ Status ProcessGDBRemote::DoLaunch(lldb_private::Module *exe_module,
     if (stderr_file_spec)
       m_gdb_comm.SetSTDERR(stderr_file_spec);
 
-    auto [terminal_cols, terminal_rows] = GetClientTerminalSize();
-    m_gdb_comm.SetSTDIOWindowSize(terminal_cols, terminal_rows);
+    if (launch_flags & eLaunchFlagUsePipes) {
+      m_gdb_comm.SetSTDIOWindowSize(0, 0);
+    } else {
+      auto [terminal_cols, terminal_rows] = GetClientTerminalSize();
+      m_gdb_comm.SetSTDIOWindowSize(terminal_cols, terminal_rows);
+    }
 
     m_gdb_comm.SetDisableASLR(launch_flags & eLaunchFlagDisableASLR);
     m_gdb_comm.SetDetachOnError(launch_flags & eLaunchFlagDetachOnError);