[Lldb-commits] [lldb] [llvm] [lldb][windows] add Windows Virtual Console support (PR #168729)

[Charles Zablit via lldb-commits]

================
@@ -0,0 +1,113 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/windows/PseudoConsole.h"
+
+#include <mutex>
+
+#include "lldb/Host/windows/windows.h"
+
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Errno.h"
+
+using namespace lldb_private;
+
+typedef HRESULT(WINAPI *CreatePseudoConsole_t)(COORD size, HANDLE hInput,
+                                               HANDLE hOutput, DWORD dwFlags,
+                                               HPCON *phPC);
+
+typedef VOID(WINAPI *ClosePseudoConsole_t)(HPCON hPC);
+
+struct Kernel32 {
+  Kernel32() {
+    hModule = LoadLibraryW(L"kernel32.dll");
+    CreatePseudoConsole_ =
+        (CreatePseudoConsole_t)GetProcAddress(hModule, "CreatePseudoConsole");
+    ClosePseudoConsole_ =
+        (ClosePseudoConsole_t)GetProcAddress(hModule, "ClosePseudoConsole");
+    isAvailable = (CreatePseudoConsole_ && ClosePseudoConsole_);
+  }
+
+  HRESULT CreatePseudoConsole(COORD size, HANDLE hInput, HANDLE hOutput,
+                              DWORD dwFlags, HPCON *phPC) {
+    return CreatePseudoConsole_(size, hInput, hOutput, dwFlags, phPC);
+  }
+
+  VOID ClosePseudoConsole(HPCON hPC) { return ClosePseudoConsole_(hPC); }
+
+  bool IsAvailable() { return isAvailable; }
+
+private:
+  HMODULE hModule;
+  CreatePseudoConsole_t CreatePseudoConsole_;
+  ClosePseudoConsole_t ClosePseudoConsole_;
+  bool isAvailable;
+};
+
+static Kernel32 kernel32;
+
+llvm::Error PseudoConsole::OpenPseudoConsole() {
+  if (!kernel32.IsAvailable())
+    return llvm::make_error<llvm::StringError>("ConPTY is not available",
+                                               llvm::errc::io_error);
+  HRESULT hr;
+  HANDLE hInputRead = INVALID_HANDLE_VALUE;
+  HANDLE hInputWrite = INVALID_HANDLE_VALUE;
+  HANDLE hOutputRead = INVALID_HANDLE_VALUE;
+  HANDLE hOutputWrite = INVALID_HANDLE_VALUE;
+
+  wchar_t pipe_name[MAX_PATH];
+  swprintf(pipe_name, MAX_PATH, L"\\\\.\\pipe\\conpty-lldb-%d-%p",
+           GetCurrentProcessId(), this);
+
+  // A 4096 bytes buffer should be large enough for the majority of console
+  // burst outputs.
+  hOutputRead =
+      CreateNamedPipeW(pipe_name, PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED,
----------------
charles-zablit wrote:

I think this should be done in a follow up: `PipeWindows::CreateNew` used `CreateNamedPipeA` instead of the `W` variant and the setup between `hOutputRead` and `hOutputWrite` differs from what `CreateNew` does.

https://github.com/llvm/llvm-project/pull/168729