[PATCH] D53567: [Support] Enable color diagnostics for mintty

[Peiyuan Song via Phabricator via llvm-commits]

SquallATF updated this revision to Diff 170797.
SquallATF added a comment.

- run clang-format


Repository:
  rL LLVM

https://reviews.llvm.org/D53567

Files:
  lib/Support/Windows/Process.inc


Index: lib/Support/Windows/Process.inc
===================================================================
--- lib/Support/Windows/Process.inc
+++ lib/Support/Windows/Process.inc
@@ -18,6 +18,7 @@
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/WindowsError.h"
 #include <malloc.h>
+#include <winternl.h>
 
 // The Windows.h header must be after LLVM and standard headers.
 #include "WindowsSupport.h"
@@ -228,6 +229,39 @@
   return std::error_code();
 }
 
+static bool IsMintty(HANDLE Console) {
+  bool ret = false;
+  HMODULE ntdll = GetModuleHandle("ntdll.dll");
+  if (ntdll != INVALID_HANDLE_VALUE) {
+
+    typedef NTSTATUS NTAPI func_NtQueryObject(HANDLE, OBJECT_INFORMATION_CLASS,
+                                              PVOID, ULONG, PULONG);
+    func_NtQueryObject *fNtQueryObject =
+        (func_NtQueryObject *)GetProcAddress(ntdll, "NtQueryObject");
+    if (fNtQueryObject) {
+
+      ULONG s = 0xffff * sizeof(WCHAR);
+      OBJECT_NAME_INFORMATION *oni = (OBJECT_NAME_INFORMATION *)malloc(s);
+      ULONG len;
+      /* mintty uses a named pipe like "ptyNNNN-to-master".  */
+      if (!fNtQueryObject(Console, ObjectNameInformation, oni, s, &len)) {
+        wchar_t namedPipe[] = L"\\Device\\NamedPipe\\";
+        size_t l1 = sizeof(namedPipe) / 2 - 1;
+        wchar_t toMaster[] = L"-to-master";
+        size_t l2 = sizeof(toMaster) / 2 - 1;
+        USHORT name_length = oni->Name.Length / 2;
+        if (name_length > l1 + l2 &&
+            !memcmp(oni->Name.Buffer, namedPipe, l1 * 2) &&
+            !memcmp(oni->Name.Buffer + (name_length - l2), toMaster, l2 * 2))
+          ret = true;
+      }
+
+      free(oni);
+    }
+  }
+  return ret;
+}
+
 std::error_code
 windows::GetCommandLineArguments(SmallVectorImpl<const char *> &Args,
                                  BumpPtrAllocator &Alloc) {
@@ -283,8 +317,14 @@
 }
 
 bool Process::FileDescriptorIsDisplayed(int fd) {
-  DWORD Mode;  // Unused
-  return (GetConsoleMode((HANDLE)_get_osfhandle(fd), &Mode) != 0);
+  DWORD Mode; // Unused
+  HANDLE Console;
+  Console = (HANDLE)_get_osfhandle(fd);
+  bool ret = (GetConsoleMode(Console, &Mode) != 0);
+  if (!ret) {
+    ret = IsMintty(Console);
+  }
+  return ret;
 }
 
 unsigned Process::StandardOutColumns() {
@@ -318,15 +358,21 @@
 
 static bool UseANSI = false;
 void Process::UseANSIEscapeCodes(bool enable) {
+  DWORD Mode;
+  HANDLE Console;
 #if defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING)
   if (enable) {
-    HANDLE Console = GetStdHandle(STD_OUTPUT_HANDLE);
-    DWORD Mode;
+    Console = GetStdHandle(STD_OUTPUT_HANDLE);
     GetConsoleMode(Console, &Mode);
     Mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
     SetConsoleMode(Console, Mode);
   }
 #endif
+  Console = GetStdHandle(STD_ERROR_HANDLE);
+  bool ret = (GetConsoleMode(Console, &Mode) != 0);
+  if (!ret) {
+    enable = enable || IsMintty(Console);
+  }
   UseANSI = enable;
 }
 


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D53567.170797.patch
Type: text/x-patch
Size: 2919 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20181024/4547a528/attachment.bin>