[PATCH] D53567: [Support] Enable color diagnostics for mintty

[Peiyuan Song via Phabricator via llvm-commits]

SquallATF created this revision.
Herald added subscribers: llvm-commits, kristina.

Repository:
  rL LLVM

https://reviews.llvm.org/D53567

Files:
  lib/Support/Windows/Process.inc


Index: lib/Support/Windows/Process.inc
===================================================================
--- lib/Support/Windows/Process.inc
+++ lib/Support/Windows/Process.inc
@@ -18,6 +18,7 @@
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/WindowsError.h"
 #include <malloc.h>
+#include <winternl.h>
 
 // The Windows.h header must be after LLVM and standard headers.
 #include "WindowsSupport.h"
@@ -228,6 +229,40 @@
   return std::error_code();
 }
 
+static bool IsMintty(HANDLE Console) {
+  bool ret = false;
+  HMODULE ntdll = GetModuleHandle ("ntdll.dll");
+  if (ntdll != INVALID_HANDLE_VALUE){
+
+    typedef NTSTATUS NTAPI func_NtQueryObject (HANDLE, OBJECT_INFORMATION_CLASS,
+                                           PVOID, ULONG, PULONG);
+    func_NtQueryObject *fNtQueryObject =
+     (func_NtQueryObject*) GetProcAddress (ntdll, "NtQueryObject");
+    if (fNtQueryObject){
+
+      ULONG s = 0xffff * sizeof (WCHAR);
+      OBJECT_NAME_INFORMATION *oni = (OBJECT_NAME_INFORMATION*) malloc (s);
+      ULONG len;
+      /* mintty uses a named pipe like "ptyNNNN-to-master".  */
+      if (!fNtQueryObject (Console, ObjectNameInformation, oni, s, &len))
+        {
+          wchar_t namedPipe[] = L"\\Device\\NamedPipe\\";
+          size_t l1 = sizeof (namedPipe) / 2 - 1;
+          wchar_t toMaster[] = L"-to-master";
+          size_t l2 = sizeof (toMaster) / 2 - 1;
+          USHORT name_length = oni->Name.Length / 2;
+          if (name_length > l1 + l2 &&
+              !memcmp (oni->Name.Buffer, namedPipe, l1 * 2) &&
+              !memcmp (oni->Name.Buffer + (name_length - l2), toMaster, l2 * 2))
+              ret = true;
+        }
+
+      free (oni);
+    }
+  }
+  return ret;
+}
+
 std::error_code
 windows::GetCommandLineArguments(SmallVectorImpl<const char *> &Args,
                                  BumpPtrAllocator &Alloc) {
@@ -284,7 +319,13 @@
 
 bool Process::FileDescriptorIsDisplayed(int fd) {
   DWORD Mode;  // Unused
-  return (GetConsoleMode((HANDLE)_get_osfhandle(fd), &Mode) != 0);
+  HANDLE Console;
+  Console = (HANDLE)_get_osfhandle(fd);
+  bool ret = (GetConsoleMode(Console, &Mode) != 0);
+  if (!ret){
+    ret = IsMintty(Console);
+  }
+  return ret;
 }
 
 unsigned Process::StandardOutColumns() {
@@ -318,15 +359,21 @@
 
 static bool UseANSI = false;
 void Process::UseANSIEscapeCodes(bool enable) {
+  DWORD Mode;
+  HANDLE Console;
 #if defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING)
   if (enable) {
-    HANDLE Console = GetStdHandle(STD_OUTPUT_HANDLE);
-    DWORD Mode;
+    Console = GetStdHandle(STD_OUTPUT_HANDLE);
     GetConsoleMode(Console, &Mode);
     Mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
     SetConsoleMode(Console, Mode);
   }
 #endif
+  Console = GetStdHandle (STD_ERROR_HANDLE);
+  bool ret = (GetConsoleMode(Console, &Mode) != 0);
+  if (!ret){
+    enable = enable || IsMintty(Console);
+  }
   UseANSI = enable;
 }
 


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D53567.170624.patch
Type: text/x-patch
Size: 2924 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20181023/8262152f/attachment.bin>