changeset 32664:2d56e6d23be7

win32: add a method to enable ANSI color code processing on Windows 10

SetConsoleMode() fails with an invalid parameter error if given this option
prior to Windows 10, so indicate that to the caller instead of doing explicit
version checks.

--- a/mercurial/win32.py	Sun Jun 04 08:57:37 2017 -0500
+++ b/mercurial/win32.py	Mon May 22 22:00:56 2017 -0400
@@ -131,6 +131,7 @@
                 ('srWindow', _SMALL_RECT),
                 ('dwMaximumWindowSize', _COORD)]
 
+_STD_OUTPUT_HANDLE = _DWORD(-11).value
 _STD_ERROR_HANDLE = _DWORD(-12).value
 
 # CreateToolhelp32Snapshot, Process32First, Process32Next
@@ -202,6 +203,12 @@
 _kernel32.SetConsoleCtrlHandler.argtypes = [_SIGNAL_HANDLER, _BOOL]
 _kernel32.SetConsoleCtrlHandler.restype = _BOOL
 
+_kernel32.SetConsoleMode.argtypes = [_HANDLE, _DWORD]
+_kernel32.SetConsoleMode.restype = _BOOL
+
+_kernel32.GetConsoleMode.argtypes = [_HANDLE, ctypes.c_void_p]
+_kernel32.GetConsoleMode.restype = _BOOL
+
 _kernel32.GetStdHandle.argtypes = [_DWORD]
 _kernel32.GetStdHandle.restype = _HANDLE
 
@@ -372,6 +379,29 @@
     height = csbi.srWindow.Bottom - csbi.srWindow.Top + 1
     return width, height
 
+def enablevtmode():
+    '''Enable virtual terminal mode for the associated console.  Return True if
+    enabled, else False.'''
+
+    ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x4
+
+    handle = _kernel32.GetStdHandle(_STD_OUTPUT_HANDLE) # don't close the handle
+    if handle == _INVALID_HANDLE_VALUE:
+        return False
+
+    mode = _DWORD(0)
+
+    if not _kernel32.GetConsoleMode(handle, ctypes.byref(mode)):
+        return False
+
+    if (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0:
+        mode.value |= ENABLE_VIRTUAL_TERMINAL_PROCESSING
+
+        if not _kernel32.SetConsoleMode(handle, mode):
+            return False
+
+    return True
+
 def _1stchild(pid):
     '''return the 1st found child of the given pid