Better method of determining a console handle.

ANSI.c

@@ -101,6 +101,9 @@
 
   v1.62, 17 & 18 July, 2013:
     another method to obtain LLW for 64->32 injection.
+
+  v1.64, 2 August, 2013:
+    better method of determining a console handle (see IsConsoleHandle).
 */
 
 #include "ansicon.h"
@@ -1524,10 +1527,30 @@ HMODULE WINAPI MyLoadLibraryExW( LPCWSTR lpFileName, HANDLE hFile,
 }
 
 
+//-----------------------------------------------------------------------------
+//   IsConsoleHandle
+// Determine if the handle is writing to the console, with processed output.
+//-----------------------------------------------------------------------------
+BOOL IsConsoleHandle( HANDLE h )
+{
+  DWORD mode;
+
+  if (!GetConsoleMode( h, &mode ))
+  {
+    // This fails if the handle isn't opened for reading.  Fortunately, it
+    // seems WriteConsole tests the handle before it tests the length.
+    return WriteConsole( h, NULL, 0, &mode, NULL );
+  }
+
+  return (mode & ENABLE_PROCESSED_OUTPUT);
+}
+
+
 //-----------------------------------------------------------------------------
 //   MyWrite...
-// It is the new function that must replace the original Write... function.
+// The new functions that must replace the original Write... functions.  These
-// This function have exactly the same signature as the original one.
+// functions have exactly the same signature as the original ones.  This
+// module is not hooked, so we can still call the original functions ourselves.
 //-----------------------------------------------------------------------------
 
 BOOL
@@ -1535,13 +1558,11 @@ WINAPI MyWriteConsoleA( HANDLE hCon, LPCVOID lpBuffer,
 			DWORD nNumberOfCharsToWrite,
 			LPDWORD lpNumberOfCharsWritten, LPVOID lpReserved )
 {
-  DWORD  Mode;
   LPWSTR buf;
   DWORD  len;
   BOOL	 rc = TRUE;
 
-  // if we write in a console buffer with processed output
+  if (IsConsoleHandle( hCon ))
-  if (GetConsoleMode( hCon, &Mode ) && (Mode & ENABLE_PROCESSED_OUTPUT))
   {
     UINT cp = GetConsoleOutputCP();
     DEBUGSTR( 4, L"\33WriteConsoleA: %lu \"%.*S\"",
@@ -1580,8 +1601,7 @@ WINAPI MyWriteConsoleW( HANDLE hCon, LPCVOID lpBuffer,
 			DWORD nNumberOfCharsToWrite,
 			LPDWORD lpNumberOfCharsWritten, LPVOID lpReserved )
 {
-  DWORD Mode;
+  if (IsConsoleHandle( hCon ))
-  if (GetConsoleMode( hCon, &Mode ) && (Mode & ENABLE_PROCESSED_OUTPUT))
   {
     DEBUGSTR( 4, L"\33WriteConsoleW: %lu \"%.*s\"",
 	      nNumberOfCharsToWrite, nNumberOfCharsToWrite, lpBuffer );
@@ -1598,8 +1618,7 @@ BOOL
 WINAPI MyWriteFile( HANDLE hFile, LPCVOID lpBuffer, DWORD nNumberOfBytesToWrite,
 		    LPDWORD lpNumberOfBytesWritten, LPOVERLAPPED lpOverlapped )
 {
-  DWORD Mode;
+  if (IsConsoleHandle( hFile ))
-  if (GetConsoleMode( hFile, &Mode ) && (Mode & ENABLE_PROCESSED_OUTPUT))
   {
     DEBUGSTR( 4, L"WriteFile->" );
     return MyWriteConsoleA( hFile, lpBuffer,
@@ -1608,7 +1627,6 @@ WINAPI MyWriteFile( HANDLE hFile, LPCVOID lpBuffer, DWORD nNumberOfBytesToWrite,
 			    lpOverlapped );
   }
 
-  // here, WriteFile is the old function (this module is not hooked)
   return WriteFile( hFile, lpBuffer, nNumberOfBytesToWrite,
 		    lpNumberOfBytesWritten, lpOverlapped );
 }
@@ -1619,9 +1637,9 @@ WINAPI MyWriteFile( HANDLE hFile, LPCVOID lpBuffer, DWORD nNumberOfBytesToWrite,
 UINT
 WINAPI My_lwrite( HFILE hFile, LPCSTR lpBuffer, UINT uBytes )
 {
-  DWORD Mode, written;
+  if (IsConsoleHandle( HHFILE hFile ))
-  if (GetConsoleMode( HHFILE hFile, &Mode ) && (Mode & ENABLE_PROCESSED_OUTPUT))
   {
+    DWORD written;
     DEBUGSTR( 4, L"_lwrite->" );
     MyWriteConsoleA( HHFILE hFile, lpBuffer, uBytes, &written, NULL );
     return written;
@@ -1633,9 +1651,9 @@ WINAPI My_lwrite( HFILE hFile, LPCSTR lpBuffer, UINT uBytes )
 long
 WINAPI My_hwrite( HFILE hFile, LPCSTR lpBuffer, long lBytes )
 {
-  DWORD Mode, written;
+  if (IsConsoleHandle( HHFILE hFile ))
-  if (GetConsoleMode( HHFILE hFile, &Mode ) && (Mode & ENABLE_PROCESSED_OUTPUT))
   {
+    DWORD written;
     DEBUGSTR( 4, L"_hwrite->" );
     MyWriteConsoleA( HHFILE hFile, lpBuffer, lBytes, &written, NULL );
     return written;

ansicon.c

@@ -78,7 +78,7 @@
     don't write the reset sequence if output is redirected.
 */
 
-#define PDATE L"25 July, 2013"
+#define PDATE L"2 August, 2013"
 
 #include "ansicon.h"
 #include "version.h"

readme.txt

@@ -3,7 +3,7 @@
 
 			 Copyright 2005-2013 Jason Hood
 
-			    Version 1.63.  Freeware
+			    Version 1.64.  Freeware
 
 
 Description
@@ -196,8 +196,8 @@ Sequences Ignored
 
     The following escape sequences are explicitly ignored.
 
-	\e(?		Designate G0 character set ('?' is anything).
+	\e(?		Designate G0 character set ('?' is any character).
-	\e)?		Designate G1 character set ('?' is anything).
+	\e)?		Designate G1 character set ('?' is any character).
 	\e[?... 	Private sequence
 	\e[>... 	Private sequence
 
@@ -270,6 +270,9 @@ Version History
 
     Legend: + added, - bug-fixed, * changed.
 
+    1.64 - 2 August, 2013:
+    - improved detection of console output.
+
     1.63 - 25 July, 2013:
     - don't write the reset sequence (when it's already installed) if output is
       redirected.
@@ -449,5 +452,5 @@ Distribution
     in LICENSE.txt.
 
 
-==========================
+===========================
-Jason Hood, 25 July, 2013.
+Jason Hood, 2 August, 2013.

version.h

@@ -2,11 +2,11 @@
   version.h - Version defines.
 */
 
-#define PVERS	L"1.63"         // wide string
+#define PVERS	L"1.64"         // wide string
-#define PVERSA	 "1.63"         // ANSI string (windres 2.16.91 didn't like L)
+#define PVERSA	 "1.64"         // ANSI string (windres 2.16.91 didn't like L)
-#define PVERE	L"163"          // wide environment string
+#define PVERE	L"164"          // wide environment string
-#define PVEREA	 "163"          // ANSI environment string
+#define PVEREA	 "164"          // ANSI environment string
-#define PVERB	1,6,3,0 	// binary (resource)
+#define PVERB	1,6,4,0 	// binary (resource)
 
 #ifdef _WIN64
 # define BITS L"64"