[cmake-developers] [PATCH v7] For Windows encode process output to internally used encoding

Dāvis Mosāns davispuh at gmail.com
Mon Aug 15 16:34:21 EDT 2016


Typically Windows applications (eg. MSVC compiler) use current console's
codepage for output to pipes so we need to encode that to internally used
encoding (KWSYS_ENCODING_DEFAULT_CODEPAGE).
---
 Source/CMakeLists.txt              |   6 ++
 Source/ProcessOutput.cxx           | 141 +++++++++++++++++++++++++++++++++++++
 Source/ProcessOutput.hxx           |  39 ++++++++++
 Source/cmExecProgramCommand.cxx    |   3 +
 Source/cmExecuteProcessCommand.cxx |  11 ++-
 Source/cmProcessTools.cxx          |   9 ++-
 Source/cmSystemTools.cxx           |  11 ++-
 bootstrap                          |   5 +-
 8 files changed, 218 insertions(+), 7 deletions(-)
 create mode 100644 Source/ProcessOutput.cxx
 create mode 100644 Source/ProcessOutput.hxx

diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt
index cdc8fb1..46dd471 100644
--- a/Source/CMakeLists.txt
+++ b/Source/CMakeLists.txt
@@ -373,8 +373,14 @@ set(SRCS
   cm_sha2.c
   cm_utf8.h
   cm_utf8.c
+
+  ProcessOutput.cxx
+  ProcessOutput.hxx
   )
 
+SET_PROPERTY(SOURCE ProcessOutput.cxx APPEND PROPERTY COMPILE_DEFINITIONS
+  KWSYS_ENCODING_DEFAULT_CODEPAGE=${KWSYS_ENCODING_DEFAULT_CODEPAGE})
+
 set(COMMAND_INCLUDES "#include \"cmTargetPropCommandBase.cxx\"\n")
 list(APPEND SRCS cmTargetPropCommandBase.cxx)
 set_property(SOURCE cmTargetPropCommandBase.cxx PROPERTY HEADER_FILE_ONLY ON)
diff --git a/Source/ProcessOutput.cxx b/Source/ProcessOutput.cxx
new file mode 100644
index 0000000..6c66087
--- /dev/null
+++ b/Source/ProcessOutput.cxx
@@ -0,0 +1,141 @@
+/*============================================================================
+  CMake - Cross Platform Makefile Generator
+  Copyright 2000-2016 Kitware, Inc., Insight Software Consortium
+
+  Distributed under the OSI-approved BSD License (the "License");
+  see accompanying file Copyright.txt for details.
+
+  This software is distributed WITHOUT ANY WARRANTY; without even the
+  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the License for more information.
+============================================================================*/
+
+#include "ProcessOutput.hxx"
+
+#if defined(_WIN32)
+# include <windows.h>
+unsigned int ProcessOutput::defaultCodepage = KWSYS_ENCODING_DEFAULT_CODEPAGE;
+#endif
+
+ProcessOutput::ProcessOutput(unsigned int maxSize)
+{
+#if defined(_WIN32)
+  bufferSize = maxSize;
+  codepage = GetConsoleCP();
+  if (!codepage) {
+    codepage = GetACP();
+  }
+#else
+  static_cast<void>(maxSize);
+#endif
+}
+
+ProcessOutput::~ProcessOutput()
+{
+}
+
+bool ProcessOutput::DecodeText(std::string raw, std::string& decoded, size_t id)
+{
+  bool success = true;
+  decoded = raw;
+#if defined(_WIN32)
+  if (id > 0) {
+    if (rawparts.size() < id) {
+      rawparts.reserve(id);
+      while (rawparts.size() < id) rawparts.push_back(std::string());
+    }
+    raw = rawparts[id - 1] + raw;
+    rawparts[id - 1].clear();
+    decoded = raw;
+  }
+  if (raw.size() > 0 && codepage != defaultCodepage) {
+    success = false;
+    CPINFOEXW cpinfo;
+    if (id > 0 && raw.size() == bufferSize && GetCPInfoExW(codepage, 0, &cpinfo) == 1 && cpinfo.MaxCharSize > 1) {
+      if (cpinfo.MaxCharSize == 2 && cpinfo.LeadByte[0] != 0) {
+        LPSTR prevChar = CharPrevExA(codepage, raw.c_str(), raw.c_str() + raw.size(), 0);
+        bool isLeadByte = (*(prevChar + 1) == 0) && IsDBCSLeadByteEx(codepage, *prevChar);
+        if (isLeadByte) {
+          rawparts[id - 1] += *(raw.end() - 1);
+          raw.resize(raw.size() - 1);
+        }
+        success = DoDecodeText(raw, decoded, NULL);
+      } else {
+        bool restoreDecoded = false;
+        std::string firstDecoded = decoded;
+        wchar_t lastChar = 0;
+        for (UINT i = 0; i < cpinfo.MaxCharSize; i++) {
+          success = DoDecodeText(raw, decoded, &lastChar);
+          if (success && lastChar != 0) {
+            if (i == 0) {
+              firstDecoded = decoded;
+            }
+            if (lastChar == cpinfo.UnicodeDefaultChar) {
+              restoreDecoded = true;
+              rawparts[id - 1] = *(raw.end() - 1) + rawparts[id - 1];
+              raw.resize(raw.size() - 1);
+            } else {
+              restoreDecoded = false;
+              break;
+            }
+          } else {
+            break;
+          }
+        }
+        if (restoreDecoded) {
+          decoded = firstDecoded;
+          rawparts[id - 1].clear();
+        }
+      }
+    } else {
+      success = DoDecodeText(raw, decoded, NULL);
+    }
+  }
+#else
+  static_cast<void>(id);
+#endif
+  return success;
+}
+
+bool ProcessOutput::DecodeText(const char* data, size_t length, std::string& decoded, size_t id)
+{
+  return DecodeText(std::string(data, length), decoded, id);
+}
+
+bool ProcessOutput::DecodeText(std::vector<char> raw, std::vector<char>& decoded, size_t id)
+{
+  std::string str;
+  const bool success = DecodeText(std::string(raw.begin(), raw.end()), str, id);
+  decoded.assign(str.begin(), str.end());
+  return success;
+}
+
+#if defined(_WIN32)
+bool ProcessOutput::DoDecodeText(std::string raw, std::string& decoded, wchar_t *lastChar)
+{
+  bool success = false;
+  const int wlength = MultiByteToWideChar(codepage, 0, raw.c_str(), int(raw.size()), NULL, 0);
+  wchar_t* wdata = new wchar_t[wlength];
+  int r = MultiByteToWideChar(codepage, 0, raw.c_str(), int(raw.size()), wdata, wlength);
+  if (r > 0) {
+    if (lastChar) {
+      *lastChar = 0;
+      if ((wlength >= 2 && wdata[wlength - 2] != wdata[wlength - 1]) || wlength >= 1) {
+        *lastChar = wdata[wlength - 1];
+      }
+    }
+    int length = WideCharToMultiByte(defaultCodepage, 0, wdata, wlength, NULL, 0, NULL, NULL);
+    char *data = new char[length + 1];
+    r = WideCharToMultiByte(defaultCodepage, 0, wdata, wlength, data, length, NULL, NULL);
+    if (r > 0) {
+      data[length] = '\0';
+      decoded = data;
+      success = true;
+    }
+    delete[] data;
+  }
+  delete[] wdata;
+  return success;
+}
+#endif
+
diff --git a/Source/ProcessOutput.hxx b/Source/ProcessOutput.hxx
new file mode 100644
index 0000000..a988a01
--- /dev/null
+++ b/Source/ProcessOutput.hxx
@@ -0,0 +1,39 @@
+/*============================================================================
+  CMake - Cross Platform Makefile Generator
+  Copyright 2000-2016 Kitware, Inc., Insight Software Consortium
+
+  Distributed under the OSI-approved BSD License (the "License");
+  see accompanying file Copyright.txt for details.
+
+  This software is distributed WITHOUT ANY WARRANTY; without even the
+  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the License for more information.
+============================================================================*/
+#ifndef ProcessOutput_hxx
+#define ProcessOutput_hxx
+
+#include <string>
+#include <vector>
+
+class ProcessOutput
+{
+    public:
+      static unsigned int defaultCodepage;
+      // must match to KWSYSPE_PIPE_BUFFER_SIZE
+      ProcessOutput(unsigned int maxSize = 1024);
+      ~ProcessOutput();
+      bool DecodeText(std::string raw, std::string& decoded, size_t id = 0);
+      bool DecodeText(const char* data, size_t length, std::string& decoded, size_t id = 0);
+      bool DecodeText(std::vector<char> raw, std::vector<char>& decoded, size_t id = 0);
+
+    private:
+      unsigned int codepage;
+      unsigned int bufferSize;
+      std::vector<std::string> rawparts;
+#if defined(_WIN32)
+      bool DoDecodeText(std::string raw, std::string& decoded, wchar_t *lastChar);
+#endif
+};
+
+#endif
+
diff --git a/Source/cmExecProgramCommand.cxx b/Source/cmExecProgramCommand.cxx
index 58bbc31..fb2e11c 100644
--- a/Source/cmExecProgramCommand.cxx
+++ b/Source/cmExecProgramCommand.cxx
@@ -12,6 +12,7 @@
 #include "cmExecProgramCommand.h"
 
 #include "cmSystemTools.h"
+#include "ProcessOutput.hxx"
 
 #include <cmsys/Process.h>
 
@@ -219,6 +220,7 @@ bool cmExecProgramCommand::RunCommand(const char* command, std::string& output,
   int length;
   char* data;
   int p;
+  ProcessOutput processOutput;
   while ((p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) {
     if (p == cmsysProcess_Pipe_STDOUT || p == cmsysProcess_Pipe_STDERR) {
       if (verbose) {
@@ -230,6 +232,7 @@ bool cmExecProgramCommand::RunCommand(const char* command, std::string& output,
 
   // All output has been read.  Wait for the process to exit.
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
+  processOutput.DecodeText(output, output);
 
   // Check the result of running the process.
   std::string msg;
diff --git a/Source/cmExecuteProcessCommand.cxx b/Source/cmExecuteProcessCommand.cxx
index d97b25f..38f1a15 100644
--- a/Source/cmExecuteProcessCommand.cxx
+++ b/Source/cmExecuteProcessCommand.cxx
@@ -12,6 +12,7 @@
 #include "cmExecuteProcessCommand.h"
 
 #include "cmSystemTools.h"
+#include "ProcessOutput.hxx"
 
 #include <cmsys/Process.h>
 
@@ -228,17 +229,21 @@ bool cmExecuteProcessCommand::InitialPass(std::vector<std::string> const& args,
   int length;
   char* data;
   int p;
+  ProcessOutput processOutput;
+  std::string strdata;
   while ((p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) {
     // Put the output in the right place.
     if (p == cmsysProcess_Pipe_STDOUT && !output_quiet) {
       if (output_variable.empty()) {
-        cmSystemTools::Stdout(data, length);
+        processOutput.DecodeText(data, length, strdata, 1);
+        cmSystemTools::Stdout(strdata.c_str(), strdata.size());
       } else {
         cmExecuteProcessCommandAppend(tempOutput, data, length);
       }
     } else if (p == cmsysProcess_Pipe_STDERR && !error_quiet) {
       if (error_variable.empty()) {
-        cmSystemTools::Stderr(data, length);
+        processOutput.DecodeText(data, length, strdata, 2);
+        cmSystemTools::Stderr(strdata.c_str(), strdata.size());
       } else {
         cmExecuteProcessCommandAppend(tempError, data, length);
       }
@@ -247,6 +252,8 @@ bool cmExecuteProcessCommand::InitialPass(std::vector<std::string> const& args,
 
   // All output has been read.  Wait for the process to exit.
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
+  processOutput.DecodeText(tempOutput, tempOutput);
+  processOutput.DecodeText(tempError, tempError);
 
   // Fix the text in the output strings.
   cmExecuteProcessCommandFixText(tempOutput, output_strip_trailing_whitespace);
diff --git a/Source/cmProcessTools.cxx b/Source/cmProcessTools.cxx
index 34b8df2..a20a04c 100644
--- a/Source/cmProcessTools.cxx
+++ b/Source/cmProcessTools.cxx
@@ -10,6 +10,7 @@
   See the License for more information.
 ============================================================================*/
 #include "cmProcessTools.h"
+#include "ProcessOutput.hxx"
 
 #include <cmsys/Process.h>
 
@@ -20,14 +21,18 @@ void cmProcessTools::RunProcess(struct cmsysProcess_s* cp, OutputParser* out,
   char* data = CM_NULLPTR;
   int length = 0;
   int p;
+  ProcessOutput processOutput;
+  std::string strdata;
   while ((out || err) &&
          (p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) {
     if (out && p == cmsysProcess_Pipe_STDOUT) {
-      if (!out->Process(data, length)) {
+      processOutput.DecodeText(data, length, strdata, 1);
+      if (!out->Process(strdata.c_str(), int(strdata.size()))) {
         out = CM_NULLPTR;
       }
     } else if (err && p == cmsysProcess_Pipe_STDERR) {
-      if (!err->Process(data, length)) {
+      processOutput.DecodeText(data, length, strdata, 2);
+      if (!err->Process(strdata.c_str(), int(strdata.size()))) {
         err = CM_NULLPTR;
       }
     }
diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx
index 5745a01..da34d38 100644
--- a/Source/cmSystemTools.cxx
+++ b/Source/cmSystemTools.cxx
@@ -26,6 +26,7 @@
 #include <cmsys/Glob.hxx>
 #include <cmsys/RegularExpression.hxx>
 #include <cmsys/System.h>
+#include "ProcessOutput.hxx"
 #if defined(CMAKE_BUILD_WITH_CMAKE)
 #include "cmArchiveWrite.h"
 #include "cmLocale.h"
@@ -612,6 +613,8 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command,
   char* data;
   int length;
   int pipe;
+  ProcessOutput processOutput;
+  std::string strdata;
   if (outputflag != OUTPUT_PASSTHROUGH &&
       (captureStdOut || captureStdErr || outputflag != OUTPUT_NONE)) {
     while ((pipe = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) >
@@ -627,14 +630,16 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command,
 
       if (pipe == cmsysProcess_Pipe_STDOUT) {
         if (outputflag != OUTPUT_NONE) {
-          cmSystemTools::Stdout(data, length);
+          processOutput.DecodeText(data, length, strdata, 1);
+          cmSystemTools::Stdout(strdata.c_str(), strdata.size());
         }
         if (captureStdOut) {
           tempStdOut.insert(tempStdOut.end(), data, data + length);
         }
       } else if (pipe == cmsysProcess_Pipe_STDERR) {
         if (outputflag != OUTPUT_NONE) {
-          cmSystemTools::Stderr(data, length);
+          processOutput.DecodeText(data, length, strdata, 2);
+          cmSystemTools::Stderr(strdata.c_str(), strdata.size());
         }
         if (captureStdErr) {
           tempStdErr.insert(tempStdErr.end(), data, data + length);
@@ -646,9 +651,11 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command,
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
   if (captureStdOut) {
     captureStdOut->assign(tempStdOut.begin(), tempStdOut.end());
+    processOutput.DecodeText(*captureStdOut, *captureStdOut);
   }
   if (captureStdErr) {
     captureStdErr->assign(tempStdErr.begin(), tempStdErr.end());
+    processOutput.DecodeText(*captureStdErr, *captureStdErr);
   }
 
   bool result = true;
diff --git a/bootstrap b/bootstrap
index 742fa2b..ba4ddb5 100755
--- a/bootstrap
+++ b/bootstrap
@@ -328,6 +328,7 @@ CMAKE_CXX_SOURCES="\
   cmExprLexer \
   cmExprParser \
   cmExprParserHelper \
+  ProcessOutput \
 "
 
 if ${cmake_system_mingw}; then
@@ -1343,6 +1344,7 @@ fi
 cmake_c_flags_String="-DKWSYS_STRING_C"
 if ${cmake_system_mingw}; then
   cmake_c_flags_EncodingC="-DKWSYS_ENCODING_DEFAULT_CODEPAGE=CP_ACP"
+  cmake_cxx_flags_ProcessOutput=${cmake_c_flags_EncodingC}
 fi
 cmake_cxx_flags_SystemTools="
   -DKWSYS_CXX_HAS_SETENV=${KWSYS_CXX_HAS_SETENV}
@@ -1359,8 +1361,9 @@ echo "cmake: ${objs}" > "${cmake_bootstrap_dir}/Makefile"
 echo "	${cmake_cxx_compiler} ${cmake_ld_flags} ${cmake_cxx_flags} ${objs} -o cmake" >> "${cmake_bootstrap_dir}/Makefile"
 for a in ${CMAKE_CXX_SOURCES}; do
   src=`cmake_escape "${cmake_source_dir}/Source/${a}.cxx"`
+  src_flags=`eval echo \\${cmake_cxx_flags_\${a}}`
   echo "${a}.o : ${src} ${dep}" >> "${cmake_bootstrap_dir}/Makefile"
-  echo "	${cmake_cxx_compiler} ${cmake_cxx_flags} -c ${src} -o ${a}.o" >> "${cmake_bootstrap_dir}/Makefile"
+  echo "	${cmake_cxx_compiler} ${cmake_cxx_flags} ${src_flags} -c ${src} -o ${a}.o" >> "${cmake_bootstrap_dir}/Makefile"
 done
 echo "cmBootstrapCommands1.o : $cmBootstrapCommands1Deps" >> "${cmake_bootstrap_dir}/Makefile"
 echo "cmBootstrapCommands2.o : $cmBootstrapCommands2Deps" >> "${cmake_bootstrap_dir}/Makefile"
-- 
2.9.3



More information about the cmake-developers mailing list