Wallet2 + CLI wallet: UTF-8 support for filenames and paths under Windows

CMakeLists.txt

@@ -774,7 +774,7 @@ if(STATIC)
   set(Boost_USE_STATIC_LIBS ON)
   set(Boost_USE_STATIC_RUNTIME ON)
 endif()
-find_package(Boost 1.58 QUIET REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options)
+find_package(Boost 1.58 QUIET REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options locale)
 
 set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES})
 if(NOT Boost_FOUND)
@@ -791,6 +791,7 @@ endif()
 include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
 if(MINGW)
   set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi)
+  set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} sicuio sicuin sicuuc sicudt sicutu iconv)
 elseif(APPLE OR OPENBSD OR ANDROID)
   set(EXTRA_LIBRARIES "")
 elseif(FREEBSD)

contrib/epee/include/file_io_utils.h

@@ -31,6 +31,31 @@
 #include <iostream>
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/operations.hpp>
+#ifdef WIN32
+#include <windows.h>
+#endif
+
+// On Windows there is a problem with non-ASCII characters in path and file names
+// as far as support by the standard components used is concerned:
+
+// The various file stream classes, e.g. std::ifstream and std::ofstream, are
+// part of the GNU C++ Library / libstdc++. On the most basic level they use the
+// fopen() call as defined / made accessible to programs compiled within MSYS2
+// by the stdio.h header file maintained by the MinGW project.
+
+// The critical point: The implementation of fopen() is part of MSVCRT, the
+// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any
+// Unicode support.
+
+// Monero code that would want to continue to use the normal file stream classes
+// but WITH Unicode support could therefore not solve this problem on its own,
+// but 2 different projects from 2 different maintaining groups would need changes
+// in this particular direction - something probably difficult to achieve and
+// with a long time to wait until all new versions / releases arrive.
+
+// Implemented solution approach: Circumvent the problem by stopping to use std
+// file stream classes on Windows and directly use Unicode-capable WIN32 API
+// calls. Most of the code doing so is concentrated in this header file here.
 
 namespace epee
 {
@@ -46,7 +71,22 @@ namespace file_io_utils
 	inline
 		bool save_string_to_file(const std::string& path_to_file, const std::string& str)
 	{
-
+#ifdef WIN32
+                WCHAR wide_path[1000];
+                int chars = MultiByteToWideChar(CP_UTF8, 0, path_to_file.c_str(), path_to_file.size() + 1, wide_path, 1000);
+                if (chars == 0)
+                    return false;
+                HANDLE file_handle = CreateFileW(wide_path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+                if (file_handle == INVALID_HANDLE_VALUE)
+                    return false;
+                DWORD bytes_written;
+                DWORD bytes_to_write = (DWORD)str.size();
+                BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL);
+                CloseHandle(file_handle);
+                if (bytes_written != bytes_to_write)
+                    result = FALSE;
+                return result;
+#else
 		try
 		{
 			std::ofstream fstream;
@@ -61,6 +101,7 @@ namespace file_io_utils
 		{
 			return false;
 		}
+#endif
 	}
 
 	inline
@@ -89,6 +130,27 @@ namespace file_io_utils
 	inline
 		bool load_file_to_string(const std::string& path_to_file, std::string& target_str)
 	{
+#ifdef WIN32
+                WCHAR wide_path[1000];
+                int chars = MultiByteToWideChar(CP_UTF8, 0, path_to_file.c_str(), path_to_file.size() + 1, wide_path, 1000);
+                if (chars == 0)
+                    return false;
+                HANDLE file_handle = CreateFileW(wide_path, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+                if (file_handle == INVALID_HANDLE_VALUE)
+                    return false;
+                DWORD file_size = GetFileSize(file_handle, NULL);
+                if ((file_size == INVALID_FILE_SIZE) || (file_size > 1000000000)) {
+                    CloseHandle(file_handle);
+                    return false;
+                }
+                target_str.resize(file_size);
+                DWORD bytes_read;
+                BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL);
+                CloseHandle(file_handle);
+                if (bytes_read != file_size)
+                    result = FALSE;
+                return result;
+#else
 		try
 		{
 			std::ifstream fstream;
@@ -113,11 +175,13 @@ namespace file_io_utils
 		{
 			return false;
 		}
+#endif
 	}
 
 	inline
 		bool append_string_to_file(const std::string& path_to_file, const std::string& str)
 	{
+                // No special Windows implementation because so far not used in Monero code
 		try
 		{
 			std::ofstream fstream;
@@ -137,6 +201,22 @@ namespace file_io_utils
 	inline
 		bool get_file_size(const std::string& path_to_file, uint64_t &size)
 	{
+#ifdef WIN32
+                WCHAR wide_path[1000];
+                int chars = MultiByteToWideChar(CP_UTF8, 0, path_to_file.c_str(), path_to_file.size() + 1, wide_path, 1000);
+                if (chars == 0)
+                    return false;
+                HANDLE file_handle = CreateFileW(wide_path, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+                if (file_handle == INVALID_HANDLE_VALUE)
+                    return false;
+                LARGE_INTEGER file_size;
+                BOOL result = GetFileSizeEx(file_handle, &file_size);
+                CloseHandle(file_handle);
+                if (result) {
+                    size = file_size.QuadPart;
+                }
+                return size;
+#else
 		try
 		{
 			std::ifstream fstream;
@@ -151,6 +231,7 @@ namespace file_io_utils
 		{
 			return false;
 		}
+#endif
 	}
 
 }

src/common/util.cpp

@@ -435,15 +435,17 @@ std::string get_nix_version_display_string()
 #ifdef WIN32
   std::string get_special_folder_path(int nfolder, bool iscreate)
   {
-    namespace fs = boost::filesystem;
+    WCHAR psz_path[MAX_PATH] = L"";
-    char psz_path[MAX_PATH] = "";
 
-    if(SHGetSpecialFolderPathA(NULL, psz_path, nfolder, iscreate))
+    if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate))
     {
-      return psz_path;
+      int size_needed = WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), NULL, 0, NULL, NULL);
+      std::string folder_name(size_needed, 0);
+      WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), &folder_name[0], size_needed, NULL, NULL);
+      return folder_name;
     }
 
-    LOG_ERROR("SHGetSpecialFolderPathA() failed, could not obtain requested path.");
+    LOG_ERROR("SHGetSpecialFolderPathW() failed, could not obtain requested path.");
     return "";
   }
 #endif
@@ -501,13 +503,18 @@ std::string get_nix_version_display_string()
     int code;
 #if defined(WIN32)
     // Maximizing chances for success
-    DWORD attributes = ::GetFileAttributes(replaced_name.c_str());
+    WCHAR wide_replacement_name[1000];
+    MultiByteToWideChar(CP_UTF8, 0, replacement_name.c_str(), replacement_name.size() + 1, wide_replacement_name, 1000);
+    WCHAR wide_replaced_name[1000];
+    MultiByteToWideChar(CP_UTF8, 0, replaced_name.c_str(), replaced_name.size() + 1, wide_replaced_name, 1000);
+
+    DWORD attributes = ::GetFileAttributesW(wide_replaced_name);
     if (INVALID_FILE_ATTRIBUTES != attributes)
     {
-      ::SetFileAttributes(replaced_name.c_str(), attributes & (~FILE_ATTRIBUTE_READONLY));
+      ::SetFileAttributesW(wide_replaced_name, attributes & (~FILE_ATTRIBUTE_READONLY));
     }
 
-    bool ok = 0 != ::MoveFileEx(replacement_name.c_str(), replaced_name.c_str(), MOVEFILE_REPLACE_EXISTING);
+    bool ok = 0 != ::MoveFileExW(wide_replacement_name, wide_replaced_name, MOVEFILE_REPLACE_EXISTING);
     code = ok ? 0 : static_cast<int>(::GetLastError());
 #else
     bool ok = 0 == std::rename(replacement_name.c_str(), replaced_name.c_str());

src/simplewallet/CMakeLists.txt

@@ -54,6 +54,7 @@ target_link_libraries(simplewallet
     ${Boost_CHRONO_LIBRARY}
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}
+    ${ICU_LIBRARIES}
     ${Boost_THREAD_LIBRARY}
     ${CMAKE_THREAD_LIBS_INIT}
     ${GNU_READLINE_LIBRARY}

src/simplewallet/simplewallet.cpp

@@ -64,6 +64,11 @@
 #include "wallet/wallet_args.h"
 #include <stdexcept>
 
+#ifdef WIN32
+#include <boost/locale.hpp>
+#include <boost/filesystem.hpp>
+#endif
+
 #ifdef HAVE_READLINE
 #include "readline_buffer.h"
 #endif
@@ -130,6 +135,28 @@ namespace
 
   const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""};
 
+#ifdef WIN32
+  // Translate from CP850 to UTF-8;
+  // std::getline for a Windows console returns a string in CP437 or CP850; as simplewallet,
+  // like all of Monero, is assumed to work internally with UTF-8 throughout, even on Windows
+  // (although only implemented partially), a translation to UTF-8 is needed for input.
+  //
+  // Note that if a program is started inside the MSYS2 shell somebody already translates
+  // console input to UTF-8, but it's not clear how one could detect that in order to avoid
+  // double-translation; this code here thus breaks UTF-8 input within a MSYS2 shell,
+  // unfortunately.
+  //
+  // Note also that input for passwords is NOT translated, to remain compatible with any
+  // passwords containing special characters that predate this switch to UTF-8 support.
+  static std::string cp850_to_utf8(const std::string &cp850_str)
+  {
+    boost::locale::generator gen;
+    gen.locale_cache_enabled(true);
+    std::locale loc = gen("en_US.CP850");
+    return boost::locale::conv::to_utf<char>(cp850_str, loc);
+  }
+#endif
+
   std::string input_line(const std::string& prompt)
   {
 #ifdef HAVE_READLINE
@@ -139,6 +166,9 @@ namespace
 
     std::string buf;
     std::getline(std::cin, buf);
+#ifdef WIN32
+    buf = cp850_to_utf8(buf);
+#endif
 
     return epee::string_tools::trim(buf);
   }
@@ -6751,6 +6781,12 @@ void simple_wallet::commit_or_save(std::vector<tools::wallet2::pending_tx>& ptx_
 //----------------------------------------------------------------------------------------------------
 int main(int argc, char* argv[])
 {
+#ifdef WIN32
+  // Activate UTF-8 support for Boost filesystem classes on Windows
+  std::locale::global(boost::locale::generator().generate(""));
+  boost::filesystem::path::imbue(std::locale());
+#endif
+
   po::options_description desc_params(wallet_args::tr("Wallet options"));
   tools::wallet2::init_options(desc_params);
   command_line::add_arg(desc_params, arg_wallet_file);

src/wallet/api/wallet.cpp

@@ -44,6 +44,11 @@
 #include <sstream>
 #include <unordered_map>
 
+#ifdef WIN32
+#include <boost/locale.hpp>
+#include <boost/filesystem.hpp>
+#endif
+
 using namespace std;
 using namespace cryptonote;
 
@@ -291,6 +296,11 @@ uint64_t Wallet::maximumAllowedAmount()
 }
 
 void Wallet::init(const char *argv0, const char *default_log_base_name) {
+#ifdef WIN32
+    // Activate UTF-8 support for Boost filesystem classes on Windows
+    std::locale::global(boost::locale::generator().generate(""));
+    boost::filesystem::path::imbue(std::locale());
+#endif
     epee::string_tools::set_module_name_and_folder(argv0);
     mlog_configure(mlog_get_default_log_path(default_log_base_name), true);
 }

src/wallet/wallet2.cpp

@@ -3737,12 +3737,24 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas
     }
   } else {
     // save to new file
+#ifdef WIN32
+    // On Windows avoid using std::ofstream which does not work with UTF-8 filenames
+    // The price to pay is temporary higher memory consumption for string stream + binary archive
+    std::ostringstream oss;
+    binary_archive<true> oar(oss);
+    bool success = ::serialization::serialize(oar, cache_file_data);
+    if (success) {
+        success = epee::file_io_utils::save_string_to_file(new_file, oss.str());
+    }
+    THROW_WALLET_EXCEPTION_IF(!success, error::file_save_error, new_file);
+#else
     std::ofstream ostr;
     ostr.open(new_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
     binary_archive<true> oar(ostr);
     bool success = ::serialization::serialize(oar, cache_file_data);
     ostr.close();
     THROW_WALLET_EXCEPTION_IF(!success || !ostr.good(), error::file_save_error, new_file);
+#endif
 
     // here we have "*.new" file, we need to rename it to be without ".new"
     std::error_code e = tools::replace_file(new_file, m_wallet_file);