Fix inability to access non-ascii path under Windows with WTF-8

Signed-off-by: NaitLee <naitli@foxmail.com>

diff --git a/filematch.c b/filematch.c
index e06439614c2174927168128adf5b1fcbe8fa268d..d9c73bbd0a0a34bd0044ca1b82d2eefedad20e67 100644 (file)
--- a/filematch.c
+++ b/filematch.c
@@ -7,6 +7,10 @@
 
 #include "darkplaces.h"
 
+#ifdef WIN32
+#include "utf8lib.h"
+#endif
+
 // LadyHavoc: some portable directory listing code I wrote for lmp2pcx, now used in darkplaces to load id1/*.pak and such...
 
 int matchpattern(const char *in, const char *pattern, int caseinsensitive)
@@ -164,20 +168,31 @@ static void adddirentry(stringlist_t *list, const char *path, const char *name)
 #ifdef WIN32
 void listdirectory(stringlist_t *list, const char *basepath, const char *path)
 {
-       char pattern[4096];
-       WIN32_FIND_DATA n_file;
+       #define BUFSIZE 4096
+       char pattern[BUFSIZE] = {0};
+       wchar patternw[BUFSIZE] = {0};
+       char filename[BUFSIZE] = {0};
+       wchar *filenamew;
+       int lenw = 0;
+       WIN32_FIND_DATAW n_file;
        HANDLE hFile;
        strlcpy (pattern, basepath, sizeof(pattern));
        strlcat (pattern, path, sizeof (pattern));
        strlcat (pattern, "*", sizeof (pattern));
+       fromwtf8(pattern, strlen(pattern), patternw, BUFSIZE);
        // ask for the directory listing handle
-       hFile = FindFirstFile(pattern, &n_file);
+       hFile = FindFirstFileW(patternw, &n_file);
        if(hFile == INVALID_HANDLE_VALUE)
                return;
        do {
-               adddirentry(list, path, n_file.cFileName);
-       } while (FindNextFile(hFile, &n_file) != 0);
+               filenamew = n_file.cFileName;
+               lenw = 0;
+               while(filenamew[lenw] != 0) ++lenw;
+               towtf8(filenamew, lenw, filename, BUFSIZE);
+               adddirentry(list, path, filename);
+       } while (FindNextFileW(hFile, &n_file) != 0);
        FindClose(hFile);
+       #undef BUFSIZE
 }
 #else
 void listdirectory(stringlist_t *list, const char *basepath, const char *path)

diff --git a/fs.c b/fs.c
index 25789e0235888c1f58d199fd65c30184e6711e70..a1a760632da8fc44769760d965428a82f94d2a14 100644 (file)
--- a/fs.c
+++ b/fs.c
@@ -49,6 +49,10 @@
 #include "fs.h"
 #include "wad.h"
 
+#ifdef WIN32
+#include "utf8lib.h"
+#endif
+
 // Win32 requires us to add O_BINARY, but the other OSes don't have it
 #ifndef O_BINARY
 # define O_BINARY 0
@@ -441,7 +445,7 @@ static dllhandle_t zlib_dll = NULL;
 static HRESULT (WINAPI *qSHGetFolderPath) (HWND hwndOwner, int nFolder, HANDLE hToken, DWORD dwFlags, LPTSTR pszPath);
 static dllfunction_t shfolderfuncs[] =
 {
-       {"SHGetFolderPathA", (void **) &qSHGetFolderPath},
+       {"SHGetFolderPathW", (void **) &qSHGetFolderPath},
        {NULL, NULL}
 };
 static const char* shfolderdllnames [] =
@@ -926,14 +930,30 @@ static packfile_t* FS_AddFileToPack (const char* name, pack_t* pack,
        return pfile;
 }
 
+#if WIN32
+#define WSTRBUF 4096
+static inline int wstrlen(wchar *wstr)
+{
+       int len = 0;
+       while (wstr[len] != 0 && len < WSTRBUF)
+               ++len;
+       return len;
+}
+#define widen(str, wstr) fromwtf8(str, strlen(str), wstr, WSTRBUF)
+#define narrow(wstr, str) towtf8(wstr, wstrlen(wstr), str, WSTRBUF)
+#endif
 
 static void FS_mkdir (const char *path)
 {
+#if WIN32
+       wchar pathw[WSTRBUF] = {0};
+#endif
        if(Sys_CheckParm("-readonly"))
                return;
 
 #if WIN32
-       if (_mkdir (path) == -1)
+       widen(path, pathw);
+       if (_wmkdir (pathw) == -1)
 #else
        if (mkdir (path, 0777) == -1)
 #endif
@@ -945,7 +965,6 @@ static void FS_mkdir (const char *path)
        }
 }
 
-
 /*
 ============
 FS_CreatePath
@@ -1849,13 +1868,14 @@ static int FS_ChooseUserDir(userdirmode_t userdirmode, char *userdir, size_t use
        return -1;
 
 #elif defined(WIN32)
-       char *homedir;
+       char homedir[WSTRBUF];
+       wchar *homedirw;
 #if _MSC_VER >= 1400
-       size_t homedirlen;
+       size_t homedirwlen;
 #endif
        TCHAR mydocsdir[MAX_PATH + 1];
        wchar_t *savedgamesdirw;
-       char savedgamesdir[MAX_OSPATH];
+       char savedgamesdir[WSTRBUF] = {0};
        int fd;
        char vabuf[1024];
 
@@ -1877,16 +1897,18 @@ static int FS_ChooseUserDir(userdirmode_t userdirmode, char *userdir, size_t use
                        break;
                }
 #if _MSC_VER >= 1400
-               _dupenv_s(&homedir, &homedirlen, "USERPROFILE");
-               if(homedir)
+               _wdupenv_s(&homedirw, &homedirwlen, L"USERPROFILE");
+               narrow(homedirw, homedir);
+               if(homedir[0])
                {
                        dpsnprintf(userdir, userdirsize, "%s/.%s/", homedir, gameuserdirname);
-                       free(homedir);
+                       free(homedirw);
                        break;
                }
 #else
-               homedir = getenv("USERPROFILE");
-               if(homedir)
+               homedirw = _wgetenv(L"USERPROFILE");
+               narrow(homedirw, homedir);
+               if(homedir[0])
                {
                        dpsnprintf(userdir, userdirsize, "%s/.%s/", homedir, gameuserdirname);
                        break;
@@ -1911,12 +1933,7 @@ static int FS_ChooseUserDir(userdirmode_t userdirmode, char *userdir, size_t use
 */
                        if (qSHGetKnownFolderPath(&qFOLDERID_SavedGames, qKF_FLAG_CREATE | qKF_FLAG_NO_ALIAS, NULL, &savedgamesdirw) == S_OK)
                        {
-                               memset(savedgamesdir, 0, sizeof(savedgamesdir));
-#if _MSC_VER >= 1400
-                               wcstombs_s(NULL, savedgamesdir, sizeof(savedgamesdir), savedgamesdirw, sizeof(savedgamesdir)-1);
-#else
-                               wcstombs(savedgamesdir, savedgamesdirw, sizeof(savedgamesdir)-1);
-#endif
+                               narrow(savedgamesdirw, savedgamesdir);
                                qCoTaskMemFree(savedgamesdirw);
                        }
                        qCoUninitialize();
@@ -2279,6 +2296,9 @@ static filedesc_t FS_SysOpenFiledesc(const char *filepath, const char *mode, qbo
        int mod, opt;
        unsigned int ind;
        qbool dolock = false;
+       #ifdef WIN32
+       wchar filepathw[WSTRBUF] = {0};
+       #endif
 
        // Parse the mode string
        switch (mode[0])
@@ -2330,10 +2350,11 @@ static filedesc_t FS_SysOpenFiledesc(const char *filepath, const char *mode, qbo
        handle = SDL_RWFromFile(filepath, mode);
 #else
 # ifdef WIN32
+       widen(filepath, filepathw);
 #  if _MSC_VER >= 1400
-       _sopen_s(&handle, filepath, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
+       _wsopen_s(&handle, filepathw, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
 #  else
-       handle = _sopen (filepath, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
+       handle = _wsopen (filepathw, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
 #  endif
 # else
        handle = open (filepath, mod | opt, 0666);
@@ -3592,8 +3613,10 @@ int FS_SysFileType (const char *path)
 # ifndef INVALID_FILE_ATTRIBUTES
 #  define INVALID_FILE_ATTRIBUTES ((DWORD)-1)
 # endif
-
-       DWORD result = GetFileAttributes(path);
+       wchar pathw[WSTRBUF] = {0};
+       DWORD result;
+       widen(path, pathw);
+       result = GetFileAttributesW(pathw);
 
        if(result == INVALID_FILE_ATTRIBUTES)
                return FS_FILETYPE_NONE;

diff --git a/netconn.c b/netconn.c
old mode 100755 (executable)
new mode 100644 (file)

diff --git a/utf8lib.c b/utf8lib.c
index 155b4aea9a8f949ecf8e2a583487c4fa122896cc..a15c25f6558384a538e4aa138661e8eeca4b0d2d 100644 (file)
--- a/utf8lib.c
+++ b/utf8lib.c
@@ -907,6 +907,112 @@ size_t u8_strpad_colorcodes(char *out, size_t outsize, const char *in, qbool lef
        return dpsnprintf(out, outsize, "%*s%.*s%*s", lpad, "", prec, in, rpad, "");
 }
 
+#ifdef WIN32
+
+/**
+ * Convert Windows "wide" characters to WTF-8 for internal manipulation
+ */
+int towtf8(const wchar *wstr, int wlen, char *cstr, int maxclen)
+{
+    int p = 0;
+       int i;
+    for (i = 0; i < wlen; ++i)
+       {
+        wchar point = wstr[i];
+        if (point < 0x80)
+               {
+            if (p + 1 >= maxclen) break;
+            cstr[p++] = point;
+        }
+               else if (point < 0x800)
+               {
+            if (p + 2 >= maxclen) break;
+            cstr[p++] = (0xc0 | ((point >>  6) & 0x1f));
+            cstr[p++] = (0x80 | ((point >>  0) & 0x3f));
+        }
+               else
+               #if U32
+               if (point < 0x10000)
+               #endif
+        {
+            if (p + 3 >= maxclen) break;
+            cstr[p++] = (0xe0 | ((point >> 12) & 0x0f));
+            cstr[p++] = (0x80 | ((point >>  6) & 0x3f));
+            cstr[p++] = (0x80 | ((point >>  0) & 0x3f));
+        }
+        #if U32
+        else
+        #if CHECKS
+        if (point < 0x110000)
+        #endif
+        {
+            if (p + 4 >= maxclen) break;
+            cstr[p++] = (0xf0 | ((point >> 18) & 0x07));
+            cstr[p++] = (0x80 | ((point >> 12) & 0x3f));
+            cstr[p++] = (0x80 | ((point >>  6) & 0x3f));
+            cstr[p++] = (0x80 | ((point >>  0) & 0x3f));
+        }
+        #endif
+    }
+    cstr[p] = 0x00;
+    return p;
+}
+
+/**
+ * Convert WTF-8 string to "wide" characters used by Windows
+ */
+int fromwtf8(const char *cstr, int clen, wchar *wstr, int maxwlen)
+{
+    int p = 0;
+       int i;
+    for (i = 0; i < clen;)
+       {
+        char byte = cstr[i++];
+        wchar point = byte;
+        int length = 1;
+        if (p + 1 >= maxwlen) break;
+        #if CHECKS
+        if ((byte & 0xf8) == 0xf8)
+            return -1;
+        #endif
+        if ((byte & 0xf8) == 0xf0)
+               {
+            length = 4;
+            point = byte & 0x07;
+        }
+               else if ((byte & 0xf0) == 0xe0)
+               {
+            length = 3;
+            point = byte & 0x0f;
+        }
+               else if ((byte & 0xe0) == 0xc0)
+               {
+            length = 2;
+            point = byte & 0x1f;
+        }
+        #if CHECKS
+        else if ((byte & 0xc0) == 0x80)
+               {
+            return -1;
+        }
+        #endif
+        while (--length)
+               {
+            byte = cstr[i++];
+            #if CHECKS
+            if (byte == -1) return -1;
+            else if ((byte & 0xc0) != 0x80) return -1;
+            #endif
+            point = (point << 6) | (byte & 0x3f);
+        }
+        wstr[p++] = point;
+    }
+    wstr[p] = 0x00;
+    return p;
+}
+
+#endif // WIN32
+
 
 /*
 The two following functions (u8_toupper, u8_tolower) are derived from

diff --git a/utf8lib.h b/utf8lib.h
index 4b8221f5f779da6d7ca6d9103c6a69173acb08fa..cb7041b961108b877c8a2206237470b9bae18d21 100644 (file)
--- a/utf8lib.h
+++ b/utf8lib.h
@@ -78,4 +78,17 @@ extern Uchar u8_quake2utf8map[256];
 Uchar u8_toupper(Uchar ch);
 Uchar u8_tolower(Uchar ch);
 
+#ifdef WIN32
+
+// WTF-8 encoding to circumvent Windows encodings, be it UTF-16 or random codepages
+// https://simonsapin.github.io/wtf-8/
+#define U32 0
+#define CHECKS 1
+typedef wchar_t wchar;
+
+int towtf8(const wchar* wstr, int wlen, char* cstr, int maxclen);
+int fromwtf8(const char* cstr, int clen, wchar* wstr, int maxwlen);
+
+#endif // WIN32
+
 #endif // UTF8LIB_H__