-futf8 implementation

diff --git a/gmqcc.h b/gmqcc.h
index 4b423a4a35e0e3ce42381f5bbccb6cae846e2a7f..9fe1b6544786214520c61fac5b01b612250bad0b 100644 (file)
--- a/gmqcc.h
+++ b/gmqcc.h
@@ -934,6 +934,7 @@ typedef uint32_t longbit;
 /*===================================================================*/
 typedef uint32_t Uchar;
 
+bool   u8_analyze (const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen);
 size_t u8_strlen  (const char*);
 size_t u8_strnlen (const char*, size_t);
 Uchar  u8_getchar (const char*, const char**);

diff --git a/lexer.c b/lexer.c
index e506aec31ef8c105f7c3ef5de7a74eefd560d809..ee8fd4a4f82552d9520fd4d44085ac1b8d218043 100644 (file)
--- a/lexer.c
+++ b/lexer.c
@@ -742,6 +742,9 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
 {
     int ch = 0;
     int nextch;
+    bool hex;
+    char u8buf[8]; /* way more than enough */
+    int  u8len, uc;
 
     while (ch != EOF)
     {
@@ -823,13 +826,51 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
             case ']':  ch = 17; break;
             case '{':
                 ch = 0;
+                nextch = lex_getch(lex);
+                hex = (nextch == 'x');
+                if (!hex)
+                    lex_ungetch(lex, nextch);
                 for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) {
-                    ch = ch * 10 + nextch - '0';
-                    if (nextch < '0' || nextch > '9' || ch > 255) {
-                        lexerror(lex, "bad character code");
+                    if (!hex) {
+                        if (nextch >= '0' && nextch <= '9')
+                            ch = ch * 10 + nextch - '0';
+                        else {
+                            lexerror(lex, "bad character code");
+                            return (lex->tok.ttype = TOKEN_ERROR);
+                        }
+                    } else {
+                        if (nextch >= '0' || nextch <= '9')
+                            ch = ch * 16 + nextch - '0';
+                        else if (nextch >= 'a' && nextch <= 'f')
+                            ch = ch * 16 + nextch - 'a' + 10;
+                        else if (nextch >= 'A' && nextch <= 'F')
+                            ch = ch * 16 + nextch - 'A' + 10;
+                        else {
+                            lexerror(lex, "bad character code");
+                            return (lex->tok.ttype = TOKEN_ERROR);
+                        }
+                    }
+                    if ( (!OPTS_FLAG(UTF8) && ch > 255) ||
+                         ( OPTS_FLAG(UTF8) && ch > 0x10FFFF) )
+                    {
+                        lexerror(lex, "character code out of range");
                         return (lex->tok.ttype = TOKEN_ERROR);
                     }
                 }
+                if (OPTS_FLAG(UTF8) && ch >= 128) {
+                    u8len = u8_fromchar((Uchar)ch, u8buf, sizeof(u8buf));
+                    if (!u8len)
+                        ch = 0;
+                    else {
+                        --u8len;
+                        for (uc = 0; uc < u8len; ++uc)
+                            lex_tokench(lex, u8buf[uc]);
+                        /* the last character will be inserted with the tokench() call
+                         * below the switch
+                         */
+                        ch = u8buf[uc];
+                    }
+                }
                 break;
             case '\n':  ch = '\n'; break;
 
@@ -1386,10 +1427,17 @@ int lex_do(lex_file *lex)
         else
         {
             if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) {
-                if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, "multibyte character: `%s`", lex->tok.value))
-                    return (lex->tok.ttype = TOKEN_ERROR);
+                Uchar u8char;
+                /* check for a valid utf8 character */
+                if (!OPTS_FLAG(UTF8) || !u8_analyze(lex->tok.value, NULL, NULL, &u8char, 8)) {
+                    if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, "multibyte character: `%s`", lex->tok.value))
+                        return (lex->tok.ttype = TOKEN_ERROR);
+                }
+                else
+                    lex->tok.constval.i = u8char;
             }
-            lex->tok.constval.i = lex->tok.value[0];
+            else
+                lex->tok.constval.i = lex->tok.value[0];
         }
 
         return lex->tok.ttype;

diff --git a/opts.def b/opts.def
index 697afdc1a15a4a49106318f52bc25137cd6b6b5a..6fbcfaa105b13c0950d3c235a04281670f208dc3 100644 (file)
--- a/opts.def
+++ b/opts.def
@@ -43,6 +43,7 @@
     GMQCC_DEFINE_FLAG(CORRECT_LOGIC)
     GMQCC_DEFINE_FLAG(TRUE_EMPTY_STRINGS)
     GMQCC_DEFINE_FLAG(FALSE_EMPTY_STRINGS)
+    GMQCC_DEFINE_FLAG(UTF8)
 #endif
 
 /* warning flags */

diff --git a/utf8lib.c b/utf8lib.c
index b26f2f151ccb5a6c8a80430b1ff5015913fbcb30..cf722296e8d68a1835626cf075067c32dcb5582b 100644 (file)
--- a/utf8lib.c
+++ b/utf8lib.c
@@ -35,12 +35,12 @@ static Uchar utf8_range[5] = {
 /** Analyze the next character and return various information if requested.
  * @param _s      An utf-8 string.
  * @param _start  Filled with the start byte-offset of the next valid character
- * @param _len    Fileed with the length of the next valid character
+ * @param _len    Filled with the length of the next valid character
  * @param _ch     Filled with the unicode value of the next character
  * @param _maxlen Maximum number of bytes to read from _s
  * @return        Whether or not another valid character is in the string
  */
-static bool u8_analyze(const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen)
+bool u8_analyze(const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen)
 {
        const unsigned char *s = (const unsigned char*)_s;
        size_t i, j;