CFLAGS += -DNOTRACK
endif
-OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o file.o utf8lib.o
+OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o file.o utf8.o
OBJ_T = test.o util.o conout.o file.o
OBJ_C = main.o lexer.o parser.o file.o
OBJ_X = exec-standalone.o util.o conout.o file.o
--- /dev/null
+/*
+ * Copyright (C) 2012
+ * Wolfgang Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is furnished to do
+ * so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "gmqcc.h"
+
+static unsigned char utf8_lengths[256] = {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, * they could be interpreted as 2-byte starts but
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, * the codepoint would be < 127
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, *
+ 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, * C0 and C1 would also result in overlong encodings
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, */
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ /* with F5 the codepoint is above 0x10FFFF,
+ * F8-FB would start 5-byte sequences
+ * FC-FD would start 6-byte sequences
+ * ...
+ */
+};
+
+static uchar_t utf8_range[5] = {
+ 1, /* invalid - let's not allow the creation of 0-bytes :P
+ 1, * ascii minimum
+ 0x80, * 2-byte minimum
+ 0x800, * 3-byte minimum
+ 0x10000, * 4-byte minimum */
+};
+
+/** Analyze the next character and return various information if requested.
+ * @param _s An utf-8 string.
+ * @param _start Filled with the start byte-offset of the next valid character
+ * @param _len Filled with the length of the next valid character
+ * @param _ch Filled with the unicode value of the next character
+ * @param _maxlen Maximum number of bytes to read from _s
+ * @return Whether or not another valid character is in the string
+ */
+bool u8_analyze(const char *_s, size_t *_start, size_t *_len, uchar_t *_ch, size_t _maxlen)
+{
+ const unsigned char *s = (const unsigned char*)_s;
+ size_t i, j;
+ size_t bits = 0;
+ uchar_t ch;
+
+ i = 0;
+/* findchar: */
+ while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0)
+ ++i;
+
+ if (i >= _maxlen || !s[i]) {
+ if (_start) *_start = i;
+ if (_len) *_len = 0;
+ return false;
+ }
+
+ if (bits == 1) { /* ascii */
+ if (_start) *_start = i;
+ if (_len) *_len = 1;
+ if (_ch) *_ch = (uchar_t)s[i];
+ return true;
+ }
+
+ ch = (s[i] & (0xFF >> bits));
+ for (j = 1; j < bits; ++j)
+ {
+ if ( (s[i+j] & 0xC0) != 0x80 )
+ {
+ i += j;
+ /* in gmqcc, invalid / overlong encodings are considered an error
+ * goto findchar;
+ */
+ if (!s[i]) goto done;
+ return false;
+ }
+ ch = (ch << 6) | (s[i+j] & 0x3F);
+ }
+ if (ch < utf8_range[bits] || ch >= 0x10FFFF)
+ {
+ /* same: error
+ * i += bits;
+ * goto findchar;
+ */
+ return false;
+ }
+
+done:
+ if (_start)
+ *_start = i;
+ if (_len)
+ *_len = bits;
+ if (_ch)
+ *_ch = ch;
+ return true;
+}
+
+/* might come in handy */
+size_t u8_strlen(const char *_s)
+{
+ size_t st, ln;
+ size_t len = 0;
+ const unsigned char *s = (const unsigned char*)_s;
+
+ while (*s)
+ {
+ /* ascii char, skip u8_analyze */
+ if (*s < 0x80)
+ {
+ ++len;
+ ++s;
+ continue;
+ }
+
+ /* invalid, skip u8_analyze */
+ if (*s < 0xC2)
+ {
+ ++s;
+ continue;
+ }
+
+ if (!u8_analyze((const char*)s, &st, &ln, NULL, 0x10))
+ break;
+ /* valid character, skip after it */
+ s += st + ln;
+ ++len;
+ }
+ return len;
+}
+
+size_t u8_strnlen(const char *_s, size_t n)
+{
+ size_t st, ln;
+ size_t len = 0;
+ const unsigned char *s = (const unsigned char*)_s;
+
+ while (*s && n)
+ {
+ /* ascii char, skip u8_analyze */
+ if (*s < 0x80)
+ {
+ ++len;
+ ++s;
+ --n;
+ continue;
+ }
+
+ /* invalid, skip u8_analyze */
+ if (*s < 0xC2)
+ {
+ ++s;
+ --n;
+ continue;
+ }
+
+ if (!u8_analyze((const char*)s, &st, &ln, NULL, n))
+ break;
+ /* valid character, see if it's still inside the range specified by n: */
+ if (n < st + ln)
+ return len;
+ ++len;
+ n -= st + ln;
+ s += st + ln;
+ }
+ return len;
+}
+
+/* Required for character constants */
+uchar_t u8_getchar(const char *_s, const char **_end)
+{
+ size_t st, ln;
+ uchar_t ch;
+
+ if (!u8_analyze(_s, &st, &ln, &ch, 0x10))
+ ch = 0;
+ if (_end)
+ *_end = _s + st + ln;
+ return ch;
+}
+
+uchar_t u8_getnchar(const char *_s, const char **_end, size_t _maxlen)
+{
+ size_t st, ln;
+ uchar_t ch;
+
+ if (!u8_analyze(_s, &st, &ln, &ch, _maxlen))
+ ch = 0;
+ if (_end)
+ *_end = _s + st + ln;
+ return ch;
+}
+
+/* required for \x{asdf}-like string escape sequences */
+int u8_fromchar(uchar_t w, char *to, size_t maxlen)
+{
+ if (maxlen < 1)
+ return 0;
+
+ if (!w)
+ return 0;
+
+/* We may want an -f flag for this behaviour...
+ if (w >= 0xE000)
+ w -= 0xE000;
+*/
+
+ if (w < 0x80)
+ {
+ to[0] = (char)w;
+ if (maxlen < 2)
+ return -1;
+ to[1] = 0;
+ return 1;
+ }
+ /* for a little speedup */
+ if (w < 0x800)
+ {
+ if (maxlen < 3)
+ {
+ to[0] = 0;
+ return -1;
+ }
+ to[2] = 0;
+ to[1] = 0x80 | (w & 0x3F); w >>= 6;
+ to[0] = 0xC0 | w;
+ return 2;
+ }
+ if (w < 0x10000)
+ {
+ if (maxlen < 4)
+ {
+ to[0] = 0;
+ return -1;
+ }
+ to[3] = 0;
+ to[2] = 0x80 | (w & 0x3F); w >>= 6;
+ to[1] = 0x80 | (w & 0x3F); w >>= 6;
+ to[0] = 0xE0 | w;
+ return 3;
+ }
+
+ /* RFC 3629 */
+ if (w <= 0x10FFFF)
+ {
+ if (maxlen < 5)
+ {
+ to[0] = 0;
+ return -1;
+ }
+ to[4] = 0;
+ to[3] = 0x80 | (w & 0x3F); w >>= 6;
+ to[2] = 0x80 | (w & 0x3F); w >>= 6;
+ to[1] = 0x80 | (w & 0x3F); w >>= 6;
+ to[0] = 0xF0 | w;
+ return 4;
+ }
+ return 0;
+}
+++ /dev/null
-/*
- * Copyright (C) 2012
- * Wolfgang Bumiller
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal in
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
- * of the Software, and to permit persons to whom the Software is furnished to do
- * so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include "gmqcc.h"
-
-static unsigned char utf8_lengths[256] = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, * they could be interpreted as 2-byte starts but
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, * the codepoint would be < 127
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, *
- 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, * C0 and C1 would also result in overlong encodings
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, */
- 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
- /* with F5 the codepoint is above 0x10FFFF,
- * F8-FB would start 5-byte sequences
- * FC-FD would start 6-byte sequences
- * ...
- */
-};
-
-static uchar_t utf8_range[5] = {
- 1, /* invalid - let's not allow the creation of 0-bytes :P
- 1, * ascii minimum
- 0x80, * 2-byte minimum
- 0x800, * 3-byte minimum
- 0x10000, * 4-byte minimum */
-};
-
-/** Analyze the next character and return various information if requested.
- * @param _s An utf-8 string.
- * @param _start Filled with the start byte-offset of the next valid character
- * @param _len Filled with the length of the next valid character
- * @param _ch Filled with the unicode value of the next character
- * @param _maxlen Maximum number of bytes to read from _s
- * @return Whether or not another valid character is in the string
- */
-bool u8_analyze(const char *_s, size_t *_start, size_t *_len, uchar_t *_ch, size_t _maxlen)
-{
- const unsigned char *s = (const unsigned char*)_s;
- size_t i, j;
- size_t bits = 0;
- uchar_t ch;
-
- i = 0;
-/* findchar: */
- while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0)
- ++i;
-
- if (i >= _maxlen || !s[i]) {
- if (_start) *_start = i;
- if (_len) *_len = 0;
- return false;
- }
-
- if (bits == 1) { /* ascii */
- if (_start) *_start = i;
- if (_len) *_len = 1;
- if (_ch) *_ch = (uchar_t)s[i];
- return true;
- }
-
- ch = (s[i] & (0xFF >> bits));
- for (j = 1; j < bits; ++j)
- {
- if ( (s[i+j] & 0xC0) != 0x80 )
- {
- i += j;
- /* in gmqcc, invalid / overlong encodings are considered an error
- * goto findchar;
- */
- if (!s[i]) goto done;
- return false;
- }
- ch = (ch << 6) | (s[i+j] & 0x3F);
- }
- if (ch < utf8_range[bits] || ch >= 0x10FFFF)
- {
- /* same: error
- * i += bits;
- * goto findchar;
- */
- return false;
- }
-
-done:
- if (_start)
- *_start = i;
- if (_len)
- *_len = bits;
- if (_ch)
- *_ch = ch;
- return true;
-}
-
-/* might come in handy */
-size_t u8_strlen(const char *_s)
-{
- size_t st, ln;
- size_t len = 0;
- const unsigned char *s = (const unsigned char*)_s;
-
- while (*s)
- {
- /* ascii char, skip u8_analyze */
- if (*s < 0x80)
- {
- ++len;
- ++s;
- continue;
- }
-
- /* invalid, skip u8_analyze */
- if (*s < 0xC2)
- {
- ++s;
- continue;
- }
-
- if (!u8_analyze((const char*)s, &st, &ln, NULL, 0x10))
- break;
- /* valid character, skip after it */
- s += st + ln;
- ++len;
- }
- return len;
-}
-
-size_t u8_strnlen(const char *_s, size_t n)
-{
- size_t st, ln;
- size_t len = 0;
- const unsigned char *s = (const unsigned char*)_s;
-
- while (*s && n)
- {
- /* ascii char, skip u8_analyze */
- if (*s < 0x80)
- {
- ++len;
- ++s;
- --n;
- continue;
- }
-
- /* invalid, skip u8_analyze */
- if (*s < 0xC2)
- {
- ++s;
- --n;
- continue;
- }
-
- if (!u8_analyze((const char*)s, &st, &ln, NULL, n))
- break;
- /* valid character, see if it's still inside the range specified by n: */
- if (n < st + ln)
- return len;
- ++len;
- n -= st + ln;
- s += st + ln;
- }
- return len;
-}
-
-/* Required for character constants */
-uchar_t u8_getchar(const char *_s, const char **_end)
-{
- size_t st, ln;
- uchar_t ch;
-
- if (!u8_analyze(_s, &st, &ln, &ch, 0x10))
- ch = 0;
- if (_end)
- *_end = _s + st + ln;
- return ch;
-}
-
-uchar_t u8_getnchar(const char *_s, const char **_end, size_t _maxlen)
-{
- size_t st, ln;
- uchar_t ch;
-
- if (!u8_analyze(_s, &st, &ln, &ch, _maxlen))
- ch = 0;
- if (_end)
- *_end = _s + st + ln;
- return ch;
-}
-
-/* required for \x{asdf}-like string escape sequences */
-int u8_fromchar(uchar_t w, char *to, size_t maxlen)
-{
- if (maxlen < 1)
- return 0;
-
- if (!w)
- return 0;
-
-/* We may want an -f flag for this behaviour...
- if (w >= 0xE000)
- w -= 0xE000;
-*/
-
- if (w < 0x80)
- {
- to[0] = (char)w;
- if (maxlen < 2)
- return -1;
- to[1] = 0;
- return 1;
- }
- /* for a little speedup */
- if (w < 0x800)
- {
- if (maxlen < 3)
- {
- to[0] = 0;
- return -1;
- }
- to[2] = 0;
- to[1] = 0x80 | (w & 0x3F); w >>= 6;
- to[0] = 0xC0 | w;
- return 2;
- }
- if (w < 0x10000)
- {
- if (maxlen < 4)
- {
- to[0] = 0;
- return -1;
- }
- to[3] = 0;
- to[2] = 0x80 | (w & 0x3F); w >>= 6;
- to[1] = 0x80 | (w & 0x3F); w >>= 6;
- to[0] = 0xE0 | w;
- return 3;
- }
-
- /* RFC 3629 */
- if (w <= 0x10FFFF)
- {
- if (maxlen < 5)
- {
- to[0] = 0;
- return -1;
- }
- to[4] = 0;
- to[3] = 0x80 | (w & 0x3F); w >>= 6;
- to[2] = 0x80 | (w & 0x3F); w >>= 6;
- to[1] = 0x80 | (w & 0x3F); w >>= 6;
- to[0] = 0xF0 | w;
- return 4;
- }
- return 0;
-}
projects / xonotic / gmqcc.git / commitdiff