Add token.c, another layer to the tokenizer. Use char instead of unsigned char for...

diff --git a/darkplaces-sdl2-vs2017.vcxproj b/darkplaces-sdl2-vs2017.vcxproj
index b0d6df69566428999fe4a0b1bac318e3634e7861..f474540ca158d8688b072ddd12feeb836066b1c2 100644 (file)
--- a/darkplaces-sdl2-vs2017.vcxproj
+++ b/darkplaces-sdl2-vs2017.vcxproj
@@ -306,6 +306,7 @@
     <ClCompile Include="sys_shared.c" />\r
     <ClCompile Include="taskqueue.c" />\r
     <ClCompile Include="thread_sdl.c" />\r
+    <ClCompile Include="token.c" />\r
     <ClCompile Include="utf8lib.c" />\r
     <ClCompile Include="vid_sdl.c" />\r
     <ClCompile Include="vid_shared.c" />\r
@@ -423,6 +424,7 @@
     <ClInclude Include="taskqueue.h" />\r
     <ClInclude Include="thread.h" />\r
     <ClInclude Include="timing.h" />\r
+    <ClInclude Include="token.h" />\r
     <ClInclude Include="utf8lib.h" />\r
     <ClInclude Include="vid.h" />\r
     <ClInclude Include="view.h" />\r

diff --git a/darkplaces-sdl2-vs2019.vcxproj b/darkplaces-sdl2-vs2019.vcxproj
index d7e975d97649919eae7b97c584144d27412f647d..d1b092294c5be8edf235e91d54313d7b862db669 100644 (file)
--- a/darkplaces-sdl2-vs2019.vcxproj
+++ b/darkplaces-sdl2-vs2019.vcxproj
@@ -307,6 +307,7 @@
     <ClCompile Include="sys_shared.c" />\r
     <ClCompile Include="taskqueue.c" />\r
     <ClCompile Include="thread_sdl.c" />\r
+    <ClCompile Include="token.c" />\r
     <ClCompile Include="utf8lib.c" />\r
     <ClCompile Include="vid_sdl.c" />\r
     <ClCompile Include="vid_shared.c" />\r
@@ -425,6 +426,7 @@
     <ClInclude Include="taskqueue.h" />\r
     <ClInclude Include="thread.h" />\r
     <ClInclude Include="timing.h" />\r
+    <ClInclude Include="token.h" />\r
     <ClInclude Include="utf8lib.h" />\r
     <ClInclude Include="vid.h" />\r
     <ClInclude Include="view.h" />\r

diff --git a/json.c b/json.c
index 9e827b88ccd5d761fce9d439f857db751583aacf..8dcbdc471ee9452617201ee80337d1652676749e 100644 (file)
--- a/json.c
+++ b/json.c
@@ -131,7 +131,7 @@ static inline void Json_Parse_String(struct qjson_state_s *json)
 static inline qbool Json_Parse_Number(struct qjson_state_s *json)
 {
        int i, numsize;
-       const unsigned char *in = json->state->pos;
+       const char *in = json->state->pos;
        //char out[128];
        qbool is_float = false;
        qbool is_exp = false;
@@ -182,7 +182,7 @@ static inline qbool Json_Parse_Keyword(struct qjson_state_s *json)
        {
                keyword_size = strlen(keyword_list[i]);
 
-               if(!strncmp(keyword_list[i], (const char *)json->state->pos, keyword_size))
+               if(!strncmp(keyword_list[i], json->state->pos, keyword_size))
                {
                        // Don't advance the entire length of the keyword or we might run into a valid token that'd go missed.
                        Parse_Next(json->state, keyword_size - 1);

diff --git a/makefile.inc b/makefile.inc
index 5967a45e1c308a04f65189e265dd1d9ea33c0707..97f7cbc32daec93db739e308cf108db0575fa7d9 100644 (file)
--- a/makefile.inc
+++ b/makefile.inc
@@ -171,6 +171,7 @@ OBJ_COMMON= \
        svvm_cmds.o \
        sys_shared.o \
        taskqueue.o \
+       token.o \
        vid_shared.o \
        view.o \
        wad.o \

diff --git a/parser.c b/parser.c
index c9e5e84f335ce82558d7a96e4495280a24d1f7c4..b5621e9878428fdad21079a99e94674a9cd43930 100644 (file)
--- a/parser.c
+++ b/parser.c
@@ -19,6 +19,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
 
 #include "darkplaces.h"
+#include "token.h"
 #include "parser.h"
 
 jmp_buf parse_error;
@@ -49,12 +50,16 @@ DP_FUNC_NORETURN void Parse_Error(struct qparser_state_s *state, qparser_err_t e
 }
 
 // Advance forward in the stream as many times as 'count', cleanly.
-void Parse_Next(struct qparser_state_s *state, size_t count)
+void Parse_Next(struct qparser_state_s *state, int count)
 {
-       state->col += count;
-       state->pos += count;
+       const char *next = Token_Next(state->pos, count);
 
-       if(!*state->pos)
+       if(next)
+       {
+               state->pos = next;
+               state->col += count;
+       }
+       else
                Parse_Error(state, PARSE_ERR_EOF, NULL);
 }
 
@@ -155,7 +160,7 @@ qparser_state_t *Parse_New(const unsigned char *in)
 
        out = (qparser_state_t *)Z_Malloc(sizeof(qparser_state_t));
 
-       out->buf = in;
+       out->buf = (const char *)in;
        out->pos = NULL;
        out->line = 1;
        out->col = 1;

diff --git a/parser.h b/parser.h
index eee99bb94083fb082a4be1634774fb93d3a691a3..57fdcae784a56b2324b93b87acb78d4528404c63 100644 (file)
--- a/parser.h
+++ b/parser.h
@@ -35,8 +35,8 @@ typedef enum qparser_err_e
 typedef struct qparser_state_s
 {
        const char *name;
-       const unsigned char *buf;
-       const unsigned char *pos;
+       const char *buf;
+       const char *pos;
        int line, col, depth;
 
        struct
@@ -50,7 +50,7 @@ typedef struct qparser_state_s
 extern jmp_buf parse_error;
 
 void Parse_Error(struct qparser_state_s *state, qparser_err_t error, const char *expected);
-void Parse_Next(struct qparser_state_s *state, size_t count);
+void Parse_Next(struct qparser_state_s *state, int count);
 char Parse_NextToken(struct qparser_state_s *state);
 qparser_state_t *Parse_New(const unsigned char *in);
 qparser_state_t *Parse_LoadFile(const char *file);

diff --git a/token.c b/token.c
new file mode 100644 (file)
index 0000000..22f66be
--- /dev/null
+++ b/token.c
@@ -0,0 +1,37 @@
+/*
+Copyright (C) 2000-2021 Ashley Rose Hale (LadyHavoc)
+Copyright (C) 2021 David Knapp (Cloudwalk)
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+*/
+
+#include "darkplaces.h"
+#include "token.h"
+
+// Advance forward in the stream as many times as 'count', cleanly.
+const char *Token_Next(const char *data, int count)
+{
+       const char *out = data;
+       for (int i = 0; i < count; i++)
+       {
+               if (!*out)
+                       break;
+               out++;
+       }
+
+       return out;
+}

diff --git a/token.h b/token.h
new file mode 100644 (file)
index 0000000..5d6c970
--- /dev/null
+++ b/token.h
@@ -0,0 +1,24 @@
+/*
+Copyright (C) 2021 David Knapp (Cloudwalk)
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+*/
+
+#include "qtypes.h"
+#include "qdefs.h"
+
+const char *Token_Next(const char *data, int count);