From 3b626119aa539133165dbd6391e07e4809d7c588 Mon Sep 17 00:00:00 2001 From: Cloudwalk Date: Mon, 10 May 2021 22:50:18 -0400 Subject: [PATCH] Add token.c, another layer to the tokenizer. Use char instead of unsigned char for the buffers. Use int instead of size_t for a parameter --- darkplaces-sdl2-vs2017.vcxproj | 2 ++ darkplaces-sdl2-vs2019.vcxproj | 2 ++ json.c | 4 ++-- makefile.inc | 1 + parser.c | 15 +++++++++----- parser.h | 6 +++--- token.c | 37 ++++++++++++++++++++++++++++++++++ token.h | 24 ++++++++++++++++++++++ 8 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 token.c create mode 100644 token.h diff --git a/darkplaces-sdl2-vs2017.vcxproj b/darkplaces-sdl2-vs2017.vcxproj index b0d6df69..f474540c 100644 --- a/darkplaces-sdl2-vs2017.vcxproj +++ b/darkplaces-sdl2-vs2017.vcxproj @@ -306,6 +306,7 @@ + @@ -423,6 +424,7 @@ + diff --git a/darkplaces-sdl2-vs2019.vcxproj b/darkplaces-sdl2-vs2019.vcxproj index d7e975d9..d1b09229 100644 --- a/darkplaces-sdl2-vs2019.vcxproj +++ b/darkplaces-sdl2-vs2019.vcxproj @@ -307,6 +307,7 @@ + @@ -425,6 +426,7 @@ + diff --git a/json.c b/json.c index 9e827b88..8dcbdc47 100644 --- a/json.c +++ b/json.c @@ -131,7 +131,7 @@ static inline void Json_Parse_String(struct qjson_state_s *json) static inline qbool Json_Parse_Number(struct qjson_state_s *json) { int i, numsize; - const unsigned char *in = json->state->pos; + const char *in = json->state->pos; //char out[128]; qbool is_float = false; qbool is_exp = false; @@ -182,7 +182,7 @@ static inline qbool Json_Parse_Keyword(struct qjson_state_s *json) { keyword_size = strlen(keyword_list[i]); - if(!strncmp(keyword_list[i], (const char *)json->state->pos, keyword_size)) + if(!strncmp(keyword_list[i], json->state->pos, keyword_size)) { // Don't advance the entire length of the keyword or we might run into a valid token that'd go missed. Parse_Next(json->state, keyword_size - 1); diff --git a/makefile.inc b/makefile.inc index 5967a45e..97f7cbc3 100644 --- a/makefile.inc +++ b/makefile.inc @@ -171,6 +171,7 @@ OBJ_COMMON= \ svvm_cmds.o \ sys_shared.o \ taskqueue.o \ + token.o \ vid_shared.o \ view.o \ wad.o \ diff --git a/parser.c b/parser.c index c9e5e84f..b5621e98 100644 --- a/parser.c +++ b/parser.c @@ -19,6 +19,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ #include "darkplaces.h" +#include "token.h" #include "parser.h" jmp_buf parse_error; @@ -49,12 +50,16 @@ DP_FUNC_NORETURN void Parse_Error(struct qparser_state_s *state, qparser_err_t e } // Advance forward in the stream as many times as 'count', cleanly. -void Parse_Next(struct qparser_state_s *state, size_t count) +void Parse_Next(struct qparser_state_s *state, int count) { - state->col += count; - state->pos += count; + const char *next = Token_Next(state->pos, count); - if(!*state->pos) + if(next) + { + state->pos = next; + state->col += count; + } + else Parse_Error(state, PARSE_ERR_EOF, NULL); } @@ -155,7 +160,7 @@ qparser_state_t *Parse_New(const unsigned char *in) out = (qparser_state_t *)Z_Malloc(sizeof(qparser_state_t)); - out->buf = in; + out->buf = (const char *)in; out->pos = NULL; out->line = 1; out->col = 1; diff --git a/parser.h b/parser.h index eee99bb9..57fdcae7 100644 --- a/parser.h +++ b/parser.h @@ -35,8 +35,8 @@ typedef enum qparser_err_e typedef struct qparser_state_s { const char *name; - const unsigned char *buf; - const unsigned char *pos; + const char *buf; + const char *pos; int line, col, depth; struct @@ -50,7 +50,7 @@ typedef struct qparser_state_s extern jmp_buf parse_error; void Parse_Error(struct qparser_state_s *state, qparser_err_t error, const char *expected); -void Parse_Next(struct qparser_state_s *state, size_t count); +void Parse_Next(struct qparser_state_s *state, int count); char Parse_NextToken(struct qparser_state_s *state); qparser_state_t *Parse_New(const unsigned char *in); qparser_state_t *Parse_LoadFile(const char *file); diff --git a/token.c b/token.c new file mode 100644 index 00000000..22f66bec --- /dev/null +++ b/token.c @@ -0,0 +1,37 @@ +/* +Copyright (C) 2000-2021 Ashley Rose Hale (LadyHavoc) +Copyright (C) 2021 David Knapp (Cloudwalk) + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +*/ + +#include "darkplaces.h" +#include "token.h" + +// Advance forward in the stream as many times as 'count', cleanly. +const char *Token_Next(const char *data, int count) +{ + const char *out = data; + for (int i = 0; i < count; i++) + { + if (!*out) + break; + out++; + } + + return out; +} diff --git a/token.h b/token.h new file mode 100644 index 00000000..5d6c970f --- /dev/null +++ b/token.h @@ -0,0 +1,24 @@ +/* +Copyright (C) 2021 David Knapp (Cloudwalk) + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +*/ + +#include "qtypes.h" +#include "qdefs.h" + +const char *Token_Next(const char *data, int count); -- 2.39.2