From: Dale Weiler Date: Sun, 2 Jun 2013 08:21:06 +0000 (+0000) Subject: Major utility rewrite for compiler memory utilization statistics. Cleanups everywhere... X-Git-Tag: v0.3.0~151^2~17 X-Git-Url: https://git.rm.cloudns.org/?a=commitdiff_plain;h=9af3c502dae7420714519dd13dff6aba40fc1fa3;p=xonotic%2Fgmqcc.git Major utility rewrite for compiler memory utilization statistics. Cleanups everywhere, no more NOTRACK stuff, all allocates are tracked. Major identifier cleanups as well. --- diff --git a/Makefile b/Makefile index fea3356..a1c08f9 100644 --- a/Makefile +++ b/Makefile @@ -44,11 +44,11 @@ ifeq ($(track), no) CFLAGS += -DNOTRACK endif -OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o fs.o utf8.o correct.o -OBJ_P = util.o fs.o conout.o opts.o pak.o -OBJ_T = test.o util.o conout.o fs.o -OBJ_C = main.o lexer.o parser.o fs.o -OBJ_X = exec-standalone.o util.o conout.o fs.o +OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o fs.o utf8.o correct.o stat.o +OBJ_P = util.o fs.o conout.o opts.o pak.o stat.o +OBJ_T = test.o util.o conout.o fs.o stat.o +OBJ_C = main.o lexer.o parser.o fs.o stat.o +OBJ_X = exec-standalone.o util.o conout.o fs.o stat.o #we have duplicate object files when dealing with creating a simple list #for dependinces. To combat this we use some clever recrusive-make to @@ -244,6 +244,7 @@ opts.o: gmqcc.h opts.def fs.o: gmqcc.h opts.def utf8.o: gmqcc.h opts.def correct.o: gmqcc.h opts.def +stat.o: gmqcc.h opts.def pak.o: gmqcc.h opts.def test.o: gmqcc.h opts.def main.o: gmqcc.h opts.def lexer.h diff --git a/gmqcc.h b/gmqcc.h index a92e109..8edcbda 100644 --- a/gmqcc.h +++ b/gmqcc.h @@ -288,20 +288,47 @@ GMQCC_IND_STRING(GMQCC_VERSION_PATCH) \ # include #endif /*! _WIN32 && !defined(__MINGW32__) */ +/*===================================================================*/ +/*=========================== stat.c ================================*/ +/*===================================================================*/ +typedef struct { + size_t key; + size_t value; +} stat_size_entry_t, **stat_size_table_t; + +void stat_info(); + +char *stat_mem_strdup (const char *, size_t, const char *, bool); +void *stat_mem_reallocate(void *, size_t, size_t, const char *); +void stat_mem_deallocate(void *); +void *stat_mem_allocate (size_t, size_t, const char *); + +stat_size_table_t stat_size_new(); +stat_size_entry_t *stat_size_get(stat_size_table_t, size_t); +void stat_size_del(stat_size_table_t); +void stat_size_put(stat_size_table_t, size_t, size_t); + +/* getters for hashtable: */ +stat_size_table_t *stat_size_hashtables_get(); +uint64_t *stat_type_hashtables_get(); +uint64_t *stat_used_hashtables_get(); +stat_size_table_t *stat_hashtables_init(); + +#define mem_a(SIZE) stat_mem_allocate ((SIZE), __LINE__, __FILE__) +#define mem_d(PTRN) stat_mem_deallocate((void*)(PTRN)) +#define mem_r(PTRN, SIZE) stat_mem_reallocate((void*)(PTRN), (SIZE), __LINE__, __FILE__) +#define mem_af(SIZE, FILE, LINE) stat_mem_allocate ((SIZE), (LINE), (FILE)) + +/* TODO: rename to mem variations */ +#define util_strdup(SRC) stat_mem_strdup((char*)(SRC), __LINE__, __FILE__, false) +#define util_strdupe(SRC) stat_mem_strdup((char*)(SRC), __LINE__, __FILE__, true) /*===================================================================*/ /*=========================== util.c ================================*/ /*===================================================================*/ -void *util_memory_a (size_t, /*****/ unsigned int, const char *); -void *util_memory_r (void *, size_t, unsigned int, const char *); -void util_memory_d (void *); -void util_meminfo (); - bool util_filexists (const char *); bool util_strupper (const char *); bool util_strdigit (const char *); -char *_util_Estrdup (const char *, const char *, size_t); -char *_util_Estrdup_empty(const char *, const char *, size_t); void util_debug (const char *, const char *, ...); void util_endianswap (void *, size_t, unsigned int); @@ -326,22 +353,6 @@ char *util_strcat (char *dest, const char *src); char *util_strncpy (char *dest, const char *src, size_t num); const char *util_strerror (int num); - -#ifdef NOTRACK -# define mem_a(x) malloc (x) -# define mem_d(x) free ((void*)x) -# define mem_r(x, n) realloc((void*)x, n) -# define mem_af(x,f,l) malloc (x) -#else -# define mem_a(x) util_memory_a((x), __LINE__, __FILE__) -# define mem_d(x) util_memory_d((void*)(x)) -# define mem_r(x, n) util_memory_r((void*)(x), (n), __LINE__, __FILE__) -# define mem_af(x,f,l) util_memory_a((x), __LINE__, __FILE__) -#endif /*! NOTRACK */ - -#define util_strdup(X) _util_Estrdup((X), __FILE__, __LINE__) -#define util_strdupe(X) _util_Estrdup_empty((X), __FILE__, __LINE__) - /* * A flexible vector implementation: all vector pointers contain some * data about themselfs exactly - sizeof(vector_t) behind the pointer diff --git a/main.c b/main.c index 3ac055d..5717fb5 100644 --- a/main.c +++ b/main.c @@ -796,8 +796,7 @@ cleanup: mem_d((void*)operators); lex_cleanup(); - util_meminfo(); - /*util_vecstats_destroy();*/ - + stat_info(); + return retval; } diff --git a/pak.c b/pak.c index 9ff7e78..ac31bdf 100644 --- a/pak.c +++ b/pak.c @@ -552,7 +552,8 @@ int main(int argc, char **argv) { /* not possible */ pak_close(pak); vec_free(files); - util_meminfo(); + stat_info(); + return EXIT_SUCCESS; } @@ -575,6 +576,6 @@ int main(int argc, char **argv) { pak_close(pak); vec_free(files); - util_meminfo(); + stat_info(); return EXIT_SUCCESS; } diff --git a/stat.c b/stat.c new file mode 100644 index 0000000..7747763 --- /dev/null +++ b/stat.c @@ -0,0 +1,590 @@ +#include "gmqcc.h" + +/* + * GMQCC performs tons of allocations, constructions, and crazyness + * all around. When trying to optimizes systems, or just get fancy + * statistics out of the compiler, it's often printf mess. This file + * implements the statistics system of the compiler. I.E the allocator + * we use to track allocations, and other systems of interest. + */ +#define ST_SIZE 1024 + +typedef struct stat_mem_block_s { + const char *file; + size_t line; + size_t size; + struct stat_mem_block_s *next; + struct stat_mem_block_s *prev; +} stat_mem_block_t; + +static uint64_t stat_mem_allocated = 0; +static uint64_t stat_mem_deallocated = 0; +static uint64_t stat_mem_allocated_total = 0; +static uint64_t stat_mem_deallocated_total = 0; +static uint64_t stat_mem_high = 0; +static uint64_t stat_mem_peak = 0; +static uint64_t stat_used_strdups = 0; +static uint64_t stat_used_vectors = 0; +static uint64_t stat_used_hashtables = 0; +static uint64_t stat_type_vectors = 0; +static uint64_t stat_type_hashtables = 0; +static stat_size_table_t stat_size_vectors = NULL; +static stat_size_table_t stat_size_hashtables = NULL; +static stat_mem_block_t *stat_mem_block_root = NULL; + +/* + * A basic header of information wrapper allocator. Simply stores + * information as a header, returns the memory + 1 past it, can be + * retrieved again with - 1. Where type is stat_mem_block_t*. + */ +void *stat_mem_allocate(size_t size, size_t line, const char *file) { + stat_mem_block_t *info = (stat_mem_block_t*)malloc(sizeof(stat_mem_block_t) + size); + void *data = (void*)(info + 1); + + if(!info) + return NULL; + + info->line = line; + info->size = size; + info->file = file; + info->prev = NULL; + info->next = stat_mem_block_root; + + if (stat_mem_block_root) + stat_mem_block_root->prev = info; + + stat_mem_block_root = info; + stat_mem_allocated += size; + stat_mem_high += size; + stat_mem_allocated_total ++; + + if (stat_mem_high > stat_mem_peak) + stat_mem_peak = stat_mem_high; + + return data; +} + +void stat_mem_deallocate(void *ptr) { + stat_mem_block_t *info = NULL; + + if (!ptr) + return; + + info = ((stat_mem_block_t*)ptr - 1); + + stat_mem_deallocated += info->size; + stat_mem_high -= info->size; + stat_mem_deallocated_total ++; + + if (info->prev) info->prev->next = info->next; + if (info->next) info->next->prev = info->prev; + + /* move ahead */ + if (info == stat_mem_block_root) + stat_mem_block_root = info->next; +} + +void *stat_mem_reallocate(void *ptr, size_t size, size_t line, const char *file) { + stat_mem_block_t *oldinfo = NULL; + stat_mem_block_t *newinfo; + + if (!ptr) + return stat_mem_allocate(size, line, file); + + /* stay consistent with glic */ + if (!size) { + stat_mem_deallocate(ptr); + return NULL; + } + + oldinfo = ((stat_mem_block_t*)ptr - 1); + newinfo = ((stat_mem_block_t*)malloc(sizeof(stat_mem_block_t) + size)); + + if (!newinfo) { + stat_mem_deallocate(ptr); + return NULL; + } + + memcpy(newinfo+1, oldinfo+1, oldinfo->size); + + if (oldinfo->prev) oldinfo->prev->next = oldinfo->next; + if (oldinfo->next) oldinfo->next->prev = oldinfo->prev; + + /* move ahead */ + if (oldinfo == stat_mem_block_root) + stat_mem_block_root = oldinfo->next; + + newinfo->line = line; + newinfo->size = size; + newinfo->file = file; + newinfo->prev = NULL; + newinfo->next = stat_mem_block_root; + + if (stat_mem_block_root) + stat_mem_block_root->prev = newinfo; + + stat_mem_block_root = newinfo; + stat_mem_allocated -= oldinfo->size; + stat_mem_high -= oldinfo->size; + stat_mem_allocated += newinfo->size; + stat_mem_high += newinfo->size; + + if (stat_mem_high > stat_mem_peak) + stat_mem_peak = stat_mem_high; + + free(oldinfo); + + return newinfo + 1; +} + +/* + * strdup does it's own malloc, we need to track malloc. We don't want + * to overwrite malloc though, infact, we can't really hook it at all + * without library specific assumptions. So we re implement strdup. + */ +char *stat_mem_strdup(const char *src, size_t line, const char *file, bool empty) { + size_t len = 0; + char *ptr = NULL; + + if (!src) + return NULL; + + len = strlen(src); + if (((!empty) ? len : true) && (ptr = (char*)stat_mem_allocate(len + 1, line, file))) { + memcpy(ptr, src, len); + ptr[len] = '\0'; + } + + stat_used_strdups ++; + return ptr; +} + +/* + * The reallocate function for resizing vectors. + */ +void _util_vec_grow(void **a, size_t i, size_t s) { + vector_t *d = vec_meta(*a); + size_t m = 0; + stat_size_entry_t *e = NULL; + void *p = NULL; + + if (*a) { + m = 2 * d->allocated + i; + p = mem_r(d, s * m + sizeof(vector_t)); + } else { + m = i + 1; + p = mem_a(s * m + sizeof(vector_t)); + ((vector_t*)p)->used = 0; + stat_used_vectors++; + } + + if (!stat_size_vectors) + stat_size_vectors = stat_size_new(); + + if ((e = stat_size_get(stat_size_vectors, s))) { + e->value ++; + } else { + stat_size_put(stat_size_vectors, s, 1); /* start off with 1 */ + stat_type_vectors++; + } + + *a = (vector_t*)p + 1; + vec_meta(*a)->allocated = m; +} + +/* + * Hash table for generic data, based on dynamic memory allocations + * all around. This is the internal interface, please look for + * EXPOSED INTERFACE comment below + */ +typedef struct hash_node_t { + char *key; /* the key for this node in table */ + void *value; /* pointer to the data as void* */ + struct hash_node_t *next; /* next node (linked list) */ +} hash_node_t; + +GMQCC_INLINE size_t util_hthash(hash_table_t *ht, const char *key) { + const uint32_t mix = 0x5BD1E995; + const uint32_t rot = 24; + size_t size = strlen(key); + uint32_t hash = 0x1EF0 /* LICRC TAB */ ^ size; + uint32_t alias = 0; + const unsigned char *data = (const unsigned char*)key; + + while (size >= 4) { + alias = (data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24)); + alias *= mix; + alias ^= alias >> rot; + alias *= mix; + + hash *= mix; + hash ^= alias; + + data += 4; + size -= 4; + } + + switch (size) { + case 3: hash ^= data[2] << 16; + case 2: hash ^= data[1] << 8; + case 1: hash ^= data[0]; + hash *= mix; + } + + hash ^= hash >> 13; + hash *= mix; + hash ^= hash >> 15; + + return (size_t) (hash % ht->size); +} + +static hash_node_t *_util_htnewpair(const char *key, void *value) { + hash_node_t *node; + if (!(node = (hash_node_t*)mem_a(sizeof(hash_node_t)))) + return NULL; + + if (!(node->key = util_strdupe(key))) { + mem_d(node); + return NULL; + } + + node->value = value; + node->next = NULL; + + return node; +} + +/* + * EXPOSED INTERFACE for the hashtable implementation + * util_htnew(size) -- to make a new hashtable + * util_htset(table, key, value, sizeof(value)) -- to set something in the table + * util_htget(table, key) -- to get something from the table + * util_htdel(table) -- to delete the table + */ +hash_table_t *util_htnew(size_t size) { + hash_table_t *hashtable = NULL; + stat_size_entry_t *find = NULL; + + if (size < 1) + return NULL; + + if (!stat_size_hashtables) + stat_size_hashtables = stat_size_new(); + + if (!(hashtable = (hash_table_t*)mem_a(sizeof(hash_table_t)))) + return NULL; + + if (!(hashtable->table = (hash_node_t**)mem_a(sizeof(hash_node_t*) * size))) { + mem_d(hashtable); + return NULL; + } + + if ((find = stat_size_get(stat_size_hashtables, size))) + find->value++; + else { + stat_used_hashtables++; + stat_size_put(stat_size_hashtables, size, 1); + } + + hashtable->size = size; + memset(hashtable->table, 0, sizeof(hash_node_t*) * size); + + stat_type_hashtables++; + return hashtable; +} + +void util_htseth(hash_table_t *ht, const char *key, size_t bin, void *value) { + hash_node_t *newnode = NULL; + hash_node_t *next = NULL; + hash_node_t *last = NULL; + + next = ht->table[bin]; + + while (next && next->key && strcmp(key, next->key) > 0) + last = next, next = next->next; + + /* already in table, do a replace */ + if (next && next->key && strcmp(key, next->key) == 0) { + next->value = value; + } else { + /* not found, grow a pair man :P */ + newnode = _util_htnewpair(key, value); + if (next == ht->table[bin]) { + newnode->next = next; + ht->table[bin] = newnode; + } else if (!next) { + last->next = newnode; + } else { + newnode->next = next; + last->next = newnode; + } + } +} + +void util_htset(hash_table_t *ht, const char *key, void *value) { + util_htseth(ht, key, util_hthash(ht, key), value); +} + +void *util_htgeth(hash_table_t *ht, const char *key, size_t bin) { + hash_node_t *pair = ht->table[bin]; + + while (pair && pair->key && strcmp(key, pair->key) > 0) + pair = pair->next; + + if (!pair || !pair->key || strcmp(key, pair->key) != 0) + return NULL; + + return pair->value; +} + +void *util_htget(hash_table_t *ht, const char *key) { + return util_htgeth(ht, key, util_hthash(ht, key)); +} + +void *code_util_str_htgeth(hash_table_t *ht, const char *key, size_t bin) { + hash_node_t *pair; + size_t len, keylen; + int cmp; + + keylen = strlen(key); + + pair = ht->table[bin]; + while (pair && pair->key) { + len = strlen(pair->key); + if (len < keylen) { + pair = pair->next; + continue; + } + if (keylen == len) { + cmp = strcmp(key, pair->key); + if (cmp == 0) + return pair->value; + if (cmp < 0) + return NULL; + pair = pair->next; + continue; + } + cmp = strcmp(key, pair->key + len - keylen); + if (cmp == 0) { + uintptr_t up = (uintptr_t)pair->value; + up += len - keylen; + return (void*)up; + } + pair = pair->next; + } + return NULL; +} + +/* + * Free all allocated data in a hashtable, this is quite the amount + * of work. + */ +void util_htrem(hash_table_t *ht, void (*callback)(void *data)) { + size_t i = 0; + for (; i < ht->size; i++) { + hash_node_t *n = ht->table[i]; + hash_node_t *p; + + /* free in list */ + while (n) { + if (n->key) + mem_d(n->key); + if (callback) + callback(n->value); + p = n; + n = n->next; + mem_d(p); + } + + } + /* free table */ + mem_d(ht->table); + mem_d(ht); +} + +void util_htrmh(hash_table_t *ht, const char *key, size_t bin, void (*cb)(void*)) { + hash_node_t **pair = &ht->table[bin]; + hash_node_t *tmp; + + while (*pair && (*pair)->key && strcmp(key, (*pair)->key) > 0) + pair = &(*pair)->next; + + tmp = *pair; + if (!tmp || !tmp->key || strcmp(key, tmp->key) != 0) + return; + + if (cb) + (*cb)(tmp->value); + + *pair = tmp->next; + mem_d(tmp->key); + mem_d(tmp); +} + +void util_htrm(hash_table_t *ht, const char *key, void (*cb)(void*)) { + util_htrmh(ht, key, util_hthash(ht, key), cb); +} + +void util_htdel(hash_table_t *ht) { + util_htrem(ht, NULL); +} + +/* + * A tiny size_t key-value hashtbale for tracking vector and hashtable + * sizes. We can use it for other things too, if we need to. This is + * very TIGHT, and efficent in terms of space though. + */ +stat_size_table_t stat_size_new() { + return (stat_size_table_t)memset( + mem_a(sizeof(stat_size_entry_t*) * ST_SIZE), + 0, ST_SIZE * sizeof(stat_size_entry_t*) + ); +} + +void stat_size_del(stat_size_table_t table) { + size_t i = 0; + for (; i < ST_SIZE; i++) if(table[i]) mem_d(table[i]); + mem_d(table); +} + +stat_size_entry_t *stat_size_get(stat_size_table_t table, size_t key) { + size_t hash = (key % ST_SIZE); + while (table[hash] && table[hash]->key != key) + hash = (hash + 1) % ST_SIZE; + return table[hash]; +} +void stat_size_put(stat_size_table_t table, size_t key, size_t value) { + size_t hash = (key % ST_SIZE); + while (table[hash] && table[hash]->key != key) + hash = (hash + 1) % ST_SIZE; + table[hash] = (stat_size_entry_t*)mem_a(sizeof(stat_size_entry_t)); + table[hash]->key = key; + table[hash]->value = value; +} + +/* + * The following functions below implement printing / dumping of statistical + * information. + */ +static void stat_dump_mem_contents(stat_mem_block_t *memory, uint16_t cols) { + uint32_t i, j; + for (i = 0; i < memory->size + ((memory->size % cols) ? (cols - memory->size % cols) : 0); i++) { + if (i % cols == 0) con_out(" 0x%06X: ", i); + if (i < memory->size) con_out("%02X " , 0xFF & ((unsigned char*)(memory + 1))[i]); + else con_out(" "); + + if ((uint16_t)(i % cols) == (cols - 1)) { + for (j = i - (cols - 1); j <= i; j++) { + con_out("%c", + (j >= memory->size) + ? ' ' + : (isprint(((unsigned char*)(memory + 1))[j])) + ? 0xFF & ((unsigned char*)(memory + 1)) [j] + : '.' + ); + } + con_out("\n"); + } + } +} + +static void stat_dump_mem_leaks() { + stat_mem_block_t *info; + for (info = stat_mem_block_root; info; info = info->next) { + con_out("lost: %u (bytes) at %s:%u\n", + info->size, + info->file, + info->line + ); + + stat_dump_mem_contents(info, OPTS_OPTION_U16(OPTION_MEMDUMPCOLS)); + } +} + +static void stat_dump_mem_info() { + con_out("Memory information:\n\ + Total allocations: %llu\n\ + Total deallocations: %llu\n\ + Total allocated: %f (MB)\n\ + Total deallocated: %f (MB)\n\ + Total peak memory: %f (MB)\n\ + Total leaked memory: %f (MB) in %llu allocations\n", + stat_mem_allocated_total, + stat_mem_deallocated_total, + (float)(stat_mem_allocated) / 1048576.0f, + (float)(stat_mem_deallocated) / 1048576.0f, + (float)(stat_mem_high) / 1048576.0f, + (float)(stat_mem_allocated - stat_mem_deallocated) / 1048576.0f, + stat_mem_allocated_total - stat_mem_deallocated_total + ); +} + +static void stat_dump_stats_table(stat_size_table_t table, const char *string, uint64_t *size) { + size_t i,j; + + for (i = 0, j = 0; i < ST_SIZE; i++) { + stat_size_entry_t *entry; + + if (!(entry = table[i])) + continue; + + con_out(string, (unsigned)j, (unsigned)entry->key, (unsigned)entry->value); + j++; + + if (size) + *size += entry->key * entry->value; + } +} + +void stat_info() { + if (OPTS_OPTION_BOOL(OPTION_DEBUG)) + stat_dump_mem_leaks(); + + if (OPTS_OPTION_BOOL(OPTION_DEBUG) || + OPTS_OPTION_BOOL(OPTION_MEMCHK)) + stat_dump_mem_info(); + + if (OPTS_OPTION_BOOL(OPTION_MEMCHK) || + OPTS_OPTION_BOOL(OPTION_STATISTICS)) { + uint64_t mem; + + con_out("\nAdditional Statistics:\n\ + Total vectors allocated: %llu\n\ + Total string duplicates: %llu\n\ + Total hashtables allocated: %llu\n\ + Total unique vector sizes: %llu\n", + stat_used_vectors, + stat_used_strdups, + stat_used_hashtables, + stat_type_vectors + ); + + stat_dump_stats_table ( + stat_size_vectors, + " %2u| # of %4u byte vectors: %u\n", + &mem + ); + + con_out ( + " Total unique hashtable sizes: %llu\n", + stat_type_hashtables + ); + + stat_dump_stats_table ( + stat_size_hashtables, + " %2u| # of %4u element hashtables: %u\n", + NULL + ); + + con_out ( + " Total vector memory: %f (MB)\n", + (float)(mem) / 1048576.0f + ); + } + + if (stat_size_vectors) + stat_size_del(stat_size_vectors); + if (stat_size_hashtables) + stat_size_del(stat_size_hashtables); +} +#undef ST_SIZE diff --git a/test.c b/test.c index 6682983..7b9254d 100644 --- a/test.c +++ b/test.c @@ -1321,7 +1321,7 @@ int main(int argc, char **argv) { } con_change(redirout, redirerr); succeed = test_perform("tests", defs); - util_meminfo(); + stat_info(); return (succeed) ? EXIT_SUCCESS : EXIT_FAILURE; diff --git a/util.c b/util.c index 3dcf873..c6c8925 100644 --- a/util.c +++ b/util.c @@ -25,338 +25,6 @@ #include #include "gmqcc.h" -/* TODO: remove globals ... */ -static uint64_t mem_ab = 0; -static uint64_t mem_db = 0; -static uint64_t mem_at = 0; -static uint64_t mem_dt = 0; -static uint64_t mem_pk = 0; -static uint64_t mem_hw = 0; - -struct memblock_t { - const char *file; - unsigned int line; - size_t byte; - struct memblock_t *next; - struct memblock_t *prev; -}; - -#define PEAK_MEM \ - do { \ - if (mem_hw > mem_pk) \ - mem_pk = mem_hw; \ - } while (0) - -static struct memblock_t *mem_start = NULL; - -void *util_memory_a(size_t byte, unsigned int line, const char *file) { - struct memblock_t *info = (struct memblock_t*)malloc(sizeof(struct memblock_t) + byte); - void *data = (void*)(info+1); - if (!info) return NULL; - info->line = line; - info->byte = byte; - info->file = file; - info->prev = NULL; - info->next = mem_start; - if (mem_start) - mem_start->prev = info; - mem_start = info; - - mem_at++; - mem_ab += info->byte; - mem_hw += info->byte; - - PEAK_MEM; - - return data; -} - -void util_memory_d(void *ptrn) { - struct memblock_t *info = NULL; - - if (!ptrn) return; - info = ((struct memblock_t*)ptrn - 1); - - mem_db += info->byte; - mem_hw -= info->byte; - mem_dt++; - - if (info->prev) - info->prev->next = info->next; - if (info->next) - info->next->prev = info->prev; - if (info == mem_start) - mem_start = info->next; - - free(info); -} - -void *util_memory_r(void *ptrn, size_t byte, unsigned int line, const char *file) { - struct memblock_t *oldinfo = NULL; - - struct memblock_t *newinfo; - - if (!ptrn) - return util_memory_a(byte, line, file); - if (!byte) { - util_memory_d(ptrn); - return NULL; - } - - oldinfo = ((struct memblock_t*)ptrn - 1); - newinfo = ((struct memblock_t*)malloc(sizeof(struct memblock_t) + byte)); - - /* new data */ - if (!newinfo) { - util_memory_d(oldinfo+1); - return NULL; - } - - /* copy old */ - memcpy(newinfo+1, oldinfo+1, oldinfo->byte); - - /* free old */ - if (oldinfo->prev) - oldinfo->prev->next = oldinfo->next; - if (oldinfo->next) - oldinfo->next->prev = oldinfo->prev; - if (oldinfo == mem_start) - mem_start = oldinfo->next; - - /* fill info */ - newinfo->line = line; - newinfo->byte = byte; - newinfo->file = file; - newinfo->prev = NULL; - newinfo->next = mem_start; - if (mem_start) - mem_start->prev = newinfo; - mem_start = newinfo; - - mem_ab -= oldinfo->byte; - mem_hw -= oldinfo->byte; - mem_ab += newinfo->byte; - mem_hw += newinfo->byte; - - PEAK_MEM; - - free(oldinfo); - - return newinfo+1; -} - -static void util_dumpmem(struct memblock_t *memory, uint16_t cols) { - uint32_t i, j; - for (i = 0; i < memory->byte + ((memory->byte % cols) ? (cols - memory->byte % cols) : 0); i++) { - if (i % cols == 0) con_out(" 0x%06X: ", i); - if (i < memory->byte) con_out("%02X " , 0xFF & ((char*)(memory + 1))[i]); - else con_out(" "); - - if ((uint16_t)(i % cols) == (cols - 1)) { - for (j = i - (cols - 1); j <= i; j++) { - con_out("%c", - (j >= memory->byte) - ? ' ' - : (isprint(((char*)(memory + 1))[j])) - ? 0xFF & ((char*)(memory + 1)) [j] - : '.' - ); - } - con_out("\n"); - } - } -} - -/* - * The following is a VERY tight, efficent, hashtable for integer - * values and keys, and for nothing more. We could make our existing - * hashtable support type-genericness through a void * pointer but, - * ideally that would make things more complicated. We also don't need - * that much of a bloat for something as basic as this. - */ -typedef struct { - size_t key; - size_t value; -} size_entry_t; -#define ST_SIZE 1024 - -typedef size_entry_t **size_table_t; - -static size_table_t util_st_new() { - return (size_table_t)memset( - mem_a(sizeof(size_entry_t*) * ST_SIZE), - 0, ST_SIZE * sizeof(size_entry_t*) - ); -} -static void util_st_del(size_table_t table) { - size_t i = 0; - for (; i < ST_SIZE; i++) if(table[i]) mem_d(table[i]); - mem_d(table); -} -static size_entry_t *util_st_get(size_table_t table, size_t key) { - size_t hash = (key % ST_SIZE); - while (table[hash] && table[hash]->key != key) - hash = (hash + 1) % ST_SIZE; - return table[hash]; -} -static void util_st_put(size_table_t table, size_t key, size_t value) { - size_t hash = (key % ST_SIZE); - while (table[hash] && table[hash]->key != key) - hash = (hash + 1) % ST_SIZE; - table[hash] = (size_entry_t*)mem_a(sizeof(size_entry_t)); - table[hash]->key = key; - table[hash]->value = value; -} - -static uint64_t strdups = 0; -static uint64_t vectors = 0; -static uint64_t vector_sizes = 0; -static uint64_t hashtables = 0; -static uint64_t hashtable_sizes = 0; -static size_table_t vector_usage = NULL; -static size_table_t hashtable_usage = NULL; - -void util_meminfo() { - struct memblock_t *info; - - if (OPTS_OPTION_BOOL(OPTION_DEBUG)) { - for (info = mem_start; info; info = info->next) { - con_out("lost: %u (bytes) at %s:%u\n", - info->byte, - info->file, - info->line); - - util_dumpmem(info, OPTS_OPTION_U16(OPTION_MEMDUMPCOLS)); - } - } - - if (OPTS_OPTION_BOOL(OPTION_DEBUG) || - OPTS_OPTION_BOOL(OPTION_MEMCHK)) { - con_out("Memory information:\n\ - Total allocations: %llu\n\ - Total deallocations: %llu\n\ - Total allocated: %f (MB)\n\ - Total deallocated: %f (MB)\n\ - Total peak memory: %f (MB)\n\ - Total leaked memory: %f (MB) in %llu allocations\n", - mem_at, - mem_dt, - (float)(mem_ab) / 1048576.0f, - (float)(mem_db) / 1048576.0f, - (float)(mem_pk) / 1048576.0f, - (float)(mem_ab - mem_db) / 1048576.0f, - - /* could be more clever */ - (mem_at - mem_dt) - ); - } - - if (OPTS_OPTION_BOOL(OPTION_STATISTICS) || - OPTS_OPTION_BOOL(OPTION_MEMCHK)) { - size_t i = 0; - size_t e = 1; - uint64_t vectormem = 0; - - con_out("\nAdditional Statistics:\n\ - Total vectors allocated: %llu\n\ - Total string duplicates: %llu\n\ - Total hashtables allocated: %llu\n\ - Total unique vector sizes: %llu\n", - vectors, - strdups, - hashtables, - vector_sizes - ); - - for (; i < ST_SIZE; i++) { - size_entry_t *entry; - - if (!(entry = vector_usage[i])) - continue; - - con_out(" %2u| # of %4u byte vectors: %u\n", - (unsigned)e, - (unsigned)entry->key, - (unsigned)entry->value - ); - e++; - - vectormem += entry->key * entry->value; - } - - con_out("\ - Total unique hashtable sizes: %llu\n", - hashtable_sizes - ); - - for (i = 0, e = 1; i < ST_SIZE; i++) { - size_entry_t *entry; - - if (!(entry = hashtable_usage[i])) - continue; - - con_out(" %2u| # of %4u element hashtables: %u\n", - (unsigned)e, - (unsigned)entry->key, - (unsigned)entry->value - ); - e++; - } - - con_out(" Total vector memory: %f (MB)\n", - (float)(vectormem) / 1048576.0f - ); - } - - if (vector_usage) - util_st_del(vector_usage); - if (hashtable_usage) - util_st_del(hashtable_usage); -} - -/* - * Some string utility functions, because strdup uses malloc, and we want - * to track all memory (without replacing malloc). - */ -char *_util_Estrdup(const char *s, const char *file, size_t line) { - size_t len = 0; - char *ptr = NULL; - - /* in case of -DNOTRACK */ - (void)file; - (void)line; - - if (!s) - return NULL; - - if ((len = strlen(s)) && (ptr = (char*)mem_af(len+1, line, file))) { - memcpy(ptr, s, len); - ptr[len] = '\0'; - } - strdups++; - return ptr; -} - -char *_util_Estrdup_empty(const char *s, const char *file, size_t line) { - size_t len = 0; - char *ptr = NULL; - - /* in case of -DNOTRACK */ - (void)file; - (void)line; - - if (!s) - return NULL; - - len = strlen(s); - if ((ptr = (char*)mem_af(len+1, line, file))) { - memcpy(ptr, s, len); - ptr[len] = '\0'; - } - strdups++; - return ptr; -} - void util_debug(const char *area, const char *ms, ...) { va_list va; if (!OPTS_OPTION_BOOL(OPTION_DEBUG)) @@ -544,274 +212,6 @@ size_t util_strtononcmd(const char *in, char *out, size_t outsz) { return sz-1; } -/* TODO: rewrite ... when I redo the ve cleanup */ -void _util_vec_grow(void **a, size_t i, size_t s) { - vector_t *d = vec_meta(*a); - size_t m = 0; - size_entry_t *e = NULL; - void *p = NULL; - - if (*a) { - m = 2 * d->allocated + i; - p = mem_r(d, s * m + sizeof(vector_t)); - } else { - m = i + 1; - p = mem_a(s * m + sizeof(vector_t)); - ((vector_t*)p)->used = 0; - vectors++; - } - - if (!vector_usage) - vector_usage = util_st_new(); - - if ((e = util_st_get(vector_usage, s))) { - e->value ++; - } else { - util_st_put(vector_usage, s, 1); /* start off with 1 */ - vector_sizes++; - } - - *a = (vector_t*)p + 1; - vec_meta(*a)->allocated = m; -} - -/* - * Hash table for generic data, based on dynamic memory allocations - * all around. This is the internal interface, please look for - * EXPOSED INTERFACE comment below - */ -typedef struct hash_node_t { - char *key; /* the key for this node in table */ - void *value; /* pointer to the data as void* */ - struct hash_node_t *next; /* next node (linked list) */ -} hash_node_t; - -GMQCC_INLINE size_t util_hthash(hash_table_t *ht, const char *key) { - const uint32_t mix = 0x5BD1E995; - const uint32_t rot = 24; - size_t size = strlen(key); - uint32_t hash = 0x1EF0 /* LICRC TAB */ ^ size; - uint32_t alias = 0; - const unsigned char *data = (const unsigned char*)key; - - while (size >= 4) { - alias = (data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24)); - alias *= mix; - alias ^= alias >> rot; - alias *= mix; - - hash *= mix; - hash ^= alias; - - data += 4; - size -= 4; - } - - switch (size) { - case 3: hash ^= data[2] << 16; - case 2: hash ^= data[1] << 8; - case 1: hash ^= data[0]; - hash *= mix; - } - - hash ^= hash >> 13; - hash *= mix; - hash ^= hash >> 15; - - return (size_t) (hash % ht->size); -} - -static hash_node_t *_util_htnewpair(const char *key, void *value) { - hash_node_t *node; - if (!(node = (hash_node_t*)mem_a(sizeof(hash_node_t)))) - return NULL; - - if (!(node->key = util_strdupe(key))) { - mem_d(node); - return NULL; - } - - node->value = value; - node->next = NULL; - - return node; -} - -/* - * EXPOSED INTERFACE for the hashtable implementation - * util_htnew(size) -- to make a new hashtable - * util_htset(table, key, value, sizeof(value)) -- to set something in the table - * util_htget(table, key) -- to get something from the table - * util_htdel(table) -- to delete the table - */ -hash_table_t *util_htnew(size_t size) { - hash_table_t *hashtable = NULL; - size_entry_t *find; - - if (size < 1) - return NULL; - - if (!hashtable_usage) - hashtable_usage = util_st_new(); - - if (!(hashtable = (hash_table_t*)mem_a(sizeof(hash_table_t)))) - return NULL; - - if (!(hashtable->table = (hash_node_t**)mem_a(sizeof(hash_node_t*) * size))) { - mem_d(hashtable); - return NULL; - } - - if ((find = util_st_get(hashtable_usage, size))) - find->value++; - else { - hashtable_sizes++; - util_st_put(hashtable_usage, size, 1); - } - - hashtable->size = size; - memset(hashtable->table, 0, sizeof(hash_node_t*) * size); - - hashtables++; - return hashtable; -} - -void util_htseth(hash_table_t *ht, const char *key, size_t bin, void *value) { - hash_node_t *newnode = NULL; - hash_node_t *next = NULL; - hash_node_t *last = NULL; - - next = ht->table[bin]; - - while (next && next->key && strcmp(key, next->key) > 0) - last = next, next = next->next; - - /* already in table, do a replace */ - if (next && next->key && strcmp(key, next->key) == 0) { - next->value = value; - } else { - /* not found, grow a pair man :P */ - newnode = _util_htnewpair(key, value); - if (next == ht->table[bin]) { - newnode->next = next; - ht->table[bin] = newnode; - } else if (!next) { - last->next = newnode; - } else { - newnode->next = next; - last->next = newnode; - } - } -} - -void util_htset(hash_table_t *ht, const char *key, void *value) { - util_htseth(ht, key, util_hthash(ht, key), value); -} - -void *util_htgeth(hash_table_t *ht, const char *key, size_t bin) { - hash_node_t *pair = ht->table[bin]; - - while (pair && pair->key && strcmp(key, pair->key) > 0) - pair = pair->next; - - if (!pair || !pair->key || strcmp(key, pair->key) != 0) - return NULL; - - return pair->value; -} - -void *util_htget(hash_table_t *ht, const char *key) { - return util_htgeth(ht, key, util_hthash(ht, key)); -} - -void *code_util_str_htgeth(hash_table_t *ht, const char *key, size_t bin) { - hash_node_t *pair; - size_t len, keylen; - int cmp; - - keylen = strlen(key); - - pair = ht->table[bin]; - while (pair && pair->key) { - len = strlen(pair->key); - if (len < keylen) { - pair = pair->next; - continue; - } - if (keylen == len) { - cmp = strcmp(key, pair->key); - if (cmp == 0) - return pair->value; - if (cmp < 0) - return NULL; - pair = pair->next; - continue; - } - cmp = strcmp(key, pair->key + len - keylen); - if (cmp == 0) { - uintptr_t up = (uintptr_t)pair->value; - up += len - keylen; - return (void*)up; - } - pair = pair->next; - } - return NULL; -} - -/* - * Free all allocated data in a hashtable, this is quite the amount - * of work. - */ -void util_htrem(hash_table_t *ht, void (*callback)(void *data)) { - size_t i = 0; - for (; i < ht->size; i++) { - hash_node_t *n = ht->table[i]; - hash_node_t *p; - - /* free in list */ - while (n) { - if (n->key) - mem_d(n->key); - if (callback) - callback(n->value); - p = n; - n = n->next; - mem_d(p); - } - - } - /* free table */ - mem_d(ht->table); - mem_d(ht); -} - -void util_htrmh(hash_table_t *ht, const char *key, size_t bin, void (*cb)(void*)) { - hash_node_t **pair = &ht->table[bin]; - hash_node_t *tmp; - - while (*pair && (*pair)->key && strcmp(key, (*pair)->key) > 0) - pair = &(*pair)->next; - - tmp = *pair; - if (!tmp || !tmp->key || strcmp(key, tmp->key) != 0) - return; - - if (cb) - (*cb)(tmp->value); - - *pair = tmp->next; - mem_d(tmp->key); - mem_d(tmp); -} - -void util_htrm(hash_table_t *ht, const char *key, void (*cb)(void*)) { - util_htrmh(ht, key, util_hthash(ht, key), cb); -} - -void util_htdel(hash_table_t *ht) { - util_htrem(ht, NULL); -} - /* * Portable implementation of vasprintf/asprintf. Assumes vsnprintf * exists, otherwise compiler error.