armory3d.armortools/base/sources/libs/gc.c

// MIT License

// Copyright (c) 2019 Marc Kirchner

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// https://github.com/mkirchner/gc

#include "gc.h"

#include <setjmp.h>
#include <stdlib.h>
#include <string.h>
// #include <iron_system.h>

#define PTRSIZE sizeof(char *)

#define GC_TAG_NONE 0x0
#define GC_TAG_LEAF 0x1
#define GC_TAG_MARK 0x2
#define GC_TAG_ROOT 0x4
#define GC_TAG_CUT 0x8

#if defined(_MSC_VER) && !defined(__clang__)
#define __builtin_frame_address(x)  ((void)(x), _AddressOfReturnAddress())
#endif

typedef struct gc_allocation {
	void *ptr; // mem pointer
	size_t size; // allocated size in bytes
	int *array_length; // if this alloc is an array
	char tag; // the tag for mark-and-sweep
	char roots; // number of root users
	struct gc_allocation *next; // separate chaining
	struct gc_allocation *cut;
} gc_allocation_t;

typedef struct gc_allocation_map {
	size_t capacity;
	size_t min_capacity;
	double downsize_factor;
	double upsize_factor;
	double sweep_factor;
	size_t sweep_limit;
	size_t size;
	gc_allocation_t **allocs;
} gc_allocation_map_t;

typedef struct garbage_collector {
    struct gc_allocation_map *allocs; // allocation map
    // struct gc_allocation_map *roots; // root map
    bool paused; // (temporarily) switch gc on/off
    void *bos; // bottom of stack
} garbage_collector_t;

garbage_collector_t _gc;
garbage_collector_t *gc = &_gc;

static gc_allocation_t *gc_allocation_new(void *ptr, size_t size) {
	gc_allocation_t *a = (gc_allocation_t *)malloc(sizeof(gc_allocation_t));
	a->ptr = ptr;
	a->size = size;
	a->array_length = NULL;
	a->tag = GC_TAG_NONE;
	a->roots = 0;
	a->next = NULL;
	a->cut = NULL;
	return a;
}

static void gc_allocation_delete(gc_allocation_t *a) {
	free(a);
}

static gc_allocation_map_t *gc_allocation_map_new(size_t min_capacity, size_t capacity,
		double sweep_factor, double downsize_factor, double upsize_factor) {
	gc_allocation_map_t *am = (gc_allocation_map_t *)malloc(sizeof(gc_allocation_map_t));
	am->min_capacity = min_capacity;
	am->capacity = capacity;
	am->sweep_factor = sweep_factor;
	am->sweep_limit = (int)(sweep_factor * am->capacity);
	am->downsize_factor = downsize_factor;
	am->upsize_factor = upsize_factor;
	am->allocs = (gc_allocation_t **)calloc(am->capacity, sizeof(gc_allocation_t *));
	am->size = 0;
	return am;
}

static void gc_allocation_map_delete(gc_allocation_map_t *am) {
	// Iterate over the map
	gc_allocation_t *alloc;
	gc_allocation_t *tmp;
	for (size_t i = 0; i < am->capacity; ++i) {
		if ((alloc = am->allocs[i])) {
			// Make sure to follow the chain inside a bucket
			while (alloc) {
				tmp = alloc;
				alloc = alloc->next;
				// free the management structure
				gc_allocation_delete(tmp);
			}
		}
	}
	free(am->allocs);
	free(am);
}

static size_t gc_hash(void *ptr) {
	return ((uintptr_t)ptr) >> 3;
}

static void gc_allocation_map_resize(gc_allocation_map_t *am, size_t new_capacity) {
	if (new_capacity <= am->min_capacity) {
		return;
	}
	// Replaces the existing items array in the hash table
	// with a resized one and pushes items into the new, correct buckets
	gc_allocation_t **resized_allocs = calloc(new_capacity, sizeof(gc_allocation_t *));

	for (size_t i = 0; i < am->capacity; ++i) {
		gc_allocation_t *alloc = am->allocs[i];
		while (alloc) {
			gc_allocation_t *next_alloc = alloc->next;
			size_t new_index = gc_hash(alloc->ptr) & (new_capacity - 1);
			alloc->next = resized_allocs[new_index];
			resized_allocs[new_index] = alloc;
			alloc = next_alloc;
		}
	}
	free(am->allocs);
	am->capacity = new_capacity;
	am->allocs = resized_allocs;
	am->sweep_limit = am->size + am->sweep_factor * (am->capacity - am->size);
}

static bool gc_allocation_map_resize_to_fit(gc_allocation_map_t *am) {
	double load_factor = (double)am->size / (double)am->capacity;
	if (load_factor > am->upsize_factor) {
		gc_allocation_map_resize(am, am->capacity * 2);
		return true;
	}
	if (load_factor < am->downsize_factor) {
		gc_allocation_map_resize(am, am->capacity / 2);
		return true;
	}
	return false;
}

static gc_allocation_t *gc_allocation_map_get(gc_allocation_map_t *am, void *ptr) {
	size_t index = gc_hash(ptr) & (am->capacity - 1); // % am->capacity
	gc_allocation_t *cur = am->allocs[index];
	while(cur) {
		if (cur->ptr == ptr) {
			return cur;
		}
		cur = cur->next;
	}
	return NULL;
}

static gc_allocation_t *gc_allocation_map_put(gc_allocation_map_t *am, gc_allocation_t *alloc) {
	size_t index = gc_hash(alloc->ptr) & (am->capacity - 1);
	gc_allocation_t *cur = am->allocs[index];
	gc_allocation_t *prev = NULL;
	/* Upsert if ptr is already known */
	while (cur != NULL) {
		if (cur->ptr == alloc->ptr) {
			// found it
			alloc->next = cur->next;
			if (!prev) {
				// position 0
				am->allocs[index] = alloc;
			}
			else {
				// in the list
				prev->next = alloc;
			}
			gc_allocation_delete(cur);
			return alloc;

		}
		prev = cur;
		cur = cur->next;
	}
	/* Insert at the front of the separate chaining list */
	cur = am->allocs[index];
	alloc->next = cur;
	am->allocs[index] = alloc;
	am->size++;
	void *p = alloc->ptr;
	if (gc_allocation_map_resize_to_fit(am)) {
		alloc = gc_allocation_map_get(am, p);
	}
	return alloc;
}

static void gc_allocation_map_remove(gc_allocation_map_t *am, void *ptr, bool allow_resize) {
	// ignores unknown keys
	size_t index = gc_hash(ptr) & (am->capacity - 1);
	gc_allocation_t *cur = am->allocs[index];
	gc_allocation_t *prev = NULL;
	gc_allocation_t *next;
	while (cur != NULL) {
		next = cur->next;
		if (cur->ptr == ptr) {
			// found it
			if (!prev) {
				// first item in list
				am->allocs[index] = cur->next;
			}
			else {
				// not the first item in the list
				prev->next = cur->next;
			}
			gc_allocation_delete(cur);
			am->size--;
		}
		else {
			// move on
			prev = cur;
		}
		cur = next;
	}
	if (allow_resize) {
		gc_allocation_map_resize_to_fit(am);
	}
}

static void* gc_mcalloc(size_t count, size_t size) {
	if (!count) {
		return malloc(size);
	}
	return calloc(count, size);
}

static void *gc_allocate(size_t count, size_t size) {
	/* Allocation logic that generalizes over malloc/calloc */
	/* Check if we reached the high-water mark and need to clean up */
	if (gc->allocs->size > gc->allocs->sweep_limit && !gc->paused) {
		size_t freed_mem = _gc_run();
	}
	/* With cleanup out of the way, attempt to allocate memory */
	void *ptr = gc_mcalloc(count, size);
	size_t alloc_size = count ? count * size : size;
	/* If allocation fails, force an out-of-policy run to free some memory and try again. */
	if (!ptr && !gc->paused) {
		_gc_run();
		ptr = gc_mcalloc(count, size);
	}
	/* Start managing the memory we received from the system */
	if (ptr) {
		gc_allocation_t *alloc = gc_allocation_map_put(gc->allocs, gc_allocation_new(ptr, alloc_size));
		/* Deal with metadata allocation failure */
		if (alloc) {
			ptr = alloc->ptr;
		}
		else {
			/* We failed to allocate the metadata, fail cleanly. */
			free(ptr);
			ptr = NULL;
		}
	}
	return ptr;
}

void _gc_array(void *ptr, int *length) {
	gc_allocation_t *alloc = gc_allocation_map_get(gc->allocs, ptr);
	if (alloc) {
		// alloc->array_length = length;
	}
}

void _gc_leaf(void *ptr) {
	gc_allocation_t *alloc = gc_allocation_map_get(gc->allocs, ptr);
	if (alloc) {
		alloc->tag |= GC_TAG_LEAF;
	}
}

void _gc_root(void *ptr) {
	gc_allocation_t *alloc = gc_allocation_map_get(gc->allocs, ptr);
	if (alloc) {
		alloc->roots++;
		alloc->tag |= GC_TAG_ROOT;
	}
}

void _gc_unroot(void *ptr) {
	gc_allocation_t *alloc = gc_allocation_map_get(gc->allocs, ptr);
	if (alloc) {
		alloc->roots--;
		if (alloc->roots == 0) {
			alloc->tag &= ~GC_TAG_ROOT;
		}
	}
}

static inline uint64_t pad(int di, int n) {
	return (n - (di % n)) % n;
}

// Cut out a leaf out of an existing allocation
void *_gc_cut(void *ptr, size_t pos, size_t size) {
	size += pad(size, 8);

	// Start
	gc_allocation_t *start = gc_allocation_map_get(gc->allocs, ptr);
	while (start->cut) {
		start = start->cut;
	}
	size_t start_size = start->size;
	pos -= (char *)start->ptr - (char *)ptr;
	start->size = pos;

	// Middle
	gc_allocation_t *middle = gc_allocation_new((char *)start->ptr + pos, size);
	middle->tag |= GC_TAG_CUT;
	middle->tag |= GC_TAG_LEAF;
	start->cut = middle;
	gc_allocation_map_put(gc->allocs, middle);

	// End
	gc_allocation_t *end = gc_allocation_new((char *)start->ptr + pos + size, start_size - size - pos);
	end->tag |= GC_TAG_CUT;
	middle->cut = end;
	gc_allocation_map_put(gc->allocs, end);

	return middle->ptr;
}

static void gc_mark_alloc(void *ptr) {
	gc_allocation_t *alloc = gc_allocation_map_get(gc->allocs, ptr);
	/* Mark if alloc exists and is not tagged already, otherwise skip */
	if (alloc && !(alloc->tag & GC_TAG_MARK)) {
		alloc->tag |= GC_TAG_MARK;
		if (!(alloc->tag & GC_TAG_LEAF)) { // Skip contents
			/* Iterate over allocation contents and mark them as well */
			int size = alloc->array_length ? (*alloc->array_length) * PTRSIZE : alloc->size;

			for (char *p = (char *)alloc->ptr; p <= (char *)alloc->ptr + size - PTRSIZE; ++p) {
				gc_mark_alloc(*(void **)p);
			}
		}
	}
}

static void gc_mark_stack() {
	void *tos = __builtin_frame_address(0);
	void *bos = gc->bos;
	/* The stack grows towards smaller memory addresses, hence we scan tos->bos.
	 * Stop scanning once the distance between tos & bos is too small to hold a valid pointer */
	for (char *p = (char *)tos; p <= (char *)bos - PTRSIZE; ++p) {
		gc_mark_alloc(*(void **)p);
	}
}

static void gc_mark_roots() {
	for (size_t i = 0; i < gc->allocs->capacity; ++i) {
		gc_allocation_t *chunk = gc->allocs->allocs[i];
		while (chunk) {
			if (chunk->tag & GC_TAG_ROOT) {
				gc_mark_alloc(chunk->ptr);
			}
			chunk = chunk->next;
		}
	}
}

static void gc_mark() {
	/* Note: We only look at the stack and the heap, and ignore BSS. */
	/* Scan the heap for roots */
	gc_mark_roots();
	/* Dump registers onto stack and scan the stack */
	void (*volatile _mark_stack)(void) = gc_mark_stack;
	jmp_buf ctx;
	memset(&ctx, 0, sizeof(jmp_buf));
	setjmp(ctx);
	_mark_stack();
}

static size_t gc_sweep() {
	size_t total = 0;
	for (size_t i = 0; i < gc->allocs->capacity; ++i) {
		gc_allocation_t *chunk = gc->allocs->allocs[i];
		gc_allocation_t *next = NULL;
		/* Iterate over separate chaining */
		while (chunk) {
			if (chunk->tag & GC_TAG_MARK) {
				/* unmark */
				chunk->tag &= ~GC_TAG_MARK;
				chunk = chunk->next;
			}
			else if (chunk->tag & GC_TAG_CUT) {
				chunk = chunk->next;
			}
			else {
				/* no reference to this chunk, hence delete it */
				total += chunk->size;
				free(chunk->ptr);
				/* and remove it from the bookkeeping */
				next = chunk->next;
				gc_allocation_t *cut = chunk->cut;
				gc_allocation_map_remove(gc->allocs, chunk->ptr, false);
				while (cut) {
					gc_allocation_t *next = cut->cut;
					gc_allocation_map_remove(gc->allocs, cut->ptr, false);
					cut = next;
				}
				chunk = next;
			}
		}
	}
	gc_allocation_map_resize_to_fit(gc->allocs);
	return total;
}

void *_gc_calloc(size_t count, size_t size) {
	return gc_allocate(count, size);
}

void *_gc_realloc(void *p, size_t size) {
	gc_allocation_t *alloc = gc_allocation_map_get(gc->allocs, p);
	if (p && !alloc) {
		// the user passed an unknown pointer
		return NULL;
	}
	void *q = realloc(p, size);
	if (!q) {
		// realloc failed but p is still valid
		return NULL;
	}
	if (!p) {
		// allocation, not reallocation
		gc_allocation_t *alloc = gc_allocation_map_put(gc->allocs, gc_allocation_new(q, size));
		return alloc->ptr;
	}
	if (p == q) {
		// successful reallocation w/o copy
		alloc->size = size;
	}
	else {
		// successful reallocation w/ copy
		gc_allocation_map_remove(gc->allocs, p, true);
		gc_allocation_map_put(gc->allocs, gc_allocation_new(q, size));
	}
	return q;
}

void _gc_free(void *ptr) {
	gc_allocation_t *alloc = gc_allocation_map_get(gc->allocs, ptr);
	if (alloc) {
		free(ptr);
		gc_allocation_map_remove(gc->allocs, ptr, true);
	}
}

void _gc_start(void *bos) {
	gc->paused = false;
	gc->bos = bos;
	// Create allocation map with no downsizing
	// Capacity must be a power of two
	gc->allocs = gc_allocation_map_new(1024 * 1024, 1024 * 1024, 0.5, 0.0, 0.8);
	// gc->roots = gc_allocation_map_new(1024, 1024, 0.5, 0.0, 0.8);
}

void _gc_pause() {
	gc->paused = true;
}

void _gc_resume() {
	gc->paused = false;
}

size_t _gc_stop() {
	size_t collected = gc_sweep();
	gc_allocation_map_delete(gc->allocs);
	return collected;
}

size_t _gc_run() {
	// double t = iron_time();
	gc_mark();
	size_t collected = gc_sweep();
	// iron_log("gc took %fms, freed %db.\n", (iron_time() - t) * 1000, collected);
	return collected;
}