|
|
@@ -29,9 +29,7 @@
|
|
|
#include "../api/timeAPI.h"
|
|
|
#include <stdio.h>
|
|
|
#include <new>
|
|
|
-
|
|
|
-// Get settings from here.
|
|
|
-#include "../settings.h"
|
|
|
+#include "simd.h"
|
|
|
|
|
|
#ifndef DISABLE_MULTI_THREADING
|
|
|
// Requires -pthread for linking
|
|
|
@@ -39,7 +37,133 @@
|
|
|
#include <mutex>
|
|
|
#endif
|
|
|
|
|
|
+#if defined(USE_MICROSOFT_WINDOWS)
|
|
|
+ #include <Windows.h> uint32_t getCacheLineSize() { return (uint32_t) (sizeof(void*) * 8); } // Approximation for ARM
|
|
|
+#elif defined(USE_MACOS)
|
|
|
+ #include <sys/sysctl.h>
|
|
|
+#elif defined(USE_LINUX)
|
|
|
+ #include <stdio.h>
|
|
|
+ #include <stdint.h>
|
|
|
+ #include <stdlib.h>
|
|
|
+#endif
|
|
|
+
|
|
|
+#ifdef SAFE_POINTER_CHECKS
|
|
|
+ #define DSR_PRINT_CACHE_LINE_SIZE
|
|
|
+ #define DSR_PRINT_NO_MEMORY_LEAK
|
|
|
+#endif
|
|
|
+
|
|
|
namespace dsr {
|
|
|
+ // The default cache line size is used when not known from asking the system.
|
|
|
+ static const uintptr_t defaultCacheLineSize = 128;
|
|
|
+ // There is no point in using a heap alignment smaller than the allocation heads, so we align to at least 64 bytes.
|
|
|
+ static const uintptr_t minimumHeapAlignment = 64;
|
|
|
+ #if defined(USE_LINUX)
|
|
|
+ static int32_t getCacheLineSizeFromIndices(int32_t cpuIndex, int32_t cacheLevel) {
|
|
|
+ char path[256];
|
|
|
+ snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%i/cache/index%i/coherency_line_size", (int)cpuIndex, (int)cacheLevel);
|
|
|
+ FILE *file = fopen(path, "r");
|
|
|
+ if (file == nullptr) {
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+ int cacheLineSize;
|
|
|
+ if (fscanf(file, "%i", &cacheLineSize) != 1) {
|
|
|
+ cacheLineSize = -1;
|
|
|
+ }
|
|
|
+ fclose(file);
|
|
|
+ return int32_t(cacheLineSize);
|
|
|
+ }
|
|
|
+ static int32_t getCacheLineSize() {
|
|
|
+ int32_t result = 0;
|
|
|
+ int32_t cpuIndex = 0;
|
|
|
+ int32_t cacheLevel = 0;
|
|
|
+ while (true) {
|
|
|
+ int32_t newSize = getCacheLineSizeFromIndices(cpuIndex, cacheLevel);
|
|
|
+ if (newSize == -1) {
|
|
|
+ if (cacheLevel == 0) {
|
|
|
+ // CPU does not exist, so we are done.
|
|
|
+ break;
|
|
|
+ } else {
|
|
|
+ // Cache level does not exist. Go to the next CPU.
|
|
|
+ cpuIndex++;
|
|
|
+ cacheLevel = 0;
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // We have found the cache line size for cpuIndex and cacheLevel, so we include it in a maximum.
|
|
|
+ //printf("CPU %i cache level %i is %i.\n", (int)cpuIndex, (int)cacheLevel, (int)newSize);
|
|
|
+ result = max(result, newSize);
|
|
|
+ cacheLevel++;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (result <= 0) {
|
|
|
+ result = defaultCacheLineSize;
|
|
|
+ printf("WARNING! Failed to read cache line size from Linux system folders. The application might not be thread-safe.\n");
|
|
|
+ }
|
|
|
+ #ifdef DSR_PRINT_CACHE_LINE_SIZE
|
|
|
+ printf("Detected a cache line width of %i bytes from reading Linux system folders.\n", (int)result);
|
|
|
+ #endif
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ #elif defined(USE_MACOS)
|
|
|
+ static int32_t getCacheLineSize() {
|
|
|
+ int32_t result = 0;
|
|
|
+ int cacheLine;
|
|
|
+ size_t size = sizeof(cacheLine);
|
|
|
+ int mib[2] = {CTL_HW, HW_CACHELINE};
|
|
|
+ int error = sysctl(mib, 2, &cacheLine, &size, NULL, 0);
|
|
|
+ if(error == 0) {
|
|
|
+ result = cacheLine;
|
|
|
+ #ifdef DSR_PRINT_CACHE_LINE_SIZE
|
|
|
+ printf("Detected a cache line width of %i bytes on MacOS by asking for HW_CACHELINE with sysctl.\n", (int)result);
|
|
|
+ #endif
|
|
|
+ } else if(error == EFAULT) {
|
|
|
+ printf("WARNING! Failed to read HW_CACHELINE on MacOS because of a bad address error. The application might not be thread-safe.\n");
|
|
|
+ } else if(error == EINVAL) {
|
|
|
+ printf("WARNING! Failed to read HW_CACHELINE on MacOS because of a bad argument error. The application might not be thread-safe.\n");
|
|
|
+ } else if(error == ENOMEM) {
|
|
|
+ printf("WARNING! Failed to read HW_CACHELINE on MacOS because of an out of memory error. The application might not be thread-safe.\n");
|
|
|
+ } else if(error == ENOTDIR) {
|
|
|
+ printf("WARNING! Failed to read HW_CACHELINE on MacOS because of a not a directory error. The application might not be thread-safe.\n");
|
|
|
+ } else if(error == EISDIR) {
|
|
|
+ printf("WARNING! Failed to read HW_CACHELINE on MacOS because of a directory error. The application might not be thread-safe.\n");
|
|
|
+ } else if(error == EPERM) {
|
|
|
+ printf("WARNING! Failed to read HW_CACHELINE on MacOS because of a permission error. The application might not be thread-safe.\n");
|
|
|
+ } else {
|
|
|
+ result = defaultCacheLineSize;
|
|
|
+ printf("WARNING! Failed to read HW_CACHELINE on MacOS with an unknown error. The application might not be thread-safe.\n");
|
|
|
+ }
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ /*
|
|
|
+ #elif defined(USE_MICROSOFT_WINDOWS)
|
|
|
+ static int32_t getCacheLineSize() {
|
|
|
+ int32_t result = ?; // TODO: Implement
|
|
|
+ return result;
|
|
|
+ }*/
|
|
|
+ #else
|
|
|
+ static int32_t getCacheLineSize() {
|
|
|
+ int32_t result = 0;
|
|
|
+ result = defaultCacheLineSize;
|
|
|
+ printf("WARNING! The target platform does not have a method for detecting cache line width in DFPSR/base/heap.cpp.\n");
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ #endif
|
|
|
+
|
|
|
+ static uintptr_t heapAlignmentAndMask = 0u;
|
|
|
+ uintptr_t heap_getHeapAlignment() {
|
|
|
+ static uintptr_t heapAlignment = 0u;
|
|
|
+ if (heapAlignment == 0) {
|
|
|
+ heapAlignment = getCacheLineSize();
|
|
|
+ if (heapAlignment < sizeof(DSR_FLOAT_VECTOR_SIZE)) heapAlignment = sizeof(DSR_FLOAT_VECTOR_SIZE);
|
|
|
+ if (heapAlignment < minimumHeapAlignment) heapAlignment = minimumHeapAlignment;
|
|
|
+ heapAlignmentAndMask = memory_createAlignmentAndMask(heapAlignment);
|
|
|
+ }
|
|
|
+ return heapAlignment;
|
|
|
+ }
|
|
|
+ static uintptr_t heap_getHeapAlignmentAndMask() {
|
|
|
+ heap_getHeapAlignment();
|
|
|
+ return heapAlignmentAndMask;
|
|
|
+ }
|
|
|
+
|
|
|
// Keep track of the program's state.
|
|
|
enum class ProgramState {
|
|
|
Starting, // A single thread does global construction without using any mutex.
|
|
|
@@ -58,9 +182,9 @@ namespace dsr {
|
|
|
threadCount++;
|
|
|
if (threadCount > 1 && programState != ProgramState::Running) {
|
|
|
if (programState == ProgramState::Starting) {
|
|
|
- throwError(U"Tried to create another thread before construction of global variables was complete!\n");
|
|
|
+ printf("Tried to create another thread before construction of global variables was complete!\n");
|
|
|
} else if (programState == ProgramState::Terminating) {
|
|
|
- throwError(U"Tried to create another thread after destruction of global variables had begun!\n");
|
|
|
+ printf("Tried to create another thread after destruction of global variables had begun!\n");
|
|
|
}
|
|
|
}
|
|
|
threadLock.unlock();
|
|
|
@@ -140,19 +264,22 @@ namespace dsr {
|
|
|
using HeapFlag = uint16_t;
|
|
|
using BinIndex = uint16_t;
|
|
|
|
|
|
- // The framework's maximum memory alignment is either the largest float SIMD vector or the thread safe alignment.
|
|
|
- static const uintptr_t heapAlignment = DSR_MAXIMUM_ALIGNMENT;
|
|
|
- static const uintptr_t heapAlignmentAndMask = memory_createAlignmentAndMask(heapAlignment);
|
|
|
-
|
|
|
// The free function is hidden, because all allocations are forced to use reference counting,
|
|
|
// so that a hard exit can disable recursive calls to heap_free by incrementing all reference counters.
|
|
|
static void heap_free(void * const allocation);
|
|
|
|
|
|
+ // The bin size equals two to the power of binIndex multiplied by minimumHeapAlignment.
|
|
|
+ // This allow knowing the number of bins in compile time by leaving a number of unused bins specified by MIN_BIN_COUNT.
|
|
|
+ inline constexpr uintptr_t getBinSize(BinIndex binIndex) {
|
|
|
+ // Index 0 starts at the minimum alignment to know the number of bins in compile time, so some bins will be unused when the alignment is bigger.
|
|
|
+ return (uintptr_t(1) << uintptr_t(binIndex)) * minimumHeapAlignment;
|
|
|
+ }
|
|
|
+
|
|
|
// Calculates the largest power of two allocation size that does not overflow a pointer on the target platform.
|
|
|
constexpr int calculateBinCount() {
|
|
|
intptr_t p = 0;
|
|
|
while (true) {
|
|
|
- uintptr_t result = ((uintptr_t)1 << p) * heapAlignment;
|
|
|
+ uintptr_t result = getBinSize(p);
|
|
|
// Once the value overflows in uintptr_t we have found the index that can not be used, which is also the bin count when including index zero.
|
|
|
if (result == 0) {
|
|
|
return p;
|
|
|
@@ -160,14 +287,12 @@ namespace dsr {
|
|
|
p++;
|
|
|
}
|
|
|
}
|
|
|
- static const int MAX_BIN_COUNT = calculateBinCount();
|
|
|
|
|
|
- inline uintptr_t getBinSize(BinIndex index) {
|
|
|
- return (((uintptr_t)1) << ((uintptr_t)index)) * heapAlignment;
|
|
|
- }
|
|
|
+ // The index of the last used bin.
|
|
|
+ static const int MAX_BIN_COUNT = calculateBinCount();
|
|
|
|
|
|
- static BinIndex getBinIndex(uintptr_t minimumSize) {
|
|
|
- for (intptr_t p = 0; p < MAX_BIN_COUNT; p++) {
|
|
|
+ static BinIndex getBinIndex(uintptr_t minimumSize, intptr_t minimumBin) {
|
|
|
+ for (intptr_t p = minimumBin; p < MAX_BIN_COUNT; p++) {
|
|
|
uintptr_t result = getBinSize(p);
|
|
|
if (result >= minimumSize) {
|
|
|
return p;
|
|
|
@@ -177,6 +302,9 @@ namespace dsr {
|
|
|
return -1;
|
|
|
}
|
|
|
|
|
|
+ // The index of the first used bin, which is also the number of unused bins.
|
|
|
+ static const int MIN_BIN_COUNT = getBinIndex(heap_getHeapAlignment(), 0);
|
|
|
+
|
|
|
static const HeapFlag heapFlag_recycled = 1 << 0;
|
|
|
struct HeapHeader : public AllocationHeader {
|
|
|
// Because nextRecycled and usedSize have mutually exclusive lifetimes, they can share memory location.
|
|
|
@@ -235,21 +363,29 @@ namespace dsr {
|
|
|
this->flags &= ~heapFlag_recycled;
|
|
|
}
|
|
|
};
|
|
|
- static const uintptr_t heapHeaderPaddedSize = memory_getPaddedSize(sizeof(HeapHeader), heapAlignment);
|
|
|
+
|
|
|
+ // TODO: Allow using the header directly for manipulation in the API, now that the offset is not known in compile time.
|
|
|
+ inline uintptr_t heap_getHeapHeaderPaddedSize() {
|
|
|
+ static uintptr_t heapHeaderPaddedSize = 0;
|
|
|
+ if (heapHeaderPaddedSize == 0) {
|
|
|
+ heapHeaderPaddedSize = memory_getPaddedSize(sizeof(HeapHeader), heap_getHeapAlignment());
|
|
|
+ }
|
|
|
+ return heapHeaderPaddedSize;
|
|
|
+ }
|
|
|
|
|
|
AllocationHeader *heap_getHeader(void * const allocation) {
|
|
|
- return (AllocationHeader*)((uint8_t*)allocation - heapHeaderPaddedSize);
|
|
|
+ return (AllocationHeader*)((uint8_t*)allocation - heap_getHeapHeaderPaddedSize());
|
|
|
}
|
|
|
|
|
|
inline HeapHeader *headerFromAllocation(void const * const allocation) {
|
|
|
- return (HeapHeader *)((uint8_t*)allocation - heapHeaderPaddedSize);
|
|
|
+ return (HeapHeader *)((uint8_t*)allocation - heap_getHeapHeaderPaddedSize());
|
|
|
}
|
|
|
|
|
|
inline void *allocationFromHeader(HeapHeader * const header) {
|
|
|
- return ((uint8_t*)header) + heapHeaderPaddedSize;
|
|
|
+ return ((uint8_t*)header) + heap_getHeapHeaderPaddedSize();
|
|
|
}
|
|
|
inline void const *allocationFromHeader(HeapHeader const * const header) {
|
|
|
- return ((uint8_t const *)header) + heapHeaderPaddedSize;
|
|
|
+ return ((uint8_t const *)header) + heap_getHeapHeaderPaddedSize();
|
|
|
}
|
|
|
|
|
|
#ifdef SAFE_POINTER_CHECKS
|
|
|
@@ -274,7 +410,7 @@ namespace dsr {
|
|
|
return 0;
|
|
|
} else {
|
|
|
HeapHeader *header = headerFromAllocation(allocation);
|
|
|
- return memory_getPaddedSize_usingAndMask(header->getUsedSize(), heapAlignmentAndMask);
|
|
|
+ return memory_getPaddedSize_usingAndMask(header->getUsedSize(), heap_getHeapAlignmentAndMask());
|
|
|
}
|
|
|
}
|
|
|
#endif
|
|
|
@@ -316,7 +452,7 @@ namespace dsr {
|
|
|
HeapHeader *header = headerFromAllocation(allocation);
|
|
|
lockMemory();
|
|
|
if (header->useCount == 0) {
|
|
|
- throwError(U"Heap error: Decreasing a count that is already zero!\n");
|
|
|
+ printf("Heap error: Decreasing a count that is already zero!\n");
|
|
|
} else {
|
|
|
header->useCount--;
|
|
|
if (header->useCount == 0) {
|
|
|
@@ -359,7 +495,7 @@ namespace dsr {
|
|
|
void cleanUp() {
|
|
|
// If memory safety checks are enabled, then we should indicate that everything is fine with the memory once cleaning up.
|
|
|
// There is however no way to distinguish between leaking memory and not yet having terminated everything, so there is no leak warning to print.
|
|
|
- #ifdef SAFE_POINTER_CHECKS
|
|
|
+ #ifdef DSR_PRINT_NO_MEMORY_LEAK
|
|
|
// Can't allocate more memory after freeing all memory.
|
|
|
printf("All heap memory was freed without leaks.\n");
|
|
|
#endif
|
|
|
@@ -375,7 +511,7 @@ namespace dsr {
|
|
|
~HeapPool() {
|
|
|
// The destruction should be ignored to manually terminate once all memory allocations have been freed.
|
|
|
if (programState != ProgramState::Terminating) {
|
|
|
- throwError(U"Heap error: Terminated the application without first calling heap_terminatingApplication or heap_hardExitCleaning!\n");
|
|
|
+ printf("Heap error: Terminated the application without first calling heap_terminatingApplication or heap_hardExitCleaning!\n");
|
|
|
}
|
|
|
}
|
|
|
};
|
|
|
@@ -383,7 +519,7 @@ namespace dsr {
|
|
|
static UnsafeAllocation tryToAllocate(HeapMemory &heap, uintptr_t paddedSize, uintptr_t alignmentAndMask) {
|
|
|
UnsafeAllocation result(nullptr, nullptr);
|
|
|
uint8_t *dataPointer = (uint8_t*)(((uintptr_t)(heap.allocationPointer) - paddedSize) & alignmentAndMask);
|
|
|
- AllocationHeader *headerPointer = (AllocationHeader*)(dataPointer - heapHeaderPaddedSize);
|
|
|
+ AllocationHeader *headerPointer = (AllocationHeader*)(dataPointer - heap_getHeapHeaderPaddedSize());
|
|
|
if ((uint8_t*)headerPointer >= heap.top) {
|
|
|
// There is enough space, so confirm the allocation.
|
|
|
result = UnsafeAllocation(dataPointer, headerPointer);
|
|
|
@@ -399,7 +535,7 @@ namespace dsr {
|
|
|
// Start with the most recent heap, which is most likely to have enough space.
|
|
|
HeapMemory *currentHeap = pool.lastHeap;
|
|
|
while (currentHeap != nullptr) {
|
|
|
- UnsafeAllocation result = tryToAllocate(*currentHeap, paddedSize, heapAlignmentAndMask);
|
|
|
+ UnsafeAllocation result = tryToAllocate(*currentHeap, paddedSize, alignmentAndMask);
|
|
|
if (result.data != nullptr) {
|
|
|
return result;
|
|
|
}
|
|
|
@@ -414,19 +550,19 @@ namespace dsr {
|
|
|
pool.lastHeap = new HeapMemory(allocationSize);
|
|
|
pool.lastHeap->prevHeap = previousHeap;
|
|
|
// Make one last attempt at allocating the memory.
|
|
|
- return tryToAllocate(*(pool.lastHeap), paddedSize, heapAlignmentAndMask);
|
|
|
+ return tryToAllocate(*(pool.lastHeap), paddedSize, alignmentAndMask);
|
|
|
}
|
|
|
|
|
|
static HeapPool defaultHeap;
|
|
|
|
|
|
UnsafeAllocation heap_allocate(uintptr_t minimumSize, bool zeroed) {
|
|
|
UnsafeAllocation result(nullptr, nullptr);
|
|
|
- int32_t binIndex = getBinIndex(minimumSize);
|
|
|
+ int32_t binIndex = getBinIndex(minimumSize, MIN_BIN_COUNT);
|
|
|
if (binIndex == -1) {
|
|
|
// If the requested allocation is so big that there is no power of two that can contain it without overflowing the address space, then it can not be allocated.
|
|
|
- throwError(U"Heap error: Exceeded the maximum size when trying to allocate!\n");
|
|
|
+ printf("Heap error: Exceeded the maximum size when trying to allocate!\n");
|
|
|
} else {
|
|
|
- uintptr_t paddedSize = ((uintptr_t)1 << binIndex) * heapAlignment;
|
|
|
+ uintptr_t paddedSize = getBinSize(binIndex);
|
|
|
lockMemory();
|
|
|
allocationCount++;
|
|
|
// Look for pre-existing allocations in the recycling bins.
|
|
|
@@ -442,9 +578,9 @@ namespace dsr {
|
|
|
result = UnsafeAllocation((uint8_t*)allocationFromHeader(binHeader), binHeader);
|
|
|
} else {
|
|
|
// Look for a heap with enough space for a new allocation.
|
|
|
- result = tryToAllocate(defaultHeap, paddedSize, heapAlignmentAndMask);
|
|
|
+ result = tryToAllocate(defaultHeap, paddedSize, heap_getHeapAlignmentAndMask());
|
|
|
if (result.data == nullptr) {
|
|
|
- throwError(U"Heap error: Failed to allocate more memory!\n");
|
|
|
+ printf("Heap error: Failed to allocate more memory!\n");
|
|
|
}
|
|
|
}
|
|
|
unlockMemory();
|
|
|
@@ -473,7 +609,7 @@ namespace dsr {
|
|
|
// Get the recycled allocation's header.
|
|
|
HeapHeader *header = headerFromAllocation(allocation);
|
|
|
if (header->isRecycled()) {
|
|
|
- throwError(U"Heap error: A heap allocation was freed twice!\n");
|
|
|
+ printf("Heap error: A heap allocation was freed twice!\n");
|
|
|
} else {
|
|
|
// Call the destructor provided with any external resource that also needs to be freed.
|
|
|
if (header->destructor.destructor) {
|
|
|
@@ -483,7 +619,7 @@ namespace dsr {
|
|
|
header->destructor = HeapDestructor();
|
|
|
int binIndex = header->binIndex;
|
|
|
if (binIndex >= MAX_BIN_COUNT) {
|
|
|
- throwError(U"Heap error: Out of bound recycling bin index in corrupted head of freed allocation!\n");
|
|
|
+ printf("Heap error: Out of bound recycling bin index in corrupted head of freed allocation!\n");
|
|
|
} else {
|
|
|
// Make any previous head from the bin into the new tail.
|
|
|
HeapHeader *oldHeader = defaultHeap.recyclingBin[binIndex];
|
|
|
@@ -541,11 +677,11 @@ namespace dsr {
|
|
|
}
|
|
|
|
|
|
void impl_throwAllocationFailure() {
|
|
|
- throwError(U"Failed to allocate memory for a new object!\n");
|
|
|
+ string_sendMessage(U"Failed to allocate memory for a new object!\n", MessageType::Error);
|
|
|
}
|
|
|
|
|
|
void impl_throwNullException() {
|
|
|
- throwError(U"Null handle exception!\n");
|
|
|
+ string_sendMessage(U"Null handle exception!\n", MessageType::Error);
|
|
|
}
|
|
|
|
|
|
void impl_throwIdentityMismatch(uint64_t allocationIdentity, uint64_t pointerIdentity) {
|