|
|
@@ -3,426 +3,793 @@ package runtime
|
|
|
import "core:intrinsics"
|
|
|
_ :: intrinsics
|
|
|
|
|
|
-INITIAL_MAP_CAP :: 16
|
|
|
+// High performance, cache-friendly, open-addressed Robin Hood hashing hash map
|
|
|
+// data structure with various optimizations for Odin.
|
|
|
+//
|
|
|
+// Copyright 2022 (c) Dale Weiler
|
|
|
+//
|
|
|
+// The core of the hash map data structure is the Raw_Map struct which is a
|
|
|
+// type-erased representation of the map. This type-erased representation is
|
|
|
+// used in two ways: static and dynamic. When static type information is known,
|
|
|
+// the procedures suffixed with _static should be used instead of _dynamic. The
|
|
|
+// static procedures are optimized since they have type information. Hashing of
|
|
|
+// keys, comparison of keys, and data lookup are all optimized. When type
|
|
|
+// information is not known, the procedures suffixed with _dynamic should be
|
|
|
+// used. The representation of the map is the same for both static and dynamic,
|
|
|
+// and procedures of each can be mixed and matched. The purpose of the dynamic
|
|
|
+// representation is to enable reflection and runtime manipulation of the map.
|
|
|
+// The dynamic procedures all take an additional Map_Info structure parameter
|
|
|
+// which carries runtime values describing the size, alignment, and offset of
|
|
|
+// various traits of a given key and value type pair. The Map_Info value can
|
|
|
+// be created by calling map_info(K, V) with the key and value typeids.
|
|
|
+//
|
|
|
+// This map implementation makes extensive use of uintptr for representing
|
|
|
+// sizes, lengths, capacities, masks, pointers, offsets, and addresses to avoid
|
|
|
+// expensive sign extension and masking that would be generated if types were
|
|
|
+// casted all over. The only place regular ints show up is in the cap() and
|
|
|
+// len() implementations.
|
|
|
+//
|
|
|
+// To make this map cache-friendly it uses a novel strategy to ensure keys and
|
|
|
+// values of the map are always cache-line aligned and that no single key or
|
|
|
+// value of any type ever straddles a cache-line. This cache efficiency makes
|
|
|
+// for quick lookups because the linear-probe always addresses data in a cache
|
|
|
+// friendly way. This is enabled through the use of a special meta-type called
|
|
|
+// a Map_Cell which packs as many values of a given type into a local array adding
|
|
|
+// internal padding to round to MAP_CACHE_LINE_SIZE. One other benefit to storing
|
|
|
+// the internal data in this manner is false sharing no longer occurs when using
|
|
|
+// a map, enabling efficient concurrent access of the map data structure with
|
|
|
+// minimal locking if desired.
|
|
|
+
|
|
|
+// With Robin Hood hashing a maximum load factor of 75% is ideal.
|
|
|
+MAP_LOAD_FACTOR :: 75
|
|
|
+
|
|
|
+// Minimum log2 capacity.
|
|
|
+MAP_MIN_LOG2_CAPACITY :: 6 // 64 elements
|
|
|
+
|
|
|
+// Has to be less than 100% though.
|
|
|
+#assert(MAP_LOAD_FACTOR < 100)
|
|
|
+
|
|
|
+// This is safe to change. The log2 size of a cache-line. At minimum it has to
|
|
|
+// be six though. Higher cache line sizes are permitted.
|
|
|
+MAP_CACHE_LINE_LOG2 :: 6
|
|
|
+
|
|
|
+// The size of a cache-line.
|
|
|
+MAP_CACHE_LINE_SIZE :: 1 << MAP_CACHE_LINE_LOG2
|
|
|
+
|
|
|
+// The minimum cache-line size allowed by this implementation is 64 bytes since
|
|
|
+// we need 6 bits in the base pointer to store the integer log2 capacity, which
|
|
|
+// at maximum is 63. Odin uses signed integers to represent length and capacity,
|
|
|
+// so only 63 bits are needed in the maximum case.
|
|
|
+#assert(MAP_CACHE_LINE_SIZE >= 64)
|
|
|
+
|
|
|
+// Map_Cell type that packs multiple T in such a way to ensure that each T stays
|
|
|
+// aligned by align_of(T) and such that align_of(Map_Cell(T)) % MAP_CACHE_LINE_SIZE == 0
|
|
|
+//
|
|
|
+// This means a value of type T will never straddle a cache-line.
|
|
|
+//
|
|
|
+// When multiple Ts can fit in a single cache-line the data array will have more
|
|
|
+// than one element. When it cannot, the data array will have one element and
|
|
|
+// an array of Map_Cell(T) will be padded to stay a multiple of MAP_CACHE_LINE_SIZE.
|
|
|
+//
|
|
|
+// We rely on the type system to do all the arithmetic and padding for us here.
|
|
|
+//
|
|
|
+// The usual array[index] indexing for []T backed by a []Map_Cell(T) becomes a bit
|
|
|
+// more involved as there now may be internal padding. The indexing now becomes
|
|
|
+//
|
|
|
+// N :: len(Map_Cell(T){}.data)
|
|
|
+// i := index / N
|
|
|
+// j := index % N
|
|
|
+// cell[i].data[j]
|
|
|
+//
|
|
|
+// However, since len(Map_Cell(T){}.data) is a compile-time constant, there are some
|
|
|
+// optimizations we can do to eliminate the need for any divisions as N will
|
|
|
+// be bounded by [1, 64).
|
|
|
+//
|
|
|
+// In the optimal case, len(Map_Cell(T){}.data) = 1 so the cell array can be treated
|
|
|
+// as a regular array of T, which is the case for hashes.
|
|
|
+Map_Cell :: struct($T: typeid) #align MAP_CACHE_LINE_SIZE {
|
|
|
+ data: [MAP_CACHE_LINE_SIZE / size_of(T) when 0 < size_of(T) && size_of(T) < MAP_CACHE_LINE_SIZE else 1]T,
|
|
|
+}
|
|
|
+
|
|
|
+// So we can operate on a cell data structure at runtime without any type
|
|
|
+// information, we have a simple table that stores some traits about the cell.
|
|
|
+//
|
|
|
+// 32-bytes on 64-bit
|
|
|
+// 16-bytes on 32-bit
|
|
|
+Map_Cell_Info :: struct {
|
|
|
+ size_of_type: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits
|
|
|
+ align_of_type: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits
|
|
|
+ size_of_cell: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits
|
|
|
+ elements_per_cell: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits
|
|
|
+}
|
|
|
+
|
|
|
+// map_cell_info :: proc "contextless" ($T: typeid) -> ^Map_Cell_Info {...}
|
|
|
+map_cell_info :: intrinsics.type_map_cell_info
|
|
|
+
|
|
|
+// Same as the above procedure but at runtime with the cell Map_Cell_Info value.
|
|
|
+@(require_results)
|
|
|
+map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, index: uintptr) -> uintptr {
|
|
|
+ // Micro-optimize the common cases to save on integer division.
|
|
|
+ elements_per_cell := uintptr(info.elements_per_cell)
|
|
|
+ size_of_cell := uintptr(info.size_of_cell)
|
|
|
+ switch elements_per_cell {
|
|
|
+ case 1:
|
|
|
+ return base + (index * size_of_cell)
|
|
|
+ case 2:
|
|
|
+ cell_index := index >> 1
|
|
|
+ data_index := index & 1
|
|
|
+ size_of_type := uintptr(info.size_of_type)
|
|
|
+ return base + (cell_index * size_of_cell) + (data_index * size_of_type)
|
|
|
+ case:
|
|
|
+ cell_index := index / elements_per_cell
|
|
|
+ data_index := index % elements_per_cell
|
|
|
+ size_of_type := uintptr(info.size_of_type)
|
|
|
+ return base + (cell_index * size_of_cell) + (data_index * size_of_type)
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
-// Temporary data structure for comparing hashes and keys
|
|
|
-Map_Hash :: struct {
|
|
|
- hash: uintptr,
|
|
|
- key_ptr: rawptr, // address of Map_Entry_Header.key
|
|
|
+// Same as above procedure but with compile-time constant index.
|
|
|
+@(require_results)
|
|
|
+map_cell_index_dynamic_const :: proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, $INDEX: uintptr) -> uintptr {
|
|
|
+ elements_per_cell := uintptr(info.elements_per_cell)
|
|
|
+ size_of_cell := uintptr(info.size_of_cell)
|
|
|
+ size_of_type := uintptr(info.size_of_type)
|
|
|
+ cell_index := INDEX / elements_per_cell
|
|
|
+ data_index := INDEX % elements_per_cell
|
|
|
+ return base + (cell_index * size_of_cell) + (data_index * size_of_type)
|
|
|
+}
|
|
|
+
|
|
|
+// We always round the capacity to a power of two so this becomes [16]Foo, which
|
|
|
+// works out to [4]Cell(Foo).
|
|
|
+//
|
|
|
+// The following compile-time procedure indexes such a [N]Cell(T) structure as
|
|
|
+// if it were a flat array accounting for the internal padding introduced by the
|
|
|
+// Cell structure.
|
|
|
+@(require_results)
|
|
|
+map_cell_index_static :: #force_inline proc "contextless" (cells: [^]Map_Cell($T), index: uintptr) -> ^T #no_bounds_check {
|
|
|
+ N :: size_of(Map_Cell(T){}.data) / size_of(T) when size_of(T) > 0 else 1
|
|
|
+
|
|
|
+ #assert(N <= MAP_CACHE_LINE_SIZE)
|
|
|
+
|
|
|
+ when size_of(Map_Cell(T)) == size_of([N]T) {
|
|
|
+ // No padding case, can treat as a regular array of []T.
|
|
|
+
|
|
|
+ return &([^]T)(cells)[index]
|
|
|
+ } else when (N & (N - 1)) == 0 && N <= 8*size_of(uintptr) {
|
|
|
+ // Likely case, N is a power of two because T is a power of two.
|
|
|
+
|
|
|
+ // Compute the integer log 2 of N, this is the shift amount to index the
|
|
|
+ // correct cell. Odin's intrinsics.count_leading_zeros does not produce a
|
|
|
+ // constant, hence this approach. We only need to check up to N = 64.
|
|
|
+ SHIFT :: 1 when N < 2 else
|
|
|
+ 2 when N < 4 else
|
|
|
+ 3 when N < 8 else
|
|
|
+ 4 when N < 16 else
|
|
|
+ 5 when N < 32 else 6
|
|
|
+ #assert(SHIFT <= MAP_CACHE_LINE_LOG2)
|
|
|
+ // Unique case, no need to index data here since only one element.
|
|
|
+ when N == 1 {
|
|
|
+ return &cells[index >> SHIFT].data[0]
|
|
|
+ } else {
|
|
|
+ return &cells[index >> SHIFT].data[index & (N - 1)]
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // Least likely (and worst case), we pay for a division operation but we
|
|
|
+ // assume the compiler does not actually generate a division. N will be in the
|
|
|
+ // range [1, CACHE_LINE_SIZE) and not a power of two.
|
|
|
+ return &cells[index / N].data[index % N]
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
-__get_map_key_hash :: #force_inline proc "contextless" (k: ^$K) -> uintptr {
|
|
|
- hasher := intrinsics.type_hasher_proc(K)
|
|
|
- return hasher(k, 0)
|
|
|
+// len() for map
|
|
|
+@(require_results)
|
|
|
+map_len :: #force_inline proc "contextless" (m: Raw_Map) -> int {
|
|
|
+ return m.len
|
|
|
}
|
|
|
|
|
|
-__get_map_entry_key_ptr :: #force_inline proc "contextless" (h: Map_Header_Table, entry: ^Map_Entry_Header) -> rawptr {
|
|
|
- return rawptr(uintptr(entry) + h.key_offset)
|
|
|
+// cap() for map
|
|
|
+@(require_results)
|
|
|
+map_cap :: #force_inline proc "contextless" (m: Raw_Map) -> int {
|
|
|
+ // The data uintptr stores the capacity in the lower six bits which gives the
|
|
|
+ // a maximum value of 2^6-1, or 63. We store the integer log2 of capacity
|
|
|
+ // since our capacity is always a power of two. We only need 63 bits as Odin
|
|
|
+ // represents length and capacity as a signed integer.
|
|
|
+ return 0 if m.data == 0 else 1 << map_log2_cap(m)
|
|
|
}
|
|
|
|
|
|
-Map_Index :: distinct uint
|
|
|
-MAP_SENTINEL :: ~Map_Index(0)
|
|
|
+// Query the load factor of the map. This is not actually configurable, but
|
|
|
+// some math is needed to compute it. Compute it as a fixed point percentage to
|
|
|
+// avoid floating point operations. This division can be optimized out by
|
|
|
+// multiplying by the multiplicative inverse of 100.
|
|
|
+@(require_results)
|
|
|
+map_load_factor :: #force_inline proc "contextless" (log2_capacity: uintptr) -> uintptr {
|
|
|
+ return ((uintptr(1) << log2_capacity) * MAP_LOAD_FACTOR) / 100
|
|
|
+}
|
|
|
|
|
|
-Map_Find_Result :: struct {
|
|
|
- hash_index: Map_Index,
|
|
|
- entry_prev: Map_Index,
|
|
|
- entry_index: Map_Index,
|
|
|
+@(require_results)
|
|
|
+map_resize_threshold :: #force_inline proc "contextless" (m: Raw_Map) -> int {
|
|
|
+ return int(map_load_factor(map_log2_cap(m)))
|
|
|
}
|
|
|
|
|
|
-Map_Entry_Header :: struct {
|
|
|
- hash: uintptr,
|
|
|
- next: Map_Index,
|
|
|
-/*
|
|
|
- key: Key_Value,
|
|
|
- value: Value_Type,
|
|
|
-*/
|
|
|
+// The data stores the log2 capacity in the lower six bits. This is primarily
|
|
|
+// used in the implementation rather than map_cap since the check for data = 0
|
|
|
+// isn't necessary in the implementation. cap() on the otherhand needs to work
|
|
|
+// when called on an empty map.
|
|
|
+@(require_results)
|
|
|
+map_log2_cap :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr {
|
|
|
+ return m.data & (64 - 1)
|
|
|
}
|
|
|
|
|
|
-Map_Header_Table :: struct {
|
|
|
- equal: Equal_Proc,
|
|
|
+// Canonicalize the data by removing the tagged capacity stored in the lower six
|
|
|
+// bits of the data uintptr.
|
|
|
+@(require_results)
|
|
|
+map_data :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr {
|
|
|
+ return m.data &~ uintptr(64 - 1)
|
|
|
+}
|
|
|
|
|
|
- entry_size: int,
|
|
|
- entry_align: int,
|
|
|
|
|
|
- key_offset: uintptr,
|
|
|
- key_size: int,
|
|
|
+Map_Hash :: uintptr
|
|
|
|
|
|
- value_offset: uintptr,
|
|
|
- value_size: int,
|
|
|
+// Procedure to check if a slot is empty for a given hash. This is represented
|
|
|
+// by the zero value to make the zero value useful. This is a procedure just
|
|
|
+// for prose reasons.
|
|
|
+@(require_results)
|
|
|
+map_hash_is_empty :: #force_inline proc "contextless" (hash: Map_Hash) -> bool {
|
|
|
+ return hash == 0
|
|
|
}
|
|
|
|
|
|
-Map_Header :: struct {
|
|
|
- m: ^Raw_Map,
|
|
|
- using table: Map_Header_Table,
|
|
|
+@(require_results)
|
|
|
+map_hash_is_deleted :: #force_no_inline proc "contextless" (hash: Map_Hash) -> bool {
|
|
|
+ // The MSB indicates a tombstone
|
|
|
+ N :: size_of(Map_Hash)*8 - 1
|
|
|
+ return hash >> N != 0
|
|
|
}
|
|
|
-
|
|
|
-// USED INTERNALLY BY THE COMPILER
|
|
|
-__dynamic_map_get :: proc "contextless" (m: rawptr, table: Map_Header_Table, key_hash: uintptr, key_ptr: rawptr) -> rawptr {
|
|
|
- if m != nil {
|
|
|
- h := Map_Header{(^Raw_Map)(m), table}
|
|
|
- index := __dynamic_map_find(h, key_hash, key_ptr).entry_index
|
|
|
- if index != MAP_SENTINEL {
|
|
|
- data := uintptr(__dynamic_map_get_entry(h, index))
|
|
|
- return rawptr(data + h.value_offset)
|
|
|
- }
|
|
|
- }
|
|
|
- return nil
|
|
|
+@(require_results)
|
|
|
+map_hash_is_valid :: #force_inline proc "contextless" (hash: Map_Hash) -> bool {
|
|
|
+ // The MSB indicates a tombstone
|
|
|
+ N :: size_of(Map_Hash)*8 - 1
|
|
|
+ return (hash != 0) & (hash >> N == 0)
|
|
|
}
|
|
|
|
|
|
-// USED INTERNALLY BY THE COMPILER
|
|
|
-__dynamic_map_set :: proc "odin" (m: rawptr, table: Map_Header_Table, key_hash: uintptr, key_ptr: rawptr, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check {
|
|
|
- add_entry :: proc "odin" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr, loc := #caller_location) -> Map_Index {
|
|
|
- prev := Map_Index(h.m.entries.len)
|
|
|
- c := Map_Index(__dynamic_array_append_nothing(&h.m.entries, h.entry_size, h.entry_align, loc))
|
|
|
- if c != prev {
|
|
|
- end := __dynamic_map_get_entry(h, c-1)
|
|
|
- end.hash = key_hash
|
|
|
- mem_copy(rawptr(uintptr(end) + h.key_offset), key_ptr, h.key_size)
|
|
|
- end.next = MAP_SENTINEL
|
|
|
- }
|
|
|
- return prev
|
|
|
- }
|
|
|
|
|
|
- h := Map_Header{(^Raw_Map)(m), table}
|
|
|
+// Computes the desired position in the array. This is just index % capacity,
|
|
|
+// but a procedure as there's some math involved here to recover the capacity.
|
|
|
+@(require_results)
|
|
|
+map_desired_position :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash) -> uintptr {
|
|
|
+ // We do not use map_cap since we know the capacity will not be zero here.
|
|
|
+ capacity := uintptr(1) << map_log2_cap(m)
|
|
|
+ return uintptr(hash & Map_Hash(capacity - 1))
|
|
|
+}
|
|
|
|
|
|
- index := MAP_SENTINEL
|
|
|
+@(require_results)
|
|
|
+map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash, slot: uintptr) -> uintptr {
|
|
|
+ // We do not use map_cap since we know the capacity will not be zero here.
|
|
|
+ capacity := uintptr(1) << map_log2_cap(m)
|
|
|
+ return (slot + capacity - map_desired_position(m, hash)) & (capacity - 1)
|
|
|
+}
|
|
|
|
|
|
- if len(h.m.hashes) == 0 {
|
|
|
- __dynamic_map_reserve(m, table, INITIAL_MAP_CAP, loc)
|
|
|
- __dynamic_map_grow(h, loc)
|
|
|
- }
|
|
|
+// When working with the type-erased structure at runtime we need information
|
|
|
+// about the map to make working with it possible. This info structure stores
|
|
|
+// that.
|
|
|
+//
|
|
|
+// `Map_Info` and `Map_Cell_Info` are read only data structures and cannot be
|
|
|
+// modified after creation
|
|
|
+//
|
|
|
+// 32-bytes on 64-bit
|
|
|
+// 16-bytes on 32-bit
|
|
|
+Map_Info :: struct {
|
|
|
+ ks: ^Map_Cell_Info, // 8-bytes on 64-bit, 4-bytes on 32-bit
|
|
|
+ vs: ^Map_Cell_Info, // 8-bytes on 64-bit, 4-bytes on 32-bit
|
|
|
+ key_hasher: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit
|
|
|
+ key_equal: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit
|
|
|
+}
|
|
|
|
|
|
- fr := __dynamic_map_find(h, key_hash, key_ptr)
|
|
|
- if fr.entry_index != MAP_SENTINEL {
|
|
|
- index = fr.entry_index
|
|
|
- } else {
|
|
|
- index = add_entry(h, key_hash, key_ptr, loc)
|
|
|
- if fr.entry_prev != MAP_SENTINEL {
|
|
|
- entry := __dynamic_map_get_entry(h, fr.entry_prev)
|
|
|
- entry.next = index
|
|
|
- } else if fr.hash_index != MAP_SENTINEL {
|
|
|
- h.m.hashes[fr.hash_index] = index
|
|
|
- } else {
|
|
|
- return nil
|
|
|
- }
|
|
|
- }
|
|
|
|
|
|
- e := __dynamic_map_get_entry(h, index)
|
|
|
- e.hash = key_hash
|
|
|
+// The Map_Info structure is basically a pseudo-table of information for a given K and V pair.
|
|
|
+// map_info :: proc "contextless" ($T: typeid/map[$K]$V) -> ^Map_Info {...}
|
|
|
+map_info :: intrinsics.type_map_info
|
|
|
|
|
|
- key := rawptr(uintptr(e) + h.key_offset)
|
|
|
- val := rawptr(uintptr(e) + h.value_offset)
|
|
|
+@(require_results)
|
|
|
+map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (ks: uintptr, vs: uintptr, hs: [^]Map_Hash, sk: uintptr, sv: uintptr) {
|
|
|
+ INFO_HS := intrinsics.type_map_cell_info(Map_Hash)
|
|
|
|
|
|
- mem_copy(key, key_ptr, h.key_size)
|
|
|
- mem_copy(val, value, h.value_size)
|
|
|
+ capacity := uintptr(1) << map_log2_cap(m)
|
|
|
+ ks = map_data(m)
|
|
|
+ vs = map_cell_index_dynamic(ks, info.ks, capacity) // Skip past ks to get start of vs
|
|
|
+ hs_ := map_cell_index_dynamic(vs, info.vs, capacity) // Skip past vs to get start of hs
|
|
|
+ sk = map_cell_index_dynamic(hs_, INFO_HS, capacity) // Skip past hs to get start of sk
|
|
|
+ // Need to skip past two elements in the scratch key space to get to the start
|
|
|
+ // of the scratch value space, of which there's only two elements as well.
|
|
|
+ sv = map_cell_index_dynamic_const(sk, info.ks, 2)
|
|
|
|
|
|
- if __dynamic_map_full(h) {
|
|
|
- __dynamic_map_grow(h, loc)
|
|
|
- }
|
|
|
+ hs = ([^]Map_Hash)(hs_)
|
|
|
+ return
|
|
|
+}
|
|
|
|
|
|
- return __dynamic_map_get_entry(h, index)
|
|
|
+@(require_results)
|
|
|
+map_kvh_data_values_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (vs: uintptr) {
|
|
|
+ capacity := uintptr(1) << map_log2_cap(m)
|
|
|
+ return map_cell_index_dynamic(map_data(m), info.ks, capacity) // Skip past ks to get start of vs
|
|
|
}
|
|
|
|
|
|
-// USED INTERNALLY BY THE COMPILER
|
|
|
-__dynamic_map_reserve :: proc "odin" (m: rawptr, table: Map_Header_Table, cap: uint, loc := #caller_location) {
|
|
|
- h := Map_Header{(^Raw_Map)(m), table}
|
|
|
|
|
|
- c := context
|
|
|
- if h.m.entries.allocator.procedure != nil {
|
|
|
- c.allocator = h.m.entries.allocator
|
|
|
+@(private, require_results)
|
|
|
+map_total_allocation_size :: #force_inline proc "contextless" (capacity: uintptr, info: ^Map_Info) -> uintptr {
|
|
|
+ round :: #force_inline proc "contextless" (value: uintptr) -> uintptr {
|
|
|
+ CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1
|
|
|
+ return (value + CACHE_MASK) &~ CACHE_MASK
|
|
|
+ }
|
|
|
+ INFO_HS := intrinsics.type_map_cell_info(Map_Hash)
|
|
|
+
|
|
|
+ size := uintptr(0)
|
|
|
+ size = round(map_cell_index_dynamic(size, info.ks, capacity))
|
|
|
+ size = round(map_cell_index_dynamic(size, info.vs, capacity))
|
|
|
+ size = round(map_cell_index_dynamic(size, INFO_HS, capacity))
|
|
|
+ size = round(map_cell_index_dynamic(size, info.ks, 2)) // Two additional ks for scratch storage
|
|
|
+ size = round(map_cell_index_dynamic(size, info.vs, 2)) // Two additional vs for scratch storage
|
|
|
+ return size
|
|
|
+}
|
|
|
+
|
|
|
+// The only procedure which needs access to the context is the one which allocates the map.
|
|
|
+@(require_results)
|
|
|
+map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator, loc := #caller_location) -> (result: Raw_Map, err: Allocator_Error) {
|
|
|
+ result.allocator = allocator // set the allocator always
|
|
|
+ if log2_capacity == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ if log2_capacity >= 64 {
|
|
|
+ // Overflowed, would be caused by log2_capacity > 64
|
|
|
+ return {}, .Out_Of_Memory
|
|
|
}
|
|
|
- context = c
|
|
|
|
|
|
- cap := cap
|
|
|
- cap = ceil_to_pow2(cap)
|
|
|
+ capacity := uintptr(1) << max(log2_capacity, MAP_MIN_LOG2_CAPACITY)
|
|
|
|
|
|
- __dynamic_array_reserve(&h.m.entries, h.entry_size, h.entry_align, int(cap), loc)
|
|
|
+ CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1
|
|
|
|
|
|
- if h.m.entries.len*2 < len(h.m.hashes) {
|
|
|
+ size := map_total_allocation_size(capacity, info)
|
|
|
+
|
|
|
+ data := mem_alloc_non_zeroed(int(size), MAP_CACHE_LINE_SIZE, allocator, loc) or_return
|
|
|
+ data_ptr := uintptr(raw_data(data))
|
|
|
+ if data_ptr == 0 {
|
|
|
+ err = .Out_Of_Memory
|
|
|
return
|
|
|
}
|
|
|
- if __slice_resize(&h.m.hashes, int(cap*2), h.m.entries.allocator, loc) {
|
|
|
- __dynamic_map_reset_entries(h, loc)
|
|
|
+ if intrinsics.expect(data_ptr & CACHE_MASK != 0, false) {
|
|
|
+ panic("allocation not aligned to a cache line", loc)
|
|
|
+ } else {
|
|
|
+ result.data = data_ptr | log2_capacity // Tagged pointer representation for capacity.
|
|
|
+ result.len = 0
|
|
|
+
|
|
|
+ map_clear_dynamic(&result, info)
|
|
|
}
|
|
|
+ return
|
|
|
}
|
|
|
|
|
|
+// This procedure has to stack allocate storage to store local keys during the
|
|
|
+// Robin Hood hashing technique where elements are swapped in the backing
|
|
|
+// arrays to reduce variance. This swapping can only be done with memcpy since
|
|
|
+// there is no type information.
|
|
|
+//
|
|
|
+// This procedure returns the address of the just inserted value.
|
|
|
+@(require_results)
|
|
|
+map_insert_hash_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) {
|
|
|
+ h := h
|
|
|
+ pos := map_desired_position(m^, h)
|
|
|
+ distance := uintptr(0)
|
|
|
+ mask := (uintptr(1) << map_log2_cap(m^)) - 1
|
|
|
|
|
|
-INITIAL_HASH_SEED :: 0xcbf29ce484222325
|
|
|
+ ks, vs, hs, sk, sv := map_kvh_data_dynamic(m^, info)
|
|
|
+
|
|
|
+ // Avoid redundant loads of these values
|
|
|
+ size_of_k := info.ks.size_of_type
|
|
|
+ size_of_v := info.vs.size_of_type
|
|
|
|
|
|
-_fnv64a :: proc "contextless" (data: []byte, seed: u64 = INITIAL_HASH_SEED) -> u64 {
|
|
|
- h: u64 = seed
|
|
|
- for b in data {
|
|
|
- h = (h ~ u64(b)) * 0x100000001b3
|
|
|
+ k := map_cell_index_dynamic(sk, info.ks, 0)
|
|
|
+ v := map_cell_index_dynamic(sv, info.vs, 0)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v)
|
|
|
+
|
|
|
+ // Temporary k and v dynamic storage for swap below
|
|
|
+ tk := map_cell_index_dynamic(sk, info.ks, 1)
|
|
|
+ tv := map_cell_index_dynamic(sv, info.vs, 1)
|
|
|
+
|
|
|
+
|
|
|
+ for {
|
|
|
+ hp := &hs[pos]
|
|
|
+ element_hash := hp^
|
|
|
+
|
|
|
+ if map_hash_is_empty(element_hash) {
|
|
|
+ k_dst := map_cell_index_dynamic(ks, info.ks, pos)
|
|
|
+ v_dst := map_cell_index_dynamic(vs, info.vs, pos)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v)
|
|
|
+ hp^ = h
|
|
|
+
|
|
|
+ return result if result != 0 else v_dst
|
|
|
+ }
|
|
|
+
|
|
|
+ if probe_distance := map_probe_distance(m^, element_hash, pos); distance > probe_distance {
|
|
|
+ if map_hash_is_deleted(element_hash) {
|
|
|
+ k_dst := map_cell_index_dynamic(ks, info.ks, pos)
|
|
|
+ v_dst := map_cell_index_dynamic(vs, info.vs, pos)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v)
|
|
|
+ hp^ = h
|
|
|
+
|
|
|
+ return result if result != 0 else v_dst
|
|
|
+ }
|
|
|
+
|
|
|
+ if result == 0 {
|
|
|
+ result = map_cell_index_dynamic(vs, info.vs, pos)
|
|
|
+ }
|
|
|
+
|
|
|
+ kp := map_cell_index_dynamic(ks, info.ks, pos)
|
|
|
+ vp := map_cell_index_dynamic(vs, info.vs, pos)
|
|
|
+
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(tk), rawptr(k), size_of_k)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(kp), size_of_k)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(kp), rawptr(tk), size_of_k)
|
|
|
+
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(tv), rawptr(v), size_of_v)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(vp), size_of_v)
|
|
|
+ intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(tv), size_of_v)
|
|
|
+
|
|
|
+ th := h
|
|
|
+ h = hp^
|
|
|
+ hp^ = th
|
|
|
+
|
|
|
+ distance = probe_distance
|
|
|
+ }
|
|
|
+
|
|
|
+ pos = (pos + 1) & mask
|
|
|
+ distance += 1
|
|
|
}
|
|
|
- return h
|
|
|
}
|
|
|
|
|
|
-default_hash :: #force_inline proc "contextless" (data: []byte) -> uintptr {
|
|
|
- return uintptr(_fnv64a(data))
|
|
|
-}
|
|
|
-default_hash_string :: #force_inline proc "contextless" (s: string) -> uintptr {
|
|
|
- return default_hash(transmute([]byte)(s))
|
|
|
-}
|
|
|
-default_hash_ptr :: #force_inline proc "contextless" (data: rawptr, size: int) -> uintptr {
|
|
|
- s := Raw_Slice{data, size}
|
|
|
- return default_hash(transmute([]byte)(s))
|
|
|
+@(require_results)
|
|
|
+map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error {
|
|
|
+ log2_capacity := map_log2_cap(m^)
|
|
|
+ new_capacity := uintptr(1) << max(log2_capacity + 1, MAP_MIN_LOG2_CAPACITY)
|
|
|
+ return map_reserve_dynamic(m, info, new_capacity, loc)
|
|
|
}
|
|
|
|
|
|
-@(private)
|
|
|
-_default_hasher_const :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, $N: uint) -> uintptr where N <= 16 {
|
|
|
- h := u64(seed) + 0xcbf29ce484222325
|
|
|
- p := uintptr(data)
|
|
|
- #unroll for _ in 0..<N {
|
|
|
- b := u64((^byte)(p)^)
|
|
|
- h = (h ~ b) * 0x100000001b3
|
|
|
- p += 1
|
|
|
- }
|
|
|
- return uintptr(h)
|
|
|
-}
|
|
|
|
|
|
-default_hasher_n :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, N: int) -> uintptr {
|
|
|
- h := u64(seed) + 0xcbf29ce484222325
|
|
|
- p := uintptr(data)
|
|
|
- for _ in 0..<N {
|
|
|
- b := u64((^byte)(p)^)
|
|
|
- h = (h ~ b) * 0x100000001b3
|
|
|
- p += 1
|
|
|
+@(require_results)
|
|
|
+map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr, loc := #caller_location) -> Allocator_Error {
|
|
|
+ @(require_results)
|
|
|
+ ceil_log2 :: #force_inline proc "contextless" (x: uintptr) -> uintptr {
|
|
|
+ z := intrinsics.count_leading_zeros(x)
|
|
|
+ if z > 0 && x & (x-1) != 0 {
|
|
|
+ z -= 1
|
|
|
+ }
|
|
|
+ return size_of(uintptr)*8 - 1 - z
|
|
|
}
|
|
|
- return uintptr(h)
|
|
|
-}
|
|
|
-
|
|
|
-// NOTE(bill): There are loads of predefined ones to improve optimizations for small types
|
|
|
-
|
|
|
-default_hasher1 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 1) }
|
|
|
-default_hasher2 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 2) }
|
|
|
-default_hasher3 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 3) }
|
|
|
-default_hasher4 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 4) }
|
|
|
-default_hasher5 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 5) }
|
|
|
-default_hasher6 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 6) }
|
|
|
-default_hasher7 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 7) }
|
|
|
-default_hasher8 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 8) }
|
|
|
-default_hasher9 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 9) }
|
|
|
-default_hasher10 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 10) }
|
|
|
-default_hasher11 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 11) }
|
|
|
-default_hasher12 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 12) }
|
|
|
-default_hasher13 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 13) }
|
|
|
-default_hasher14 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 14) }
|
|
|
-default_hasher15 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 15) }
|
|
|
-default_hasher16 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 16) }
|
|
|
|
|
|
-default_hasher_string :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr {
|
|
|
- h := u64(seed) + 0xcbf29ce484222325
|
|
|
- str := (^[]byte)(data)^
|
|
|
- for b in str {
|
|
|
- h = (h ~ u64(b)) * 0x100000001b3
|
|
|
+ if m.allocator.procedure == nil {
|
|
|
+ m.allocator = context.allocator
|
|
|
}
|
|
|
- return uintptr(h)
|
|
|
-}
|
|
|
-default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr {
|
|
|
- h := u64(seed) + 0xcbf29ce484222325
|
|
|
- ptr := (^uintptr)(data)^
|
|
|
- for (^byte)(ptr)^ != 0 {
|
|
|
- b := (^byte)(ptr)^
|
|
|
- h = (h ~ u64(b)) * 0x100000001b3
|
|
|
- ptr += 1
|
|
|
- }
|
|
|
- return uintptr(h)
|
|
|
-}
|
|
|
|
|
|
+ new_capacity := new_capacity
|
|
|
+ old_capacity := uintptr(map_cap(m^))
|
|
|
|
|
|
-__get_map_header :: proc "contextless" (m: ^$T/map[$K]$V) -> (header: Map_Header) {
|
|
|
- header.m = (^Raw_Map)(m)
|
|
|
- header.table = #force_inline __get_map_header_table(T)
|
|
|
- return
|
|
|
-}
|
|
|
+ if old_capacity >= new_capacity {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
|
|
|
-__get_map_header_runtime :: proc "contextless" (m: ^Raw_Map, ti: Type_Info_Map) -> (header: Map_Header) {
|
|
|
- header.m = m
|
|
|
- header.table = #force_inline __get_map_header_table_runtime(ti)
|
|
|
- return
|
|
|
-}
|
|
|
+ // ceiling nearest power of two
|
|
|
+ log2_new_capacity := ceil_log2(new_capacity)
|
|
|
|
|
|
-__get_map_header_table :: proc "contextless" ($T: typeid/map[$K]$V) -> (header: Map_Header_Table) {
|
|
|
- Entry :: struct {
|
|
|
- hash: uintptr,
|
|
|
- next: Map_Index,
|
|
|
- key: K,
|
|
|
- value: V,
|
|
|
+ log2_min_cap := max(MAP_MIN_LOG2_CAPACITY, log2_new_capacity)
|
|
|
+
|
|
|
+ if m.data == 0 {
|
|
|
+ m^ = map_alloc_dynamic(info, log2_min_cap, m.allocator, loc) or_return
|
|
|
+ return nil
|
|
|
}
|
|
|
|
|
|
- header.equal = intrinsics.type_equal_proc(K)
|
|
|
+ resized := map_alloc_dynamic(info, log2_min_cap, m.allocator, loc) or_return
|
|
|
|
|
|
- header.entry_size = size_of(Entry)
|
|
|
- header.entry_align = align_of(Entry)
|
|
|
+ ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info)
|
|
|
|
|
|
- header.key_offset = offset_of(Entry, key)
|
|
|
- header.key_size = size_of(K)
|
|
|
+ // Cache these loads to avoid hitting them in the for loop.
|
|
|
+ n := m.len
|
|
|
+ for i in 0..<old_capacity {
|
|
|
+ hash := hs[i]
|
|
|
+ if map_hash_is_empty(hash) {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if map_hash_is_deleted(hash) {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ k := map_cell_index_dynamic(ks, info.ks, i)
|
|
|
+ v := map_cell_index_dynamic(vs, info.vs, i)
|
|
|
+ _ = map_insert_hash_dynamic(&resized, info, hash, k, v)
|
|
|
+ // Only need to do this comparison on each actually added pair, so do not
|
|
|
+ // fold it into the for loop comparator as a micro-optimization.
|
|
|
+ n -= 1
|
|
|
+ if n == 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- header.value_offset = offset_of(Entry, value)
|
|
|
- header.value_size = size_of(V)
|
|
|
+ map_free_dynamic(m^, info, loc) or_return
|
|
|
|
|
|
- return
|
|
|
+ m.data = resized.data
|
|
|
+
|
|
|
+ return nil
|
|
|
}
|
|
|
|
|
|
-__get_map_header_table_runtime :: proc "contextless" (ti: Type_Info_Map) -> (header: Map_Header) {
|
|
|
- header.equal = ti.key_equal
|
|
|
|
|
|
- entries := ti.generated_struct.variant.(Type_Info_Struct).types[1]
|
|
|
- entry := entries.variant.(Type_Info_Dynamic_Array).elem
|
|
|
- e := entry.variant.(Type_Info_Struct)
|
|
|
+@(require_results)
|
|
|
+map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error {
|
|
|
+ if m.allocator.procedure == nil {
|
|
|
+ m.allocator = context.allocator
|
|
|
+ }
|
|
|
|
|
|
- header.entry_size = entry.size
|
|
|
- header.entry_align = entry.align
|
|
|
+ // Cannot shrink the capacity if the number of items in the map would exceed
|
|
|
+ // one minus the current log2 capacity's resize threshold. That is the shrunk
|
|
|
+ // map needs to be within the max load factor.
|
|
|
+ log2_capacity := map_log2_cap(m^)
|
|
|
+ if uintptr(m.len) >= map_load_factor(log2_capacity - 1) {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
|
|
|
- header.key_offset = e.offsets[2]
|
|
|
- header.key_size = e.types[2].size
|
|
|
+ shrunk := map_alloc_dynamic(info, log2_capacity - 1, m.allocator) or_return
|
|
|
|
|
|
- header.value_offset = e.offsets[3]
|
|
|
- header.value_size = e.types[3].size
|
|
|
+ capacity := uintptr(1) << log2_capacity
|
|
|
|
|
|
- return
|
|
|
-}
|
|
|
+ ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info)
|
|
|
|
|
|
+ n := m.len
|
|
|
+ for i in 0..<capacity {
|
|
|
+ hash := hs[i]
|
|
|
+ if map_hash_is_empty(hash) {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if map_hash_is_deleted(hash) {
|
|
|
+ continue
|
|
|
+ }
|
|
|
|
|
|
+ k := map_cell_index_dynamic(ks, info.ks, i)
|
|
|
+ v := map_cell_index_dynamic(vs, info.vs, i)
|
|
|
+ _ = map_insert_hash_dynamic(&shrunk, info, hash, k, v)
|
|
|
+ // Only need to do this comparison on each actually added pair, so do not
|
|
|
+ // fold it into the for loop comparator as a micro-optimization.
|
|
|
+ n -= 1
|
|
|
+ if n == 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
-__slice_resize :: proc "odin" (array_: ^$T/[]$E, new_count: int, allocator: Allocator, loc := #caller_location) -> bool {
|
|
|
- array := (^Raw_Slice)(array_)
|
|
|
+ map_free_dynamic(m^, info, loc) or_return
|
|
|
|
|
|
- if new_count < array.len {
|
|
|
- return true
|
|
|
- }
|
|
|
+ m.data = shrunk.data
|
|
|
|
|
|
- old_size := array.len*size_of(T)
|
|
|
- new_size := new_count*size_of(T)
|
|
|
+ return nil
|
|
|
+}
|
|
|
+
|
|
|
+@(require_results)
|
|
|
+map_free_dynamic :: proc "odin" (m: Raw_Map, info: ^Map_Info, loc := #caller_location) -> Allocator_Error {
|
|
|
+ ptr := rawptr(map_data(m))
|
|
|
+ size := int(map_total_allocation_size(uintptr(map_cap(m)), info))
|
|
|
+ return mem_free_with_size(ptr, size, m.allocator, loc)
|
|
|
+}
|
|
|
|
|
|
- new_data, err := mem_resize(array.data, old_size, new_size, align_of(T), allocator, loc)
|
|
|
- if err != nil {
|
|
|
+@(require_results)
|
|
|
+map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (index: uintptr, ok: bool) {
|
|
|
+ if map_len(m) == 0 {
|
|
|
+ return 0, false
|
|
|
+ }
|
|
|
+ h := info.key_hasher(rawptr(k), 0)
|
|
|
+ p := map_desired_position(m, h)
|
|
|
+ d := uintptr(0)
|
|
|
+ c := (uintptr(1) << map_log2_cap(m)) - 1
|
|
|
+ ks, _, hs, _, _ := map_kvh_data_dynamic(m, info)
|
|
|
+ for {
|
|
|
+ element_hash := hs[p]
|
|
|
+ if map_hash_is_empty(element_hash) {
|
|
|
+ return 0, false
|
|
|
+ } else if d > map_probe_distance(m, element_hash, p) {
|
|
|
+ return 0, false
|
|
|
+ } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) {
|
|
|
+ return p, true
|
|
|
+ }
|
|
|
+ p = (p + 1) & c
|
|
|
+ d += 1
|
|
|
+ }
|
|
|
+}
|
|
|
+@(require_results)
|
|
|
+map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (ok: bool) {
|
|
|
+ if map_len(m) == 0 {
|
|
|
return false
|
|
|
}
|
|
|
- if new_data != nil || size_of(E) == 0 {
|
|
|
- array.data = raw_data(new_data)
|
|
|
- array.len = new_count
|
|
|
- return true
|
|
|
+ h := info.key_hasher(rawptr(k), 0)
|
|
|
+ p := map_desired_position(m, h)
|
|
|
+ d := uintptr(0)
|
|
|
+ c := (uintptr(1) << map_log2_cap(m)) - 1
|
|
|
+ ks, _, hs, _, _ := map_kvh_data_dynamic(m, info)
|
|
|
+ for {
|
|
|
+ element_hash := hs[p]
|
|
|
+ if map_hash_is_empty(element_hash) {
|
|
|
+ return false
|
|
|
+ } else if d > map_probe_distance(m, element_hash, p) {
|
|
|
+ return false
|
|
|
+ } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ p = (p + 1) & c
|
|
|
+ d += 1
|
|
|
}
|
|
|
- return false
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_reset_entries :: proc "contextless" (h: Map_Header, loc := #caller_location) {
|
|
|
- for i in 0..<len(h.m.hashes) {
|
|
|
- h.m.hashes[i] = MAP_SENTINEL
|
|
|
- }
|
|
|
|
|
|
- for i in 0..<Map_Index(h.m.entries.len) {
|
|
|
- entry_header := __dynamic_map_get_entry(h, i)
|
|
|
- entry_header.next = MAP_SENTINEL
|
|
|
|
|
|
- fr := __dynamic_map_find_from_entry(h, entry_header)
|
|
|
- if fr.entry_prev != MAP_SENTINEL {
|
|
|
- e := __dynamic_map_get_entry(h, fr.entry_prev)
|
|
|
- e.next = i
|
|
|
- } else {
|
|
|
- h.m.hashes[fr.hash_index] = i
|
|
|
- }
|
|
|
- }
|
|
|
+@(require_results)
|
|
|
+map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (old_k, old_v: uintptr, ok: bool) {
|
|
|
+ MASK :: 1 << (size_of(Map_Hash)*8 - 1)
|
|
|
+
|
|
|
+ index := map_lookup_dynamic(m^, info, k) or_return
|
|
|
+ ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info)
|
|
|
+ hs[index] |= MASK
|
|
|
+ old_k = map_cell_index_dynamic(ks, info.ks, index)
|
|
|
+ old_v = map_cell_index_dynamic(vs, info.vs, index)
|
|
|
+ m.len -= 1
|
|
|
+ ok = true
|
|
|
+ return
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_shrink :: proc "odin" (h: Map_Header, cap: int, loc := #caller_location) -> (did_shrink: bool) {
|
|
|
- c := context
|
|
|
- if h.m.entries.allocator.procedure != nil {
|
|
|
- c.allocator = h.m.entries.allocator
|
|
|
+map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) {
|
|
|
+ if m.data == 0 {
|
|
|
+ return
|
|
|
}
|
|
|
- context = c
|
|
|
+ _, _, hs, _, _ := map_kvh_data_dynamic(m^, info)
|
|
|
+ intrinsics.mem_zero(rawptr(hs), map_cap(m^) * size_of(Map_Hash))
|
|
|
+ m.len = 0
|
|
|
+}
|
|
|
+
|
|
|
|
|
|
- return __dynamic_array_shrink(&h.m.entries, h.entry_size, h.entry_align, cap, loc)
|
|
|
+@(require_results)
|
|
|
+map_kvh_data_static :: #force_inline proc "contextless" (m: $T/map[$K]$V) -> (ks: [^]Map_Cell(K), vs: [^]Map_Cell(V), hs: [^]Map_Hash) {
|
|
|
+ capacity := uintptr(cap(m))
|
|
|
+ ks = ([^]Map_Cell(K))(map_data(transmute(Raw_Map)m))
|
|
|
+ vs = ([^]Map_Cell(V))(map_cell_index_static(ks, capacity))
|
|
|
+ hs = ([^]Map_Hash)(map_cell_index_static(vs, capacity))
|
|
|
+ return
|
|
|
}
|
|
|
|
|
|
|
|
|
-@(private="file")
|
|
|
-ceil_to_pow2 :: proc "contextless" (n: uint) -> uint {
|
|
|
- if n <= 2 {
|
|
|
- return n
|
|
|
+@(require_results)
|
|
|
+map_get :: proc "contextless" (m: $T/map[$K]$V, key: K) -> (stored_key: K, stored_value: V, ok: bool) {
|
|
|
+ rm := transmute(Raw_Map)m
|
|
|
+ if rm.len == 0 {
|
|
|
+ return
|
|
|
}
|
|
|
- n := n
|
|
|
- n -= 1
|
|
|
- n |= n >> 1
|
|
|
- n |= n >> 2
|
|
|
- n |= n >> 4
|
|
|
- n |= n >> 8
|
|
|
- n |= n >> 16
|
|
|
- when size_of(int) == 8 {
|
|
|
- n |= n >> 32
|
|
|
+ info := intrinsics.type_map_info(T)
|
|
|
+ key := key
|
|
|
+
|
|
|
+ h := info.key_hasher(&key, 0)
|
|
|
+ pos := map_desired_position(rm, h)
|
|
|
+ distance := uintptr(0)
|
|
|
+ mask := (uintptr(1) << map_log2_cap(rm)) - 1
|
|
|
+ ks, vs, hs := map_kvh_data_static(m)
|
|
|
+ for {
|
|
|
+ element_hash := hs[pos]
|
|
|
+ if map_hash_is_empty(element_hash) {
|
|
|
+ return
|
|
|
+ } else if distance > map_probe_distance(rm, element_hash, pos) {
|
|
|
+ return
|
|
|
+ } else if element_hash == h {
|
|
|
+ element_key := map_cell_index_static(ks, pos)
|
|
|
+ if info.key_equal(&key, rawptr(element_key)) {
|
|
|
+ element_value := map_cell_index_static(vs, pos)
|
|
|
+ stored_key = (^K)(element_key)^
|
|
|
+ stored_value = (^V)(element_value)^
|
|
|
+ ok = true
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ pos = (pos + 1) & mask
|
|
|
+ distance += 1
|
|
|
}
|
|
|
- n += 1
|
|
|
- return n
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_grow :: proc "odin" (h: Map_Header, loc := #caller_location) {
|
|
|
- new_count := max(uint(h.m.entries.cap) * 2, INITIAL_MAP_CAP)
|
|
|
- // Rehash through Reserve
|
|
|
- __dynamic_map_reserve(h.m, h.table, new_count, loc)
|
|
|
+// IMPORTANT: USED WITHIN THE COMPILER
|
|
|
+__dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, key: rawptr) -> (ptr: rawptr) {
|
|
|
+ if m.len == 0 {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+ pos := map_desired_position(m^, h)
|
|
|
+ distance := uintptr(0)
|
|
|
+ mask := (uintptr(1) << map_log2_cap(m^)) - 1
|
|
|
+ ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info)
|
|
|
+ for {
|
|
|
+ element_hash := hs[pos]
|
|
|
+ if map_hash_is_empty(element_hash) {
|
|
|
+ return nil
|
|
|
+ } else if distance > map_probe_distance(m^, element_hash, pos) {
|
|
|
+ return nil
|
|
|
+ } else if element_hash == h && info.key_equal(key, rawptr(map_cell_index_dynamic(ks, info.ks, pos))) {
|
|
|
+ return rawptr(map_cell_index_dynamic(vs, info.vs, pos))
|
|
|
+ }
|
|
|
+ pos = (pos + 1) & mask
|
|
|
+ distance += 1
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_full :: #force_inline proc "contextless" (h: Map_Header) -> bool {
|
|
|
- return int(0.75 * f64(len(h.m.hashes))) <= h.m.entries.len
|
|
|
+// IMPORTANT: USED WITHIN THE COMPILER
|
|
|
+__dynamic_map_check_grow :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error {
|
|
|
+ if m.len >= map_resize_threshold(m^) {
|
|
|
+ return map_grow_dynamic(m, info, loc)
|
|
|
+ }
|
|
|
+ return nil
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_find_from_entry :: proc "contextless" (h: Map_Header, e: ^Map_Entry_Header) -> Map_Find_Result #no_bounds_check {
|
|
|
- key_ptr := __get_map_entry_key_ptr(h, e)
|
|
|
- return __dynamic_map_find(h, e.hash, key_ptr)
|
|
|
-
|
|
|
+__dynamic_map_set_without_hash :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr {
|
|
|
+ return __dynamic_map_set(m, info, info.key_hasher(key, 0), key, value, loc)
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_find :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> Map_Find_Result #no_bounds_check {
|
|
|
- fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL}
|
|
|
- if n := uintptr(len(h.m.hashes)); n != 0 {
|
|
|
- fr.hash_index = Map_Index(key_hash & (n-1))
|
|
|
- fr.entry_index = h.m.hashes[fr.hash_index]
|
|
|
- for fr.entry_index != MAP_SENTINEL {
|
|
|
- entry := __dynamic_map_get_entry(h, fr.entry_index)
|
|
|
- entry_key_ptr := __get_map_entry_key_ptr(h, entry)
|
|
|
- if entry.hash == key_hash && h.equal(entry_key_ptr, key_ptr) {
|
|
|
- return fr
|
|
|
- }
|
|
|
-
|
|
|
- fr.entry_prev = fr.entry_index
|
|
|
- fr.entry_index = entry.next
|
|
|
- }
|
|
|
+
|
|
|
+// IMPORTANT: USED WITHIN THE COMPILER
|
|
|
+__dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, hash: Map_Hash, key, value: rawptr, loc := #caller_location) -> rawptr {
|
|
|
+ if found := __dynamic_map_get(m, info, hash, key); found != nil {
|
|
|
+ intrinsics.mem_copy_non_overlapping(found, value, info.vs.size_of_type)
|
|
|
+ return found
|
|
|
}
|
|
|
- return fr
|
|
|
-}
|
|
|
|
|
|
-// Utility procedure used by other runtime procedures
|
|
|
-__map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Result #no_bounds_check {
|
|
|
- hash := __get_map_key_hash(key_ptr)
|
|
|
- return #force_inline __dynamic_map_find(h, hash, key_ptr)
|
|
|
+ if __dynamic_map_check_grow(m, info, loc) != nil {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+
|
|
|
+ result := map_insert_hash_dynamic(m, info, hash, uintptr(key), uintptr(value))
|
|
|
+ m.len += 1
|
|
|
+ return rawptr(result)
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_get_entry :: #force_inline proc "contextless" (h: Map_Header, index: Map_Index) -> ^Map_Entry_Header {
|
|
|
- return (^Map_Entry_Header)(uintptr(h.m.entries.data) + uintptr(index*Map_Index(h.entry_size)))
|
|
|
+// IMPORTANT: USED WITHIN THE COMPILER
|
|
|
+@(private)
|
|
|
+__dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) -> Allocator_Error {
|
|
|
+ return map_reserve_dynamic(m, info, uintptr(new_capacity), loc)
|
|
|
}
|
|
|
|
|
|
-__dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) #no_bounds_check {
|
|
|
- if fr.entry_prev != MAP_SENTINEL {
|
|
|
- prev := __dynamic_map_get_entry(h, fr.entry_prev)
|
|
|
- curr := __dynamic_map_get_entry(h, fr.entry_index)
|
|
|
- prev.next = curr.next
|
|
|
- } else {
|
|
|
- h.m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next
|
|
|
+
|
|
|
+
|
|
|
+// NOTE: the default hashing algorithm derives from fnv64a, with some minor modifications to work for `map` type:
|
|
|
+//
|
|
|
+// * Convert a `0` result to `1`
|
|
|
+// * "empty entry"
|
|
|
+// * Prevent the top bit from being set
|
|
|
+// * "deleted entry"
|
|
|
+//
|
|
|
+// Both of these modification are necessary for the implementation of the `map`
|
|
|
+
|
|
|
+INITIAL_HASH_SEED :: 0xcbf29ce484222325
|
|
|
+
|
|
|
+HASH_MASK :: 1 << (8*size_of(uintptr) - 1) -1
|
|
|
+
|
|
|
+default_hasher :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, N: int) -> uintptr {
|
|
|
+ h := u64(seed) + INITIAL_HASH_SEED
|
|
|
+ p := ([^]byte)(data)
|
|
|
+ for _ in 0..<N {
|
|
|
+ h = (h ~ u64(p[0])) * 0x100000001b3
|
|
|
+ p = p[1:]
|
|
|
}
|
|
|
- last_index := Map_Index(h.m.entries.len-1)
|
|
|
- if fr.entry_index != last_index {
|
|
|
- old := __dynamic_map_get_entry(h, fr.entry_index)
|
|
|
- end := __dynamic_map_get_entry(h, last_index)
|
|
|
- mem_copy(old, end, h.entry_size)
|
|
|
-
|
|
|
- last := __dynamic_map_find_from_entry(h, old)
|
|
|
- if last.entry_prev != MAP_SENTINEL {
|
|
|
- e := __dynamic_map_get_entry(h, last.entry_prev)
|
|
|
- e.next = fr.entry_index
|
|
|
- } else {
|
|
|
- h.m.hashes[last.hash_index] = fr.entry_index
|
|
|
+ h &= HASH_MASK
|
|
|
+ return uintptr(h) | uintptr(uintptr(h) == 0)
|
|
|
+}
|
|
|
+
|
|
|
+default_hasher_string :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr {
|
|
|
+ str := (^[]byte)(data)
|
|
|
+ return default_hasher(raw_data(str^), seed, len(str))
|
|
|
+}
|
|
|
+default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr {
|
|
|
+ h := u64(seed) + INITIAL_HASH_SEED
|
|
|
+ if ptr := (^[^]byte)(data)^; ptr != nil {
|
|
|
+ for ptr[0] != 0 {
|
|
|
+ h = (h ~ u64(ptr[0])) * 0x100000001b3
|
|
|
+ ptr = ptr[1:]
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- h.m.entries.len -= 1
|
|
|
+ h &= HASH_MASK
|
|
|
+ return uintptr(h) | uintptr(uintptr(h) == 0)
|
|
|
}
|
gingerBill