internal static void GetKeyValueArrays <TKey, TValue>(UnsafeHashMapData *data, NativeKeyValueArrays <TKey, TValue> result) where TKey : struct where TValue : struct { var bucketArray = (int *)data->buckets; var bucketNext = (int *)data->next; int o = 0; for (int i = 0; i <= data->bucketCapacityMask; ++i) { int b = bucketArray[i]; while (b != -1) { result.Keys[o] = UnsafeUtility.ReadArrayElement <TKey>(data->keys, b); result.Values[o] = UnsafeUtility.ReadArrayElement <TValue>(data->values, b); o++; b = bucketNext[b]; } } Assert.AreEqual(result.Keys.Length, o); Assert.AreEqual(result.Values.Length, o); }
internal static unsafe bool TryGetNextValueAtomic(UnsafeHashMapData *data, out TValue item, ref NativeMultiHashMapIterator <TKey> it) { int entryIdx = it.NextEntryIndex; it.NextEntryIndex = -1; it.EntryIndex = -1; item = default; if (entryIdx < 0 || entryIdx >= data->keyCapacity) { return(false); } int *pNextIndexChain = (int *)data->next; while (!UnsafeUtility.ReadArrayElement <TKey>(data->keys, entryIdx).Equals(it.key)) { entryIdx = pNextIndexChain[entryIdx]; if (entryIdx < 0 || entryIdx >= data->keyCapacity) { return(false); } } it.NextEntryIndex = pNextIndexChain[entryIdx]; it.EntryIndex = entryIdx; // Read the value item = UnsafeUtility.ReadArrayElement <TValue>(data->values, entryIdx); return(true); }
internal static unsafe void RemoveKeyValue <TValueEQ>(UnsafeHashMapData *data, TKey key, TValueEQ value) where TValueEQ : struct, IEquatable <TValueEQ> { var buckets = (int *)data->buckets; var keyCapacity = (uint)data->keyCapacity; var prevNextPtr = buckets + (key.GetHashCode() & data->bucketCapacityMask); var entryIdx = *prevNextPtr; if ((uint)entryIdx >= keyCapacity) { return; } var pNextIndexChain = (int *)data->next; var keys = data->keys; var values = data->values; var firstFreeTLS = data->firstFreeTLS; do { if (UnsafeUtility.ReadArrayElement <TKey>(keys, entryIdx).Equals(key) && UnsafeUtility.ReadArrayElement <TValueEQ>(values, entryIdx).Equals(value)) { int nextIdx = pNextIndexChain[entryIdx]; pNextIndexChain[entryIdx] = firstFreeTLS[0]; firstFreeTLS[0] = entryIdx; *prevNextPtr = entryIdx = nextIdx; } else { prevNextPtr = pNextIndexChain + entryIdx; entryIdx = *prevNextPtr; } }while ((uint)entryIdx < keyCapacity); }
public static unsafe void Execute(ref NativeMultiHashMapVisitKeyValueJobStruct <TJob, TKey, TValue> fullData, IntPtr additionalPtr, IntPtr bufferRangePatchData, ref JobRanges ranges, int jobIndex) { while (true) { int begin; int end; if (!JobsUtility.GetWorkStealingRange(ref ranges, jobIndex, out begin, out end)) { return; } UnsafeHashMapData *hashMapData = fullData.HashMap.m_MultiHashMapData.m_Buffer; var buckets = (int *)hashMapData->buckets; var nextPtrs = (int *)hashMapData->next; var keys = hashMapData->keys; var values = hashMapData->values; for (int i = begin; i < end; i++) { int entryIndex = buckets[i]; while (entryIndex != -1) { var key = UnsafeUtility.ReadArrayElement <TKey>(keys, entryIndex); var value = UnsafeUtility.ReadArrayElement <TValue>(values, entryIndex); fullData.JobData.ExecuteNext(key, value); entryIndex = nextPtrs[entryIndex]; } } } }
internal static unsafe void Remove(UnsafeHashMapData *data, NativeMultiHashMapIterator <TKey> it) { // First find the slot based on the hash int *buckets = (int *)data->buckets; int *pNextIndexChain = (int *)data->next; int bucket = it.key.GetHashCode() & data->bucketCapacityMask; int entryIdx = buckets[bucket]; if (entryIdx == it.EntryIndex) { buckets[bucket] = pNextIndexChain[entryIdx]; } else { while (entryIdx >= 0 && pNextIndexChain[entryIdx] != it.EntryIndex) { entryIdx = pNextIndexChain[entryIdx]; } if (entryIdx < 0) { throw new InvalidOperationException("Invalid iterator passed to HashMap remove"); } pNextIndexChain[entryIdx] = pNextIndexChain[it.EntryIndex]; } // And free the index pNextIndexChain[it.EntryIndex] = data->firstFreeTLS[0]; data->firstFreeTLS[0] = it.EntryIndex; }
public static unsafe void Execute(ref NativeMultiHashMapUniqueHashJobStruct <TJob> fullData, IntPtr additionalPtr, IntPtr bufferRangePatchData, ref JobRanges ranges, int jobIndex) { while (true) { int begin; int end; if (!JobsUtility.GetWorkStealingRange(ref ranges, jobIndex, out begin, out end)) { return; } UnsafeHashMapData *hashMapData = fullData.HashMap.m_MultiHashMapData.m_Buffer; var buckets = (int *)hashMapData->buckets; var nextPtrs = (int *)hashMapData->next; var keys = hashMapData->keys; var values = hashMapData->values; for (int i = begin; i < end; i++) { int entryIndex = buckets[i]; while (entryIndex != -1) { var key = UnsafeUtility.ReadArrayElement <int>(keys, entryIndex); var value = UnsafeUtility.ReadArrayElement <int>(values, entryIndex); int firstValue; NativeMultiHashMapIterator <int> it; fullData.HashMap.TryGetFirstValue(key, out firstValue, out it); // [macton] Didn't expect a usecase for this with multiple same values // (since it's intended use was for unique indices.) // https://forum.unity.com/threads/ijobnativemultihashmapmergedsharedkeyindices-unexpected-behavior.569107/#post-3788170 if (entryIndex == it.EntryIndex) { #if ENABLE_UNITY_COLLECTIONS_CHECKS JobsUtility.PatchBufferMinMaxRanges(bufferRangePatchData, UnsafeUtility.AddressOf(ref fullData), value, 1); #endif fullData.JobData.ExecuteFirst(value); } else { #if ENABLE_UNITY_COLLECTIONS_CHECKS var startIndex = Math.Min(firstValue, value); var lastIndex = Math.Max(firstValue, value); var rangeLength = (lastIndex - startIndex) + 1; JobsUtility.PatchBufferMinMaxRanges(bufferRangePatchData, UnsafeUtility.AddressOf(ref fullData), startIndex, rangeLength); #endif fullData.JobData.ExecuteNext(firstValue, value); } entryIndex = nextPtrs[entryIndex]; } } } }
public JobHandle Dispose(JobHandle inputDeps) { var jobHandle = new UnsafeHashMapDisposeJob { Data = m_Buffer, Allocator = m_AllocatorLabel }.Schedule(inputDeps); m_Buffer = null; return(jobHandle); }
internal static unsafe bool SetValue(UnsafeHashMapData *data, ref NativeMultiHashMapIterator <TKey> it, ref TValue item) { int entryIdx = it.EntryIndex; if (entryIdx < 0 || entryIdx >= data->keyCapacity) { return(false); } UnsafeUtility.WriteArrayElement(data->values, entryIdx, item); return(true); }
internal static unsafe void Clear(UnsafeHashMapData *data) { UnsafeUtility.MemSet(data->buckets, 0xff, (data->bucketCapacityMask + 1) * 4); UnsafeUtility.MemSet(data->next, 0xff, (data->keyCapacity) * 4); for (int tls = 0; tls < JobsUtility.MaxJobThreadCount; ++tls) { data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine] = -1; } data->allocatedIndexLength = 0; }
static void EvaluateBucketImp(int maskedHash, UnsafeHashMapData *data) { Assert.IsTrue(maskedHash >= 0 && maskedHash <= data->bucketCapacityMask, "Masked Hash Code out of range!"); int entryIndex = data->buckets[maskedHash]; var pNextIndexChain = data->next; while (entryIndex != -1) { UnsafeUtility.ReadArrayElement <TAggregator>(data->values, entryIndex).Evaluate(); entryIndex = pNextIndexChain[entryIndex]; } }
internal static int CountImp(UnsafeHashMapData *data) { int *pNextIndexes = (int *)data->next; int freeListSize = 0; for (int tls = 0; tls < JobsUtility.MaxJobThreadCount; ++tls) { int freeIdx = data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine]; for (; freeIdx >= 0; freeIdx = pNextIndexes[freeIdx]) { ++freeListSize; } } return(math.min(data->keyCapacity, data->allocatedIndexLength) - freeListSize); }
internal static unsafe int Remove(UnsafeHashMapData *data, TKey key, bool isMultiHashMap) { var removed = 0; // First find the slot based on the hash var buckets = (int *)data->buckets; var pNextIndexChain = (int *)data->next; var bucket = key.GetHashCode() & data->bucketCapacityMask; var prevEntry = -1; var entryIdx = buckets[bucket]; while (entryIdx >= 0 && entryIdx < data->keyCapacity) { if (UnsafeUtility.ReadArrayElement <TKey>(data->keys, entryIdx).Equals(key)) { ++removed; // Found matching element, remove it if (prevEntry < 0) { buckets[bucket] = pNextIndexChain[entryIdx]; } else { pNextIndexChain[prevEntry] = pNextIndexChain[entryIdx]; } // And free the index int nextIdx = pNextIndexChain[entryIdx]; pNextIndexChain[entryIdx] = data->firstFreeTLS[0]; data->firstFreeTLS[0] = entryIdx; entryIdx = nextIdx; // Can only be one hit in regular hashmaps, so return if (!isMultiHashMap) { break; } } else { prevEntry = entryIdx; entryIdx = pNextIndexChain[entryIdx]; } } return(removed); }
internal static unsafe bool TryGetFirstValueAtomic(UnsafeHashMapData *data, TKey key, out TValue item, out NativeMultiHashMapIterator <TKey> it) { it.key = key; if (data->allocatedIndexLength <= 0) { it.EntryIndex = it.NextEntryIndex = -1; item = default; return(false); } // First find the slot based on the hash int *buckets = (int *)data->buckets; int bucket = key.GetHashCode() & data->bucketCapacityMask; it.EntryIndex = it.NextEntryIndex = buckets[bucket]; return(TryGetNextValueAtomic(data, out item, ref it)); }
internal static unsafe void AddAtomicMulti(UnsafeHashMapData *data, TKey key, TValue item, int threadIndex) { // Allocate an entry from the free list int idx = AllocEntry(data, threadIndex); // Write the new value to the entry UnsafeUtility.WriteArrayElement(data->keys, idx, key); UnsafeUtility.WriteArrayElement(data->values, idx, item); int bucket = key.GetHashCode() & data->bucketCapacityMask; // Add the index to the hash-map int *buckets = (int *)data->buckets; int nextPtr; int *pNextIndexChain = (int *)data->next; do { nextPtr = buckets[bucket]; pNextIndexChain[idx] = nextPtr; }while (Interlocked.CompareExchange(ref buckets[bucket], idx, nextPtr) != nextPtr); }
internal static unsafe bool TryAddAtomic(UnsafeHashMapData *data, TKey key, TValue item, int threadIndex) { if (TryGetFirstValueAtomic(data, key, out _, out _)) { return(false); } // Allocate an entry from the free list int idx = AllocEntry(data, threadIndex); // Write the new value to the entry UnsafeUtility.WriteArrayElement(data->keys, idx, key); UnsafeUtility.WriteArrayElement(data->values, idx, item); int bucket = key.GetHashCode() & data->bucketCapacityMask; // Add the index to the hash-map int *buckets = (int *)data->buckets; if (Interlocked.CompareExchange(ref buckets[bucket], idx, -1) != -1) { int *pNextIndexChain = (int *)data->next; do { pNextIndexChain[idx] = buckets[bucket]; if (TryGetFirstValueAtomic(data, key, out _, out _)) { // Put back the entry in the free list if someone else added it while trying to add do { pNextIndexChain[idx] = data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine]; }while (Interlocked.CompareExchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], idx, pNextIndexChain[idx]) != pNextIndexChain[idx]); return(false); } }while (Interlocked.CompareExchange(ref buckets[bucket], idx, pNextIndexChain[idx]) != pNextIndexChain[idx]); } return(true); }
internal static void AllocateHashMap <TKey, TValue>(int length, int bucketLength, Allocator label, out UnsafeHashMapData *outBuf) where TKey : struct where TValue : struct { IsBlittableAndThrow <TKey, TValue>(); UnsafeHashMapData *data = (UnsafeHashMapData *)UnsafeUtility.Malloc(sizeof(UnsafeHashMapData), UnsafeUtility.AlignOf <UnsafeHashMapData>(), label); bucketLength = math.ceilpow2(bucketLength); data->keyCapacity = length; data->bucketCapacityMask = bucketLength - 1; int keyOffset, nextOffset, bucketOffset; int totalSize = CalculateDataSize <TKey, TValue>(length, bucketLength, out keyOffset, out nextOffset, out bucketOffset); data->values = (byte *)UnsafeUtility.Malloc(totalSize, JobsUtility.CacheLineSize, label); data->keys = data->values + keyOffset; data->next = data->values + nextOffset; data->buckets = data->values + bucketOffset; outBuf = data; }
/// <summary> /// Disposes of this multi-hashmap and deallocates its memory immediately. /// </summary> public void Dispose() { UnsafeHashMapData.DeallocateHashMap(m_Buffer, m_AllocatorLabel); m_Buffer = null; }
internal static void ReallocateHashMap <TKey, TValue>(UnsafeHashMapData *data, int newCapacity, int newBucketCapacity, Allocator label) where TKey : struct where TValue : struct { newBucketCapacity = math.ceilpow2(newBucketCapacity); if (data->keyCapacity == newCapacity && (data->bucketCapacityMask + 1) == newBucketCapacity) { return; } if (data->keyCapacity > newCapacity) { throw new Exception("Shrinking a hash map is not supported"); } int keyOffset, nextOffset, bucketOffset; int totalSize = CalculateDataSize <TKey, TValue>(newCapacity, newBucketCapacity, out keyOffset, out nextOffset, out bucketOffset); byte *newData = (byte *)UnsafeUtility.Malloc(totalSize, JobsUtility.CacheLineSize, label); byte *newKeys = newData + keyOffset; byte *newNext = newData + nextOffset; byte *newBuckets = newData + bucketOffset; // The items are taken from a free-list and might not be tightly packed, copy all of the old capcity UnsafeUtility.MemCpy(newData, data->values, data->keyCapacity * UnsafeUtility.SizeOf <TValue>()); UnsafeUtility.MemCpy(newKeys, data->keys, data->keyCapacity * UnsafeUtility.SizeOf <TKey>()); UnsafeUtility.MemCpy(newNext, data->next, data->keyCapacity * UnsafeUtility.SizeOf <int>()); for (int emptyNext = data->keyCapacity; emptyNext < newCapacity; ++emptyNext) { ((int *)newNext)[emptyNext] = -1; } // re-hash the buckets, first clear the new bucket list, then insert all values from the old list for (int bucket = 0; bucket < newBucketCapacity; ++bucket) { ((int *)newBuckets)[bucket] = -1; } for (int bucket = 0; bucket <= data->bucketCapacityMask; ++bucket) { int *buckets = (int *)data->buckets; int *pNextIndexChain = (int *)newNext; while (buckets[bucket] >= 0) { int curEntry = buckets[bucket]; buckets[bucket] = pNextIndexChain[curEntry]; int newBucket = UnsafeUtility.ReadArrayElement <TKey>(data->keys, curEntry).GetHashCode() & (newBucketCapacity - 1); pNextIndexChain[curEntry] = ((int *)newBuckets)[newBucket]; ((int *)newBuckets)[newBucket] = curEntry; } } UnsafeUtility.Free(data->values, label); if (data->allocatedIndexLength > data->keyCapacity) { data->allocatedIndexLength = data->keyCapacity; } data->values = newData; data->keys = newKeys; data->next = newNext; data->buckets = newBuckets; data->keyCapacity = newCapacity; data->bucketCapacityMask = newBucketCapacity - 1; }
internal static void DeallocateHashMap(UnsafeHashMapData *data, Allocator allocator) { UnsafeUtility.Free(data->values, allocator); UnsafeUtility.Free(data, allocator); }
internal static unsafe int AllocEntry(UnsafeHashMapData *data, int threadIndex) { int idx; int *pNextIndexChain = (int *)data->next; do { idx = data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine]; if (idx < 0) { // Try to refill local cache Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], -2); // If it failed try to get one from the never-allocated array if (data->allocatedIndexLength < data->keyCapacity) { idx = Interlocked.Add(ref data->allocatedIndexLength, 16) - 16; if (idx < data->keyCapacity - 1) { int count = math.min(16, data->keyCapacity - idx); for (int i = 1; i < count; ++i) { pNextIndexChain[idx + i] = idx + i + 1; } pNextIndexChain[idx + count - 1] = -1; pNextIndexChain[idx] = -1; Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], idx + 1); return(idx); } if (idx == data->keyCapacity - 1) { Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], -1); return(idx); } } Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], -1); // Failed to get any, try to get one from another free list bool again = true; while (again) { again = false; for (int other = (threadIndex + 1) % JobsUtility.MaxJobThreadCount; other != threadIndex; other = (other + 1) % JobsUtility.MaxJobThreadCount) { do { idx = data->firstFreeTLS[other * UnsafeHashMapData.IntPerCacheLine]; if (idx < 0) { break; } }while (Interlocked.CompareExchange(ref data->firstFreeTLS[other * UnsafeHashMapData.IntPerCacheLine], pNextIndexChain[idx], idx) != idx); if (idx == -2) { again = true; } else if (idx >= 0) { pNextIndexChain[idx] = -1; return(idx); } } } throw new InvalidOperationException("HashMap is full"); } if (idx >= data->keyCapacity) { throw new InvalidOperationException(string.Format("nextPtr idx {0} beyond capacity {1}", idx, data->keyCapacity)); } }while (Interlocked.CompareExchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], pNextIndexChain[idx], idx) != idx); pNextIndexChain[idx] = -1; return(idx); }
internal static unsafe bool TryAdd(UnsafeHashMapData *data, TKey key, TValue item, bool isMultiHashMap, Allocator allocation) { if (!isMultiHashMap && TryGetFirstValueAtomic(data, key, out _, out _)) { return(false); } // Allocate an entry from the free list int idx; int *pNextIndexChain; if (data->allocatedIndexLength >= data->keyCapacity && data->firstFreeTLS[0] < 0) { for (int tls = 1; tls < JobsUtility.MaxJobThreadCount; ++tls) { if (data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine] >= 0) { idx = data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine]; pNextIndexChain = (int *)data->next; data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine] = pNextIndexChain[idx]; pNextIndexChain[idx] = -1; data->firstFreeTLS[0] = idx; break; } } if (data->firstFreeTLS[0] < 0) { int newCap = UnsafeHashMapData.GrowCapacity(data->keyCapacity); UnsafeHashMapData.ReallocateHashMap <TKey, TValue>(data, newCap, UnsafeHashMapData.GetBucketSize(newCap), allocation); } } idx = data->firstFreeTLS[0]; if (idx >= 0) { data->firstFreeTLS[0] = ((int *)data->next)[idx]; } else { idx = data->allocatedIndexLength++; } if (idx < 0 || idx >= data->keyCapacity) { throw new InvalidOperationException("Internal HashMap error"); } // Write the new value to the entry UnsafeUtility.WriteArrayElement(data->keys, idx, key); UnsafeUtility.WriteArrayElement(data->values, idx, item); int bucket = key.GetHashCode() & data->bucketCapacityMask; // Add the index to the hash-map int *buckets = (int *)data->buckets; pNextIndexChain = (int *)data->next; pNextIndexChain[idx] = buckets[bucket]; buckets[bucket] = idx; return(true); }