internal static void GetKeyValueArrays <TKey, TValue>(UnsafeHashMapData *data, NativeKeyValueArrays <TKey, TValue> result)
where TKey : struct
where TValue : struct
{
var bucketArray = (int *)data->buckets;
var bucketNext = (int *)data->next;
int o = 0;
for (int i = 0; i <= data->bucketCapacityMask; ++i)
{
int b = bucketArray[i];
while (b != -1)
{
result.Keys[o] = UnsafeUtility.ReadArrayElement <TKey>(data->keys, b);
result.Values[o] = UnsafeUtility.ReadArrayElement <TValue>(data->values, b);
o++;
b = bucketNext[b];
}
}
Assert.AreEqual(result.Keys.Length, o);
Assert.AreEqual(result.Values.Length, o);
}
internal static unsafe bool TryGetNextValueAtomic(UnsafeHashMapData *data, out TValue item, ref NativeMultiHashMapIterator <TKey> it)
{
int entryIdx = it.NextEntryIndex;
it.NextEntryIndex = -1;
it.EntryIndex = -1;
item = default;
if (entryIdx < 0 || entryIdx >= data->keyCapacity)
{
return(false);
}
int *pNextIndexChain = (int *)data->next;
while (!UnsafeUtility.ReadArrayElement <TKey>(data->keys, entryIdx).Equals(it.key))
{
entryIdx = pNextIndexChain[entryIdx];
if (entryIdx < 0 || entryIdx >= data->keyCapacity)
{
return(false);
}
}
it.NextEntryIndex = pNextIndexChain[entryIdx];
it.EntryIndex = entryIdx;
// Read the value
item = UnsafeUtility.ReadArrayElement <TValue>(data->values, entryIdx);
return(true);
}
internal static unsafe void RemoveKeyValue <TValueEQ>(UnsafeHashMapData *data, TKey key, TValueEQ value)
where TValueEQ : struct, IEquatable <TValueEQ>
{
var buckets = (int *)data->buckets;
var keyCapacity = (uint)data->keyCapacity;
var prevNextPtr = buckets + (key.GetHashCode() & data->bucketCapacityMask);
var entryIdx = *prevNextPtr;
if ((uint)entryIdx >= keyCapacity)
{
return;
}
var pNextIndexChain = (int *)data->next;
var keys = data->keys;
var values = data->values;
var firstFreeTLS = data->firstFreeTLS;
do
{
if (UnsafeUtility.ReadArrayElement <TKey>(keys, entryIdx).Equals(key) &&
UnsafeUtility.ReadArrayElement <TValueEQ>(values, entryIdx).Equals(value))
{
int nextIdx = pNextIndexChain[entryIdx];
pNextIndexChain[entryIdx] = firstFreeTLS[0];
firstFreeTLS[0] = entryIdx;
*prevNextPtr = entryIdx = nextIdx;
}
else
{
prevNextPtr = pNextIndexChain + entryIdx;
entryIdx = *prevNextPtr;
}
}while ((uint)entryIdx < keyCapacity);
}
public static unsafe void Execute(ref NativeMultiHashMapVisitKeyValueJobStruct <TJob, TKey, TValue> fullData, IntPtr additionalPtr, IntPtr bufferRangePatchData, ref JobRanges ranges, int jobIndex)
{
while (true)
{
int begin;
int end;
if (!JobsUtility.GetWorkStealingRange(ref ranges, jobIndex, out begin, out end))
{
return;
}
UnsafeHashMapData *hashMapData = fullData.HashMap.m_MultiHashMapData.m_Buffer;
var buckets = (int *)hashMapData->buckets;
var nextPtrs = (int *)hashMapData->next;
var keys = hashMapData->keys;
var values = hashMapData->values;
for (int i = begin; i < end; i++)
{
int entryIndex = buckets[i];
while (entryIndex != -1)
{
var key = UnsafeUtility.ReadArrayElement <TKey>(keys, entryIndex);
var value = UnsafeUtility.ReadArrayElement <TValue>(values, entryIndex);
fullData.JobData.ExecuteNext(key, value);
entryIndex = nextPtrs[entryIndex];
}
}
}
}
internal static unsafe void Remove(UnsafeHashMapData *data, NativeMultiHashMapIterator <TKey> it)
{
// First find the slot based on the hash
int *buckets = (int *)data->buckets;
int *pNextIndexChain = (int *)data->next;
int bucket = it.key.GetHashCode() & data->bucketCapacityMask;
int entryIdx = buckets[bucket];
if (entryIdx == it.EntryIndex)
{
buckets[bucket] = pNextIndexChain[entryIdx];
}
else
{
while (entryIdx >= 0 && pNextIndexChain[entryIdx] != it.EntryIndex)
{
entryIdx = pNextIndexChain[entryIdx];
}
if (entryIdx < 0)
{
throw new InvalidOperationException("Invalid iterator passed to HashMap remove");
}
pNextIndexChain[entryIdx] = pNextIndexChain[it.EntryIndex];
}
// And free the index
pNextIndexChain[it.EntryIndex] = data->firstFreeTLS[0];
data->firstFreeTLS[0] = it.EntryIndex;
}
public static unsafe void Execute(ref NativeMultiHashMapUniqueHashJobStruct <TJob> fullData, IntPtr additionalPtr, IntPtr bufferRangePatchData, ref JobRanges ranges, int jobIndex)
{
while (true)
{
int begin;
int end;
if (!JobsUtility.GetWorkStealingRange(ref ranges, jobIndex, out begin, out end))
{
return;
}
UnsafeHashMapData *hashMapData = fullData.HashMap.m_MultiHashMapData.m_Buffer;
var buckets = (int *)hashMapData->buckets;
var nextPtrs = (int *)hashMapData->next;
var keys = hashMapData->keys;
var values = hashMapData->values;
for (int i = begin; i < end; i++)
{
int entryIndex = buckets[i];
while (entryIndex != -1)
{
var key = UnsafeUtility.ReadArrayElement <int>(keys, entryIndex);
var value = UnsafeUtility.ReadArrayElement <int>(values, entryIndex);
int firstValue;
NativeMultiHashMapIterator <int> it;
fullData.HashMap.TryGetFirstValue(key, out firstValue, out it);
// [macton] Didn't expect a usecase for this with multiple same values
// (since it's intended use was for unique indices.)
// https://forum.unity.com/threads/ijobnativemultihashmapmergedsharedkeyindices-unexpected-behavior.569107/#post-3788170
if (entryIndex == it.EntryIndex)
{
#if ENABLE_UNITY_COLLECTIONS_CHECKS
JobsUtility.PatchBufferMinMaxRanges(bufferRangePatchData, UnsafeUtility.AddressOf(ref fullData), value, 1);
#endif
fullData.JobData.ExecuteFirst(value);
}
else
{
#if ENABLE_UNITY_COLLECTIONS_CHECKS
var startIndex = Math.Min(firstValue, value);
var lastIndex = Math.Max(firstValue, value);
var rangeLength = (lastIndex - startIndex) + 1;
JobsUtility.PatchBufferMinMaxRanges(bufferRangePatchData, UnsafeUtility.AddressOf(ref fullData), startIndex, rangeLength);
#endif
fullData.JobData.ExecuteNext(firstValue, value);
}
entryIndex = nextPtrs[entryIndex];
}
}
}
}
public JobHandle Dispose(JobHandle inputDeps)
{
var jobHandle = new UnsafeHashMapDisposeJob {
Data = m_Buffer, Allocator = m_AllocatorLabel
}.Schedule(inputDeps);
m_Buffer = null;
return(jobHandle);
}
internal static unsafe bool SetValue(UnsafeHashMapData *data, ref NativeMultiHashMapIterator <TKey> it, ref TValue item)
{
int entryIdx = it.EntryIndex;
if (entryIdx < 0 || entryIdx >= data->keyCapacity)
{
return(false);
}
UnsafeUtility.WriteArrayElement(data->values, entryIdx, item);
return(true);
}
internal static unsafe void Clear(UnsafeHashMapData *data)
{
UnsafeUtility.MemSet(data->buckets, 0xff, (data->bucketCapacityMask + 1) * 4);
UnsafeUtility.MemSet(data->next, 0xff, (data->keyCapacity) * 4);
for (int tls = 0; tls < JobsUtility.MaxJobThreadCount; ++tls)
{
data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine] = -1;
}
data->allocatedIndexLength = 0;
}
static void EvaluateBucketImp(int maskedHash, UnsafeHashMapData *data)
{
Assert.IsTrue(maskedHash >= 0 && maskedHash <= data->bucketCapacityMask, "Masked Hash Code out of range!");
int entryIndex = data->buckets[maskedHash];
var pNextIndexChain = data->next;
while (entryIndex != -1)
{
UnsafeUtility.ReadArrayElement <TAggregator>(data->values, entryIndex).Evaluate();
entryIndex = pNextIndexChain[entryIndex];
}
}
internal static int CountImp(UnsafeHashMapData *data)
{
int *pNextIndexes = (int *)data->next;
int freeListSize = 0;
for (int tls = 0; tls < JobsUtility.MaxJobThreadCount; ++tls)
{
int freeIdx = data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine];
for (; freeIdx >= 0; freeIdx = pNextIndexes[freeIdx])
{
++freeListSize;
}
}
return(math.min(data->keyCapacity, data->allocatedIndexLength) - freeListSize);
}
internal static unsafe int Remove(UnsafeHashMapData *data, TKey key, bool isMultiHashMap)
{
var removed = 0;
// First find the slot based on the hash
var buckets = (int *)data->buckets;
var pNextIndexChain = (int *)data->next;
var bucket = key.GetHashCode() & data->bucketCapacityMask;
var prevEntry = -1;
var entryIdx = buckets[bucket];
while (entryIdx >= 0 && entryIdx < data->keyCapacity)
{
if (UnsafeUtility.ReadArrayElement <TKey>(data->keys, entryIdx).Equals(key))
{
++removed;
// Found matching element, remove it
if (prevEntry < 0)
{
buckets[bucket] = pNextIndexChain[entryIdx];
}
else
{
pNextIndexChain[prevEntry] = pNextIndexChain[entryIdx];
}
// And free the index
int nextIdx = pNextIndexChain[entryIdx];
pNextIndexChain[entryIdx] = data->firstFreeTLS[0];
data->firstFreeTLS[0] = entryIdx;
entryIdx = nextIdx;
// Can only be one hit in regular hashmaps, so return
if (!isMultiHashMap)
{
break;
}
}
else
{
prevEntry = entryIdx;
entryIdx = pNextIndexChain[entryIdx];
}
}
return(removed);
}
internal static unsafe bool TryGetFirstValueAtomic(UnsafeHashMapData *data, TKey key, out TValue item, out NativeMultiHashMapIterator <TKey> it)
{
it.key = key;
if (data->allocatedIndexLength <= 0)
{
it.EntryIndex = it.NextEntryIndex = -1;
item = default;
return(false);
}
// First find the slot based on the hash
int *buckets = (int *)data->buckets;
int bucket = key.GetHashCode() & data->bucketCapacityMask;
it.EntryIndex = it.NextEntryIndex = buckets[bucket];
return(TryGetNextValueAtomic(data, out item, ref it));
}
internal static unsafe void AddAtomicMulti(UnsafeHashMapData *data, TKey key, TValue item, int threadIndex)
{
// Allocate an entry from the free list
int idx = AllocEntry(data, threadIndex);
// Write the new value to the entry
UnsafeUtility.WriteArrayElement(data->keys, idx, key);
UnsafeUtility.WriteArrayElement(data->values, idx, item);
int bucket = key.GetHashCode() & data->bucketCapacityMask;
// Add the index to the hash-map
int *buckets = (int *)data->buckets;
int nextPtr;
int *pNextIndexChain = (int *)data->next;
do
{
nextPtr = buckets[bucket];
pNextIndexChain[idx] = nextPtr;
}while (Interlocked.CompareExchange(ref buckets[bucket], idx, nextPtr) != nextPtr);
}
internal static unsafe bool TryAddAtomic(UnsafeHashMapData *data, TKey key, TValue item, int threadIndex)
{
if (TryGetFirstValueAtomic(data, key, out _, out _))
{
return(false);
}
// Allocate an entry from the free list
int idx = AllocEntry(data, threadIndex);
// Write the new value to the entry
UnsafeUtility.WriteArrayElement(data->keys, idx, key);
UnsafeUtility.WriteArrayElement(data->values, idx, item);
int bucket = key.GetHashCode() & data->bucketCapacityMask;
// Add the index to the hash-map
int *buckets = (int *)data->buckets;
if (Interlocked.CompareExchange(ref buckets[bucket], idx, -1) != -1)
{
int *pNextIndexChain = (int *)data->next;
do
{
pNextIndexChain[idx] = buckets[bucket];
if (TryGetFirstValueAtomic(data, key, out _, out _))
{
// Put back the entry in the free list if someone else added it while trying to add
do
{
pNextIndexChain[idx] = data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine];
}while (Interlocked.CompareExchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], idx, pNextIndexChain[idx]) != pNextIndexChain[idx]);
return(false);
}
}while (Interlocked.CompareExchange(ref buckets[bucket], idx, pNextIndexChain[idx]) != pNextIndexChain[idx]);
}
return(true);
}
internal static void AllocateHashMap <TKey, TValue>(int length, int bucketLength, Allocator label, out UnsafeHashMapData *outBuf)
where TKey : struct
where TValue : struct
{
IsBlittableAndThrow <TKey, TValue>();
UnsafeHashMapData *data = (UnsafeHashMapData *)UnsafeUtility.Malloc(sizeof(UnsafeHashMapData), UnsafeUtility.AlignOf <UnsafeHashMapData>(), label);
bucketLength = math.ceilpow2(bucketLength);
data->keyCapacity = length;
data->bucketCapacityMask = bucketLength - 1;
int keyOffset, nextOffset, bucketOffset;
int totalSize = CalculateDataSize <TKey, TValue>(length, bucketLength, out keyOffset, out nextOffset, out bucketOffset);
data->values = (byte *)UnsafeUtility.Malloc(totalSize, JobsUtility.CacheLineSize, label);
data->keys = data->values + keyOffset;
data->next = data->values + nextOffset;
data->buckets = data->values + bucketOffset;
outBuf = data;
}
/// <summary>
/// Disposes of this multi-hashmap and deallocates its memory immediately.
/// </summary>
public void Dispose()
{
UnsafeHashMapData.DeallocateHashMap(m_Buffer, m_AllocatorLabel);
m_Buffer = null;
}
internal static void ReallocateHashMap <TKey, TValue>(UnsafeHashMapData *data, int newCapacity, int newBucketCapacity, Allocator label)
where TKey : struct
where TValue : struct
{
newBucketCapacity = math.ceilpow2(newBucketCapacity);
if (data->keyCapacity == newCapacity && (data->bucketCapacityMask + 1) == newBucketCapacity)
{
return;
}
if (data->keyCapacity > newCapacity)
{
throw new Exception("Shrinking a hash map is not supported");
}
int keyOffset, nextOffset, bucketOffset;
int totalSize = CalculateDataSize <TKey, TValue>(newCapacity, newBucketCapacity, out keyOffset, out nextOffset, out bucketOffset);
byte *newData = (byte *)UnsafeUtility.Malloc(totalSize, JobsUtility.CacheLineSize, label);
byte *newKeys = newData + keyOffset;
byte *newNext = newData + nextOffset;
byte *newBuckets = newData + bucketOffset;
// The items are taken from a free-list and might not be tightly packed, copy all of the old capcity
UnsafeUtility.MemCpy(newData, data->values, data->keyCapacity * UnsafeUtility.SizeOf <TValue>());
UnsafeUtility.MemCpy(newKeys, data->keys, data->keyCapacity * UnsafeUtility.SizeOf <TKey>());
UnsafeUtility.MemCpy(newNext, data->next, data->keyCapacity * UnsafeUtility.SizeOf <int>());
for (int emptyNext = data->keyCapacity; emptyNext < newCapacity; ++emptyNext)
{
((int *)newNext)[emptyNext] = -1;
}
// re-hash the buckets, first clear the new bucket list, then insert all values from the old list
for (int bucket = 0; bucket < newBucketCapacity; ++bucket)
{
((int *)newBuckets)[bucket] = -1;
}
for (int bucket = 0; bucket <= data->bucketCapacityMask; ++bucket)
{
int *buckets = (int *)data->buckets;
int *pNextIndexChain = (int *)newNext;
while (buckets[bucket] >= 0)
{
int curEntry = buckets[bucket];
buckets[bucket] = pNextIndexChain[curEntry];
int newBucket = UnsafeUtility.ReadArrayElement <TKey>(data->keys, curEntry).GetHashCode() & (newBucketCapacity - 1);
pNextIndexChain[curEntry] = ((int *)newBuckets)[newBucket];
((int *)newBuckets)[newBucket] = curEntry;
}
}
UnsafeUtility.Free(data->values, label);
if (data->allocatedIndexLength > data->keyCapacity)
{
data->allocatedIndexLength = data->keyCapacity;
}
data->values = newData;
data->keys = newKeys;
data->next = newNext;
data->buckets = newBuckets;
data->keyCapacity = newCapacity;
data->bucketCapacityMask = newBucketCapacity - 1;
}
internal static void DeallocateHashMap(UnsafeHashMapData *data, Allocator allocator)
{
UnsafeUtility.Free(data->values, allocator);
UnsafeUtility.Free(data, allocator);
}
internal static unsafe int AllocEntry(UnsafeHashMapData *data, int threadIndex)
{
int idx;
int *pNextIndexChain = (int *)data->next;
do
{
idx = data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine];
if (idx < 0)
{
// Try to refill local cache
Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], -2);
// If it failed try to get one from the never-allocated array
if (data->allocatedIndexLength < data->keyCapacity)
{
idx = Interlocked.Add(ref data->allocatedIndexLength, 16) - 16;
if (idx < data->keyCapacity - 1)
{
int count = math.min(16, data->keyCapacity - idx);
for (int i = 1; i < count; ++i)
{
pNextIndexChain[idx + i] = idx + i + 1;
}
pNextIndexChain[idx + count - 1] = -1;
pNextIndexChain[idx] = -1;
Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], idx + 1);
return(idx);
}
if (idx == data->keyCapacity - 1)
{
Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], -1);
return(idx);
}
}
Interlocked.Exchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], -1);
// Failed to get any, try to get one from another free list
bool again = true;
while (again)
{
again = false;
for (int other = (threadIndex + 1) % JobsUtility.MaxJobThreadCount; other != threadIndex; other = (other + 1) % JobsUtility.MaxJobThreadCount)
{
do
{
idx = data->firstFreeTLS[other * UnsafeHashMapData.IntPerCacheLine];
if (idx < 0)
{
break;
}
}while (Interlocked.CompareExchange(ref data->firstFreeTLS[other * UnsafeHashMapData.IntPerCacheLine], pNextIndexChain[idx], idx) != idx);
if (idx == -2)
{
again = true;
}
else if (idx >= 0)
{
pNextIndexChain[idx] = -1;
return(idx);
}
}
}
throw new InvalidOperationException("HashMap is full");
}
if (idx >= data->keyCapacity)
{
throw new InvalidOperationException(string.Format("nextPtr idx {0} beyond capacity {1}", idx, data->keyCapacity));
}
}while (Interlocked.CompareExchange(ref data->firstFreeTLS[threadIndex * UnsafeHashMapData.IntPerCacheLine], pNextIndexChain[idx], idx) != idx);
pNextIndexChain[idx] = -1;
return(idx);
}
internal static unsafe bool TryAdd(UnsafeHashMapData *data, TKey key, TValue item, bool isMultiHashMap, Allocator allocation)
{
if (!isMultiHashMap && TryGetFirstValueAtomic(data, key, out _, out _))
{
return(false);
}
// Allocate an entry from the free list
int idx;
int *pNextIndexChain;
if (data->allocatedIndexLength >= data->keyCapacity && data->firstFreeTLS[0] < 0)
{
for (int tls = 1; tls < JobsUtility.MaxJobThreadCount; ++tls)
{
if (data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine] >= 0)
{
idx = data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine];
pNextIndexChain = (int *)data->next;
data->firstFreeTLS[tls * UnsafeHashMapData.IntPerCacheLine] = pNextIndexChain[idx];
pNextIndexChain[idx] = -1;
data->firstFreeTLS[0] = idx;
break;
}
}
if (data->firstFreeTLS[0] < 0)
{
int newCap = UnsafeHashMapData.GrowCapacity(data->keyCapacity);
UnsafeHashMapData.ReallocateHashMap <TKey, TValue>(data, newCap, UnsafeHashMapData.GetBucketSize(newCap), allocation);
}
}
idx = data->firstFreeTLS[0];
if (idx >= 0)
{
data->firstFreeTLS[0] = ((int *)data->next)[idx];
}
else
{
idx = data->allocatedIndexLength++;
}
if (idx < 0 || idx >= data->keyCapacity)
{
throw new InvalidOperationException("Internal HashMap error");
}
// Write the new value to the entry
UnsafeUtility.WriteArrayElement(data->keys, idx, key);
UnsafeUtility.WriteArrayElement(data->values, idx, item);
int bucket = key.GetHashCode() & data->bucketCapacityMask;
// Add the index to the hash-map
int *buckets = (int *)data->buckets;
pNextIndexChain = (int *)data->next;
pNextIndexChain[idx] = buckets[bucket];
buckets[bucket] = idx;
return(true);
}
