private unsafe void BeginMainIndexRecovery(
int version,
IDevice device,
ulong num_bytes,
bool isAsync = false)
{
long totalSize = state[version].size * sizeof(HashBucket);
int numChunks = 1;
if (totalSize > uint.MaxValue)
{
numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue);
numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2)));
}
uint chunkSize = (uint)(totalSize / numChunks);
recoveryCountdown = new CountdownWrapper(numChunks, isAsync);
HashBucket *start = state[version].tableAligned;
ulong numBytesRead = 0;
for (int index = 0; index < numChunks; index++)
{
IntPtr chunkStartBucket = (IntPtr)(((byte *)start) + (index * chunkSize));
HashIndexPageAsyncReadResult result = default;
result.chunkIndex = index;
device.ReadAsync(numBytesRead, chunkStartBucket, chunkSize, AsyncPageReadCallback, result);
numBytesRead += chunkSize;
}
Debug.Assert(numBytesRead == num_bytes);
}
private unsafe void BeginMainIndexCheckpoint(
int version,
IDevice device,
out ulong numBytesWritten)
{
long totalSize = state[version].size * sizeof(HashBucket);
int numChunks = 1;
if (totalSize > uint.MaxValue)
{
numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue);
numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2)));
}
uint chunkSize = (uint)(totalSize / numChunks);
mainIndexCheckpointCallbackCount = numChunks;
mainIndexCheckpointSemaphore = new SemaphoreSlim(0);
HashBucket *start = state[version].tableAligned;
numBytesWritten = 0;
for (int index = 0; index < numChunks; index++)
{
long chunkStartBucket = (long)start + (index * chunkSize);
HashIndexPageAsyncFlushResult result = default;
result.chunkIndex = index;
device.WriteAsync((IntPtr)chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result);
numBytesWritten += chunkSize;
}
}
private unsafe void BeginMainIndexCheckpoint(
int version,
IDevice device,
out ulong numBytesWritten)
{
int numChunks = 1;
long totalSize = state[version].size * sizeof(HashBucket);
Debug.Assert(totalSize < (long)uint.MaxValue); // required since numChunks = 1
uint chunkSize = (uint)(totalSize / numChunks);
mainIndexCheckpointCallbackCount = numChunks;
mainIndexCheckpointSemaphore = new SemaphoreSlim(0);
HashBucket *start = state[version].tableAligned;
numBytesWritten = 0;
for (int index = 0; index < numChunks; index++)
{
long chunkStartBucket = (long)start + (index * chunkSize);
HashIndexPageAsyncFlushResult result = default;
result.chunkIndex = index;
device.WriteAsync((IntPtr)chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result);
numBytesWritten += chunkSize;
}
}
private void BeginMainIndexRecovery(
int version,
IDevice device,
ulong num_bytes)
{
int numChunksToBeRecovered = 1;
long totalSize = state[version].size * sizeof(HashBucket);
Debug.Assert(totalSize < (long)uint.MaxValue); // required since numChunks = 1
uint chunkSize = (uint)(totalSize / numChunksToBeRecovered);
mainIndexRecoveryEvent = new CountdownEvent(numChunksToBeRecovered);
HashBucket *start = state[version].tableAligned;
ulong numBytesRead = 0;
for (int index = 0; index < numChunksToBeRecovered; index++)
{
long chunkStartBucket = (long)start + (index * chunkSize);
HashIndexPageAsyncReadResult result = default(HashIndexPageAsyncReadResult);
result.chunkIndex = index;
device.ReadAsync(numBytesRead, (IntPtr)chunkStartBucket, chunkSize, AsyncPageReadCallback, result);
numBytesRead += chunkSize;
}
Debug.Assert(numBytesRead == num_bytes);
}
public static void ReleaseExclusiveLatch(HashBucket *bucket)
{
long expected_word = bucket->bucket_entries[Constants.kOverflowBucketIndex];
long desired_word = expected_word & Constants.kAddressMask;
var found_word = Interlocked.Exchange(
ref bucket->bucket_entries[Constants.kOverflowBucketIndex],
desired_word);
}
internal unsafe void BeginMainIndexCheckpoint(int version, IDevice device, out ulong numBytesWritten, bool useReadCache = false, SkipReadCache skipReadCache = default)
{
long totalSize = state[version].size * sizeof(HashBucket);
int numChunks = 1;
if (useReadCache && (totalSize > (1L << 25)))
{
numChunks = (int)Math.Ceiling((double)totalSize / (1L << 25));
numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2)));
}
else if (totalSize > uint.MaxValue)
{
numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue);
numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2)));
}
uint chunkSize = (uint)(totalSize / numChunks);
mainIndexCheckpointCallbackCount = numChunks;
mainIndexCheckpointSemaphore = new SemaphoreSlim(0);
HashBucket *start = state[version].tableAligned;
numBytesWritten = 0;
for (int index = 0; index < numChunks; index++)
{
IntPtr chunkStartBucket = (IntPtr)((byte *)start + (index * chunkSize));
HashIndexPageAsyncFlushResult result = default;
result.chunkIndex = index;
if (!useReadCache)
{
device.WriteAsync(chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result);
}
else
{
result.mem = new SectorAlignedMemory((int)chunkSize, (int)device.SectorSize);
bool prot = false;
if (!epoch.ThisInstanceProtected())
{
prot = true;
epoch.Resume();
}
Buffer.MemoryCopy((void *)chunkStartBucket, result.mem.aligned_pointer, chunkSize, chunkSize);
for (int j = 0; j < chunkSize; j += sizeof(HashBucket))
{
skipReadCache((HashBucket *)(result.mem.aligned_pointer + j));
}
if (prot)
{
epoch.Suspend();
}
device.WriteAsync((IntPtr)result.mem.aligned_pointer, numBytesWritten, chunkSize, AsyncPageFlushCallback, result);
}
numBytesWritten += chunkSize;
}
}
internal bool UpdateSlot(HashBucket *bucket, int entrySlot, long expected, long desired, out long found)
{
found = Interlocked.CompareExchange(
ref bucket->bucket_entries[entrySlot],
desired,
expected);
return(found == expected);
}
public static bool TryAcquireExclusiveLatch(HashBucket *bucket)
{
long expected_word = bucket->bucket_entries[Constants.kOverflowBucketIndex];
if ((expected_word & ~Constants.kAddressMask) == 0)
{
long desired_word = expected_word | kExclusiveLatchBitMask;
var found_word = Interlocked.CompareExchange(
ref bucket->bucket_entries[Constants.kOverflowBucketIndex],
desired_word,
expected_word);
return(found_word == expected_word);
}
return(false);
}
internal bool FindTag(long hash, ushort tag, ref HashBucket *bucket, ref int slot, ref HashBucketEntry entry)
{
var target_entry_word = default(long);
var entry_slot_bucket = default(HashBucket *);
var version = resizeInfo.version;
var masked_entry_word = hash & state[version].size_mask;
bucket = state[version].tableAligned + masked_entry_word;
slot = Constants.kInvalidEntrySlot;
do
{
// Search through the bucket looking for our key. Last entry is reserved
// for the overflow pointer.
for (int index = 0; index < Constants.kOverflowBucketIndex; ++index)
{
target_entry_word = *(((long *)bucket) + index);
if (0 == target_entry_word)
{
continue;
}
entry.word = target_entry_word;
if (tag == entry.Tag)
{
slot = index;
// If (final key, return immediately)
//if ((entry.tag & ~Constants.kTagMask) == 0) -- Guna: Is this correct?
if (!entry.Tentative)
{
return(true);
}
}
}
target_entry_word = *(((long *)bucket) + Constants.kOverflowBucketIndex) & Constants.kAddressMask;
// Go to next bucket in the chain
if (target_entry_word == 0)
{
entry = default(HashBucketEntry);
return(false);
}
bucket = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word);
} while (true);
}
private void BeginMainIndexCheckpoint(
int version,
IDevice device,
out ulong numBytesWritten)
{
long totalSize = state[version].size * sizeof(HashBucket);
int numChunks = (int)(totalSize >> 23);
if (numChunks == 0)
{
numChunks = 1;
}
uint chunkSize = (uint)(totalSize / numChunks);
mainIndexCheckpointEvent = new CountdownEvent(1);
HashBucket *start = state[version].tableAligned;
int startNumChunks = 1;
var sw = new Stopwatch();
HashIndexPageAsyncFlushResult result = new HashIndexPageAsyncFlushResult
{
start = start,
numChunks = numChunks,
numIssued = startNumChunks,
numFinished = 0,
chunkSize = chunkSize,
device = device,
sw = sw
};
numBytesWritten = 0;
sw.Start();
for (int index = 0; index < startNumChunks; index++)
{
long chunkStartBucket = (long)start + (index * chunkSize);
//sw.Restart();
device.WriteAsync((IntPtr)chunkStartBucket, ((ulong)index) * chunkSize, chunkSize, AsyncPageFlushCallback, result);
//sw.Stop();
//Console.WriteLine("Initial WriteAsync {0}: {1}", index, sw.ElapsedMilliseconds);
}
numBytesWritten = ((ulong)numChunks) * chunkSize;
}
private bool FindOtherTagMaybeTentativeInternal(long hash, ushort tag, ref HashBucket *bucket, ref int slot, HashBucket *except_bucket, int except_entry_slot)
{
var target_entry_word = default(long);
var entry_slot_bucket = default(HashBucket *);
do
{
// Search through the bucket looking for our key. Last entry is reserved
// for the overflow pointer.
for (int index = 0; index < Constants.kOverflowBucketIndex; ++index)
{
target_entry_word = *(((long *)bucket) + index);
if (0 == target_entry_word)
{
continue;
}
HashBucketEntry entry = default(HashBucketEntry);
entry.word = target_entry_word;
if (tag == entry.Tag)
{
if ((except_entry_slot == index) && (except_bucket == bucket))
{
continue;
}
slot = index;
return(true);
}
}
target_entry_word = *(((long *)bucket) + Constants.kOverflowBucketIndex) & Constants.kAddressMask;
// Go to next bucket in the chain
if (target_entry_word == 0)
{
return(false);
}
bucket = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word);
} while (true);
}
internal void FindOrCreateTag(long hash, ushort tag, ref HashBucket *bucket, ref int slot, ref HashBucketEntry entry, long BeginAddress)
{
var version = resizeInfo.version;
var masked_entry_word = hash & state[version].size_mask;
while (true)
{
bucket = state[version].tableAligned + masked_entry_word;
slot = Constants.kInvalidEntrySlot;
if (FindTagOrFreeInternal(hash, tag, ref bucket, ref slot, ref entry, BeginAddress))
{
return;
}
// Install tentative tag in free slot
entry = default(HashBucketEntry);
entry.Tag = tag;
entry.Address = Constants.kTempInvalidAddress;
entry.Pending = false;
entry.Tentative = true;
if (0 == Interlocked.CompareExchange(ref bucket->bucket_entries[slot], entry.word, 0))
{
var orig_bucket = state[version].tableAligned + masked_entry_word;
var orig_slot = Constants.kInvalidEntrySlot;
if (FindOtherTagMaybeTentativeInternal(hash, tag, ref orig_bucket, ref orig_slot, bucket, slot))
{
bucket->bucket_entries[slot] = 0;
}
else
{
entry.Tentative = false;
*((long *)bucket + slot) = entry.word;
break;
}
}
}
}
private void _BeginMainIndexRecovery(
int version,
IDevice device,
ulong num_bytes)
{
numChunksToBeRecovered = 1; // Constants.kNumMergeChunks;
long chunkSize = state[version].size / 1; // Constants.kNumMergeChunks;
HashBucket *start = state[version].tableAligned;
uint sizeOfPage = (uint)chunkSize * (uint)sizeof(HashBucket);
uint num_bytes_read = 0;
for (int index = 0; index < 1 /*Constants.kNumMergeChunks*/; index++)
{
HashBucket *chunkStartBucket = start + (index * chunkSize);
HashIndexPageAsyncReadResult result = default(HashIndexPageAsyncReadResult);
result.chunkIndex = index;
device.ReadAsync(num_bytes_read, (IntPtr)chunkStartBucket, sizeOfPage, AsyncPageReadCallback, result);
num_bytes_read += sizeOfPage;
}
Debug.Assert(num_bytes_read == num_bytes);
}
private void BeginMainIndexCheckpoint(
int version,
IDevice device,
out ulong numBytesWritten)
{
long totalSize = state[version].size * sizeof(HashBucket);
int numChunks = (int)(totalSize >> 23);
if (numChunks == 0)
{
numChunks = 1;
}
uint chunkSize = (uint)(totalSize / numChunks);
mainIndexCheckpointEvent = new CountdownEvent(1);
HashBucket *start = state[version].tableAligned;
int startNumChunks = 1;
HashIndexPageAsyncFlushResult result = new HashIndexPageAsyncFlushResult
{
start = start,
numChunks = numChunks,
numIssued = startNumChunks,
numFinished = 0,
chunkSize = chunkSize,
device = device
};
numBytesWritten = 0;
for (int index = 0; index < startNumChunks; index++)
{
long chunkStartBucket = (long)start + (index * chunkSize);
device.WriteAsync((IntPtr)chunkStartBucket, ((ulong)index) * chunkSize, chunkSize, AsyncPageFlushCallback, result);
}
numBytesWritten = ((ulong)numChunks) * chunkSize;
}
public static void ReleaseSharedLatch(HashBucket *bucket)
{
Interlocked.Add(ref bucket->bucket_entries[Constants.kOverflowBucketIndex],
-kPinConstant);
}
public static bool TryAcquireSharedLatch(HashBucket *bucket)
{
return(Interlocked.Add(ref bucket->bucket_entries[Constants.kOverflowBucketIndex],
kPinConstant) > 0);
}
private HashBucket* GetBucket()
{
var bucket = _hb_cp;
_hb_cp = bucket + (iHSubMask + 1);
if (_hb_cp <= _hb_ep)
{
_cbuckets++;
return bucket;
}
uint hbSize;
if (_cbuckets < 2048)
{
_hb_cp = (HashBucket*)GetBig();
hbSize = iBigBlockSize + SizeofRcp;
}
else
{
var hash = new LargePage(_virtual_memory, 4) { next = _lpHash };
hbSize = hash.size;
_lpHash = hash;
_hb_cp = (HashBucket*)hash.data;
}
_hb_ep = (HashBucket*)((byte*)_hb_cp + hbSize);
return GetBucket();
}
private bool FindTagOrFreeInternal(long hash, ushort tag, ref HashBucket *bucket, ref int slot, ref HashBucketEntry entry, long BeginAddress = 0)
{
var target_entry_word = default(long);
var recordExists = false;
var entry_slot_bucket = default(HashBucket *);
do
{
// Search through the bucket looking for our key. Last entry is reserved
// for the overflow pointer.
for (int index = 0; index < Constants.kOverflowBucketIndex; ++index)
{
target_entry_word = *(((long *)bucket) + index);
if (0 == target_entry_word)
{
if (slot == Constants.kInvalidEntrySlot)
{
slot = index;
entry_slot_bucket = bucket;
}
continue;
}
entry.word = target_entry_word;
if (entry.Address < BeginAddress && entry.Address != Constants.kTempInvalidAddress)
{
if (entry.word == Interlocked.CompareExchange(ref bucket->bucket_entries[index], Constants.kInvalidAddress, target_entry_word))
{
if (slot == Constants.kInvalidEntrySlot)
{
slot = index;
entry_slot_bucket = bucket;
}
continue;
}
}
if (tag == entry.Tag && !entry.Tentative)
{
slot = index;
recordExists = true;
return(recordExists);
}
}
target_entry_word = *(((long *)bucket) + Constants.kOverflowBucketIndex);
// Go to next bucket in the chain
if ((target_entry_word & Constants.kAddressMask) == 0)
{
if (slot == Constants.kInvalidEntrySlot)
{
// Allocate new bucket
var logicalBucketAddress = overflowBucketsAllocator.Allocate();
var physicalBucketAddress = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(logicalBucketAddress);
long compare_word = target_entry_word;
target_entry_word = logicalBucketAddress;
target_entry_word |= (compare_word & ~Constants.kAddressMask);
long result_word = Interlocked.CompareExchange(
ref bucket->bucket_entries[Constants.kOverflowBucketIndex],
target_entry_word,
compare_word);
if (compare_word != result_word)
{
// Install failed, undo allocation; use the winner's entry
overflowBucketsAllocator.Free(logicalBucketAddress);
target_entry_word = result_word;
}
else
{
// Install succeeded
bucket = physicalBucketAddress;
slot = 0;
entry = default(HashBucketEntry);
return(recordExists);
}
}
else
{
if (!recordExists)
{
bucket = entry_slot_bucket;
}
entry = default(HashBucketEntry);
break;
}
}
bucket = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word & Constants.kAddressMask);
} while (true);
return(recordExists);
}
private void ResetSlabs()
{
_cbuckets = 0;
// hash bucket extra mem pages are returned to system on reset.
if (_lpHash != null)
{
for (var ls = _lpHash; ls != null; ls = ls.next)
ls.Release();
_lpHash = null;
}
if (_lpBase == null)
_lpBase = new LargePage(_virtual_memory, 4);
_htable = (HashBucket*)_lpBase.data;
// init shared tables with clean values.
for (var i = 0; i < _config.HashTableSize; i++)
_htable[i] = new HashBucket();
var bp = (byte*)(_htable + _config.HashTableSize);
_stabs = (int*)bp;
// encoding slab max size and pos like -> 32 : 0
_stabs[0] = 0x200000; _stabs[1] = 0x400001;
_slabs = (SlabList*)_align((byte*)(_stabs + 1 + (iLastSlabSize + SizeofRcp) / iQuantum));
// calculate slabs indexes and max sizes.
int sbts = 64, scnt = 2, pos = scnt;
for (var half = sbts / 2; sbts < iLastSlabSize; half = sbts / 2)
{
for (var es = sbts + half; sbts < es; sbts += iQuantum) _stabs[pos++] = scnt | (es << 16);
scnt++;
for (var es = sbts + half; sbts < es; sbts += iQuantum) _stabs[pos++] = scnt | (es << 16);
scnt++;
}
_stabs[pos] = scnt;
bp = _align((byte*)(_slabs + scnt + 1));
// pre-allocate ~32k into every slab slot.
var quot = sbts / 4;
while (scnt-- > 0)
{
RefCountPtr* ls = null;
var count = 0;
for (var todo = 32768; todo > 0; count++, todo -= sbts, bp += sbts)
{
var rc = (RefCountPtr*)bp;
rc->_list = ls; ls = rc;
}
_slabs[scnt].Set(ls, count);
sbts -= quot;
if ((scnt & 1) != 0) quot = sbts / 4;
}
// pre-allocate 256k into hash table buckets;
_hb_cp = (HashBucket*)bp;
_hb_ep = (HashBucket*)(bp += 256 * 1024);
// pre-allocate ~1mb into big-mem chunks.
_sbigs = null; _cntBigs = 0; sbts = iBigBlockSize + SizeofRcp;
for (int left = 1024 * (1024 + 16); left > 0; _cntBigs++, left -= sbts, bp += sbts)
{
var rc = (RefCountPtr*)bp;
rc->_list = _sbigs; _sbigs = rc;
}
// pre-allocate large memory pages for cache purposes.
if (!_config.ReserveMemory)
_cntPages = 1 + (int)((Stats.MemoryLimit - 1) / _config.AllocPageSize);
if (_lpList != null)
if (_config.ReserveMemory) { }
else
{
for (var ls = _lpList; ls != null; ls = ls.next)
ls.Release();
_lpList = null;
}
else if (_config.ReserveMemory)
for (var bytes = Stats.MemoryLimit; bytes > 0; bytes -= _lpList.size)
_lpList = new LargePage(_virtual_memory, _config.AllocPageSize) { next = _lpList };
_lpNext = _lpList;
_cp = bp; _ep = _lpBase.data + _lpBase.size;
}
public static bool NoSharedLatches(HashBucket *bucket)
{
long word = bucket->bucket_entries[Constants.kOverflowBucketIndex];
return((word & ~Constants.kAddressMask) == 0);
}
