private unsafe void BeginMainIndexRecovery( int version, IDevice device, ulong num_bytes, bool isAsync = false) { long totalSize = state[version].size * sizeof(HashBucket); int numChunks = 1; if (totalSize > uint.MaxValue) { numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue); numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); } uint chunkSize = (uint)(totalSize / numChunks); recoveryCountdown = new CountdownWrapper(numChunks, isAsync); HashBucket *start = state[version].tableAligned; ulong numBytesRead = 0; for (int index = 0; index < numChunks; index++) { IntPtr chunkStartBucket = (IntPtr)(((byte *)start) + (index * chunkSize)); HashIndexPageAsyncReadResult result = default; result.chunkIndex = index; device.ReadAsync(numBytesRead, chunkStartBucket, chunkSize, AsyncPageReadCallback, result); numBytesRead += chunkSize; } Debug.Assert(numBytesRead == num_bytes); }
private unsafe void BeginMainIndexCheckpoint( int version, IDevice device, out ulong numBytesWritten) { long totalSize = state[version].size * sizeof(HashBucket); int numChunks = 1; if (totalSize > uint.MaxValue) { numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue); numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); } uint chunkSize = (uint)(totalSize / numChunks); mainIndexCheckpointCallbackCount = numChunks; mainIndexCheckpointSemaphore = new SemaphoreSlim(0); HashBucket *start = state[version].tableAligned; numBytesWritten = 0; for (int index = 0; index < numChunks; index++) { long chunkStartBucket = (long)start + (index * chunkSize); HashIndexPageAsyncFlushResult result = default; result.chunkIndex = index; device.WriteAsync((IntPtr)chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); numBytesWritten += chunkSize; } }
private unsafe void BeginMainIndexCheckpoint( int version, IDevice device, out ulong numBytesWritten) { int numChunks = 1; long totalSize = state[version].size * sizeof(HashBucket); Debug.Assert(totalSize < (long)uint.MaxValue); // required since numChunks = 1 uint chunkSize = (uint)(totalSize / numChunks); mainIndexCheckpointCallbackCount = numChunks; mainIndexCheckpointSemaphore = new SemaphoreSlim(0); HashBucket *start = state[version].tableAligned; numBytesWritten = 0; for (int index = 0; index < numChunks; index++) { long chunkStartBucket = (long)start + (index * chunkSize); HashIndexPageAsyncFlushResult result = default; result.chunkIndex = index; device.WriteAsync((IntPtr)chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); numBytesWritten += chunkSize; } }
private void BeginMainIndexRecovery( int version, IDevice device, ulong num_bytes) { int numChunksToBeRecovered = 1; long totalSize = state[version].size * sizeof(HashBucket); Debug.Assert(totalSize < (long)uint.MaxValue); // required since numChunks = 1 uint chunkSize = (uint)(totalSize / numChunksToBeRecovered); mainIndexRecoveryEvent = new CountdownEvent(numChunksToBeRecovered); HashBucket *start = state[version].tableAligned; ulong numBytesRead = 0; for (int index = 0; index < numChunksToBeRecovered; index++) { long chunkStartBucket = (long)start + (index * chunkSize); HashIndexPageAsyncReadResult result = default(HashIndexPageAsyncReadResult); result.chunkIndex = index; device.ReadAsync(numBytesRead, (IntPtr)chunkStartBucket, chunkSize, AsyncPageReadCallback, result); numBytesRead += chunkSize; } Debug.Assert(numBytesRead == num_bytes); }
public static void ReleaseExclusiveLatch(HashBucket *bucket) { long expected_word = bucket->bucket_entries[Constants.kOverflowBucketIndex]; long desired_word = expected_word & Constants.kAddressMask; var found_word = Interlocked.Exchange( ref bucket->bucket_entries[Constants.kOverflowBucketIndex], desired_word); }
internal unsafe void BeginMainIndexCheckpoint(int version, IDevice device, out ulong numBytesWritten, bool useReadCache = false, SkipReadCache skipReadCache = default) { long totalSize = state[version].size * sizeof(HashBucket); int numChunks = 1; if (useReadCache && (totalSize > (1L << 25))) { numChunks = (int)Math.Ceiling((double)totalSize / (1L << 25)); numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); } else if (totalSize > uint.MaxValue) { numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue); numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); } uint chunkSize = (uint)(totalSize / numChunks); mainIndexCheckpointCallbackCount = numChunks; mainIndexCheckpointSemaphore = new SemaphoreSlim(0); HashBucket *start = state[version].tableAligned; numBytesWritten = 0; for (int index = 0; index < numChunks; index++) { IntPtr chunkStartBucket = (IntPtr)((byte *)start + (index * chunkSize)); HashIndexPageAsyncFlushResult result = default; result.chunkIndex = index; if (!useReadCache) { device.WriteAsync(chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); } else { result.mem = new SectorAlignedMemory((int)chunkSize, (int)device.SectorSize); bool prot = false; if (!epoch.ThisInstanceProtected()) { prot = true; epoch.Resume(); } Buffer.MemoryCopy((void *)chunkStartBucket, result.mem.aligned_pointer, chunkSize, chunkSize); for (int j = 0; j < chunkSize; j += sizeof(HashBucket)) { skipReadCache((HashBucket *)(result.mem.aligned_pointer + j)); } if (prot) { epoch.Suspend(); } device.WriteAsync((IntPtr)result.mem.aligned_pointer, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); } numBytesWritten += chunkSize; } }
internal bool UpdateSlot(HashBucket *bucket, int entrySlot, long expected, long desired, out long found) { found = Interlocked.CompareExchange( ref bucket->bucket_entries[entrySlot], desired, expected); return(found == expected); }
public static bool TryAcquireExclusiveLatch(HashBucket *bucket) { long expected_word = bucket->bucket_entries[Constants.kOverflowBucketIndex]; if ((expected_word & ~Constants.kAddressMask) == 0) { long desired_word = expected_word | kExclusiveLatchBitMask; var found_word = Interlocked.CompareExchange( ref bucket->bucket_entries[Constants.kOverflowBucketIndex], desired_word, expected_word); return(found_word == expected_word); } return(false); }
internal bool FindTag(long hash, ushort tag, ref HashBucket *bucket, ref int slot, ref HashBucketEntry entry) { var target_entry_word = default(long); var entry_slot_bucket = default(HashBucket *); var version = resizeInfo.version; var masked_entry_word = hash & state[version].size_mask; bucket = state[version].tableAligned + masked_entry_word; slot = Constants.kInvalidEntrySlot; do { // Search through the bucket looking for our key. Last entry is reserved // for the overflow pointer. for (int index = 0; index < Constants.kOverflowBucketIndex; ++index) { target_entry_word = *(((long *)bucket) + index); if (0 == target_entry_word) { continue; } entry.word = target_entry_word; if (tag == entry.Tag) { slot = index; // If (final key, return immediately) //if ((entry.tag & ~Constants.kTagMask) == 0) -- Guna: Is this correct? if (!entry.Tentative) { return(true); } } } target_entry_word = *(((long *)bucket) + Constants.kOverflowBucketIndex) & Constants.kAddressMask; // Go to next bucket in the chain if (target_entry_word == 0) { entry = default(HashBucketEntry); return(false); } bucket = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word); } while (true); }
private void BeginMainIndexCheckpoint( int version, IDevice device, out ulong numBytesWritten) { long totalSize = state[version].size * sizeof(HashBucket); int numChunks = (int)(totalSize >> 23); if (numChunks == 0) { numChunks = 1; } uint chunkSize = (uint)(totalSize / numChunks); mainIndexCheckpointEvent = new CountdownEvent(1); HashBucket *start = state[version].tableAligned; int startNumChunks = 1; var sw = new Stopwatch(); HashIndexPageAsyncFlushResult result = new HashIndexPageAsyncFlushResult { start = start, numChunks = numChunks, numIssued = startNumChunks, numFinished = 0, chunkSize = chunkSize, device = device, sw = sw }; numBytesWritten = 0; sw.Start(); for (int index = 0; index < startNumChunks; index++) { long chunkStartBucket = (long)start + (index * chunkSize); //sw.Restart(); device.WriteAsync((IntPtr)chunkStartBucket, ((ulong)index) * chunkSize, chunkSize, AsyncPageFlushCallback, result); //sw.Stop(); //Console.WriteLine("Initial WriteAsync {0}: {1}", index, sw.ElapsedMilliseconds); } numBytesWritten = ((ulong)numChunks) * chunkSize; }
private bool FindOtherTagMaybeTentativeInternal(long hash, ushort tag, ref HashBucket *bucket, ref int slot, HashBucket *except_bucket, int except_entry_slot) { var target_entry_word = default(long); var entry_slot_bucket = default(HashBucket *); do { // Search through the bucket looking for our key. Last entry is reserved // for the overflow pointer. for (int index = 0; index < Constants.kOverflowBucketIndex; ++index) { target_entry_word = *(((long *)bucket) + index); if (0 == target_entry_word) { continue; } HashBucketEntry entry = default(HashBucketEntry); entry.word = target_entry_word; if (tag == entry.Tag) { if ((except_entry_slot == index) && (except_bucket == bucket)) { continue; } slot = index; return(true); } } target_entry_word = *(((long *)bucket) + Constants.kOverflowBucketIndex) & Constants.kAddressMask; // Go to next bucket in the chain if (target_entry_word == 0) { return(false); } bucket = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word); } while (true); }
internal void FindOrCreateTag(long hash, ushort tag, ref HashBucket *bucket, ref int slot, ref HashBucketEntry entry, long BeginAddress) { var version = resizeInfo.version; var masked_entry_word = hash & state[version].size_mask; while (true) { bucket = state[version].tableAligned + masked_entry_word; slot = Constants.kInvalidEntrySlot; if (FindTagOrFreeInternal(hash, tag, ref bucket, ref slot, ref entry, BeginAddress)) { return; } // Install tentative tag in free slot entry = default(HashBucketEntry); entry.Tag = tag; entry.Address = Constants.kTempInvalidAddress; entry.Pending = false; entry.Tentative = true; if (0 == Interlocked.CompareExchange(ref bucket->bucket_entries[slot], entry.word, 0)) { var orig_bucket = state[version].tableAligned + masked_entry_word; var orig_slot = Constants.kInvalidEntrySlot; if (FindOtherTagMaybeTentativeInternal(hash, tag, ref orig_bucket, ref orig_slot, bucket, slot)) { bucket->bucket_entries[slot] = 0; } else { entry.Tentative = false; *((long *)bucket + slot) = entry.word; break; } } } }
private void _BeginMainIndexRecovery( int version, IDevice device, ulong num_bytes) { numChunksToBeRecovered = 1; // Constants.kNumMergeChunks; long chunkSize = state[version].size / 1; // Constants.kNumMergeChunks; HashBucket *start = state[version].tableAligned; uint sizeOfPage = (uint)chunkSize * (uint)sizeof(HashBucket); uint num_bytes_read = 0; for (int index = 0; index < 1 /*Constants.kNumMergeChunks*/; index++) { HashBucket *chunkStartBucket = start + (index * chunkSize); HashIndexPageAsyncReadResult result = default(HashIndexPageAsyncReadResult); result.chunkIndex = index; device.ReadAsync(num_bytes_read, (IntPtr)chunkStartBucket, sizeOfPage, AsyncPageReadCallback, result); num_bytes_read += sizeOfPage; } Debug.Assert(num_bytes_read == num_bytes); }
private void BeginMainIndexCheckpoint( int version, IDevice device, out ulong numBytesWritten) { long totalSize = state[version].size * sizeof(HashBucket); int numChunks = (int)(totalSize >> 23); if (numChunks == 0) { numChunks = 1; } uint chunkSize = (uint)(totalSize / numChunks); mainIndexCheckpointEvent = new CountdownEvent(1); HashBucket *start = state[version].tableAligned; int startNumChunks = 1; HashIndexPageAsyncFlushResult result = new HashIndexPageAsyncFlushResult { start = start, numChunks = numChunks, numIssued = startNumChunks, numFinished = 0, chunkSize = chunkSize, device = device }; numBytesWritten = 0; for (int index = 0; index < startNumChunks; index++) { long chunkStartBucket = (long)start + (index * chunkSize); device.WriteAsync((IntPtr)chunkStartBucket, ((ulong)index) * chunkSize, chunkSize, AsyncPageFlushCallback, result); } numBytesWritten = ((ulong)numChunks) * chunkSize; }
public static void ReleaseSharedLatch(HashBucket *bucket) { Interlocked.Add(ref bucket->bucket_entries[Constants.kOverflowBucketIndex], -kPinConstant); }
public static bool TryAcquireSharedLatch(HashBucket *bucket) { return(Interlocked.Add(ref bucket->bucket_entries[Constants.kOverflowBucketIndex], kPinConstant) > 0); }
private HashBucket* GetBucket() { var bucket = _hb_cp; _hb_cp = bucket + (iHSubMask + 1); if (_hb_cp <= _hb_ep) { _cbuckets++; return bucket; } uint hbSize; if (_cbuckets < 2048) { _hb_cp = (HashBucket*)GetBig(); hbSize = iBigBlockSize + SizeofRcp; } else { var hash = new LargePage(_virtual_memory, 4) { next = _lpHash }; hbSize = hash.size; _lpHash = hash; _hb_cp = (HashBucket*)hash.data; } _hb_ep = (HashBucket*)((byte*)_hb_cp + hbSize); return GetBucket(); }
private bool FindTagOrFreeInternal(long hash, ushort tag, ref HashBucket *bucket, ref int slot, ref HashBucketEntry entry, long BeginAddress = 0) { var target_entry_word = default(long); var recordExists = false; var entry_slot_bucket = default(HashBucket *); do { // Search through the bucket looking for our key. Last entry is reserved // for the overflow pointer. for (int index = 0; index < Constants.kOverflowBucketIndex; ++index) { target_entry_word = *(((long *)bucket) + index); if (0 == target_entry_word) { if (slot == Constants.kInvalidEntrySlot) { slot = index; entry_slot_bucket = bucket; } continue; } entry.word = target_entry_word; if (entry.Address < BeginAddress && entry.Address != Constants.kTempInvalidAddress) { if (entry.word == Interlocked.CompareExchange(ref bucket->bucket_entries[index], Constants.kInvalidAddress, target_entry_word)) { if (slot == Constants.kInvalidEntrySlot) { slot = index; entry_slot_bucket = bucket; } continue; } } if (tag == entry.Tag && !entry.Tentative) { slot = index; recordExists = true; return(recordExists); } } target_entry_word = *(((long *)bucket) + Constants.kOverflowBucketIndex); // Go to next bucket in the chain if ((target_entry_word & Constants.kAddressMask) == 0) { if (slot == Constants.kInvalidEntrySlot) { // Allocate new bucket var logicalBucketAddress = overflowBucketsAllocator.Allocate(); var physicalBucketAddress = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(logicalBucketAddress); long compare_word = target_entry_word; target_entry_word = logicalBucketAddress; target_entry_word |= (compare_word & ~Constants.kAddressMask); long result_word = Interlocked.CompareExchange( ref bucket->bucket_entries[Constants.kOverflowBucketIndex], target_entry_word, compare_word); if (compare_word != result_word) { // Install failed, undo allocation; use the winner's entry overflowBucketsAllocator.Free(logicalBucketAddress); target_entry_word = result_word; } else { // Install succeeded bucket = physicalBucketAddress; slot = 0; entry = default(HashBucketEntry); return(recordExists); } } else { if (!recordExists) { bucket = entry_slot_bucket; } entry = default(HashBucketEntry); break; } } bucket = (HashBucket *)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word & Constants.kAddressMask); } while (true); return(recordExists); }
private void ResetSlabs() { _cbuckets = 0; // hash bucket extra mem pages are returned to system on reset. if (_lpHash != null) { for (var ls = _lpHash; ls != null; ls = ls.next) ls.Release(); _lpHash = null; } if (_lpBase == null) _lpBase = new LargePage(_virtual_memory, 4); _htable = (HashBucket*)_lpBase.data; // init shared tables with clean values. for (var i = 0; i < _config.HashTableSize; i++) _htable[i] = new HashBucket(); var bp = (byte*)(_htable + _config.HashTableSize); _stabs = (int*)bp; // encoding slab max size and pos like -> 32 : 0 _stabs[0] = 0x200000; _stabs[1] = 0x400001; _slabs = (SlabList*)_align((byte*)(_stabs + 1 + (iLastSlabSize + SizeofRcp) / iQuantum)); // calculate slabs indexes and max sizes. int sbts = 64, scnt = 2, pos = scnt; for (var half = sbts / 2; sbts < iLastSlabSize; half = sbts / 2) { for (var es = sbts + half; sbts < es; sbts += iQuantum) _stabs[pos++] = scnt | (es << 16); scnt++; for (var es = sbts + half; sbts < es; sbts += iQuantum) _stabs[pos++] = scnt | (es << 16); scnt++; } _stabs[pos] = scnt; bp = _align((byte*)(_slabs + scnt + 1)); // pre-allocate ~32k into every slab slot. var quot = sbts / 4; while (scnt-- > 0) { RefCountPtr* ls = null; var count = 0; for (var todo = 32768; todo > 0; count++, todo -= sbts, bp += sbts) { var rc = (RefCountPtr*)bp; rc->_list = ls; ls = rc; } _slabs[scnt].Set(ls, count); sbts -= quot; if ((scnt & 1) != 0) quot = sbts / 4; } // pre-allocate 256k into hash table buckets; _hb_cp = (HashBucket*)bp; _hb_ep = (HashBucket*)(bp += 256 * 1024); // pre-allocate ~1mb into big-mem chunks. _sbigs = null; _cntBigs = 0; sbts = iBigBlockSize + SizeofRcp; for (int left = 1024 * (1024 + 16); left > 0; _cntBigs++, left -= sbts, bp += sbts) { var rc = (RefCountPtr*)bp; rc->_list = _sbigs; _sbigs = rc; } // pre-allocate large memory pages for cache purposes. if (!_config.ReserveMemory) _cntPages = 1 + (int)((Stats.MemoryLimit - 1) / _config.AllocPageSize); if (_lpList != null) if (_config.ReserveMemory) { } else { for (var ls = _lpList; ls != null; ls = ls.next) ls.Release(); _lpList = null; } else if (_config.ReserveMemory) for (var bytes = Stats.MemoryLimit; bytes > 0; bytes -= _lpList.size) _lpList = new LargePage(_virtual_memory, _config.AllocPageSize) { next = _lpList }; _lpNext = _lpList; _cp = bp; _ep = _lpBase.data + _lpBase.size; }
public static bool NoSharedLatches(HashBucket *bucket) { long word = bucket->bucket_entries[Constants.kOverflowBucketIndex]; return((word & ~Constants.kAddressMask) == 0); }