public DirectBuffer DangerousGet(BufferRef bufferRef) { var bucketIndex = bufferRef.BucketIndex; var bufferIndex = bufferRef.BufferIndex - 1; // BufferRef is one-based SharedMemoryBucket[] bucketFiles = _buckets[bucketIndex]; var bufferFileIndex = bufferIndex >> (BufferRef.MaxBucketIdx - bucketIndex); var bufferIndexInFile = bufferIndex & ((1 << (BufferRef.MaxBucketIdx - bucketIndex)) - 1); SharedMemoryBucket bucket = default; if (bucketFiles != null) { // we need < cmp for not throwing (and it could be <), so add >= 0 // in a *hope* that compiler then eliminates bound check and // we have 2 cmp instead of 3. Benchmark inconclusive, // and this not that important to spend time on disassembling. if (bufferFileIndex >= 0 && bufferFileIndex < bucketFiles.Length) { bucket = bucketFiles[bufferFileIndex]; } } if (bucket.DirectFile == null) { bucket = CreateBucket(ref bucketFiles, bufferRef, bufferFileIndex, bucketIndex); } return(bucket.DangerousGet(bufferIndexInFile)); }
public DirectBuffer this[BufferRef bufferRef] { [MethodImpl(MethodImplOptions.AggressiveInlining)] get { var bucketIndex = bufferRef.BucketIndex; var bufferIndex = bufferRef.BufferIndex - 1; // BufferRef is one-based if (bucketIndex > _maxBucketIndex) { ThrowBucketIndexAboveMax(); } SharedMemoryBucket[] bucketFiles = _buckets[bucketIndex]; var bufferFileIndex = bufferIndex >> (BufferRef.MaxBucketIdx - bucketIndex); var bufferIndexInFile = bufferIndex & ((1 << (BufferRef.MaxBucketIdx - bucketIndex)) - 1); SharedMemoryBucket bucket = default; if (bucketFiles != null && bufferFileIndex < bucketFiles.Length) { bucket = bucketFiles[bufferFileIndex]; } if (bucket.DirectFile == null) { bucket = CreateBucket(ref bucketFiles, bufferRef, bufferFileIndex, bucketIndex); } return(bucket[bufferIndexInFile]); } }
/// <summary> /// Try to return a buffer from allocated to free list if /// detected a buffer with (Releasing | Disposed) state /// in the allocated list. /// </summary> internal unsafe void RetryReturnNative(BufferRef bufferRef) { // It should not be possible that we have SM instance that is in free list // If we detected a buffer with Releasing in allocated list we should just // re-release it via BRA without creating a SM instance. Otherwise // someone could take that Releasing+Free buffer that will become owned // but SM finalizer will kill it while thinking it is re-releasing it. // If Releasing flag is not cleared after BRA.Free that is not stopping // RentNative from taking that buffer. // TODO try/catch? or trace warning // this operation could be done by any process concurrently, the first that detects // the condition tries to clean up. #pragma warning disable 618 var db = _buckets.DangerousGet(bufferRef); #pragma warning restore 618 if (Unsafe.Read <uint>(db.Data) != (HeaderFlags.Releasing | HeaderFlags.IsDisposed)) { ThrowHelper.ThrowInvalidOperationException($"bufferRef {bufferRef} in not in (Releasing | Disposed) state, cannot retry ReturnNative."); } if (_bra.IsInFreeList(bufferRef)) { ThrowHelper.ThrowInvalidOperationException($"bufferRef {bufferRef} in in free list."); } _bra.Free(bufferRef); }
internal DirectBuffer TryGetAllocatedNativeBuffer(BufferRef bufferRef) { #if DEBUG // we will do atomic header checks later anyway, but keep this in debug if (_bra.IsInFreeList(bufferRef)) { ThrowHelper.ThrowInvalidOperationException($"Cannot get non-allocated buffer directly."); } #endif #pragma warning disable 618 var nativeBuffer = _buckets.DangerousGet(bufferRef); #pragma warning restore 618 // At least is must be owned, other flag checks are done by callers of this method // TODO (!) review all usages, ensure that flags are checked depending on the context var header = nativeBuffer.Read <SharedMemory.BufferHeader>(0); if ((header.FlagsCounter & HeaderFlags.IsOwned) == 0) { ThrowNotOwned(); } return(nativeBuffer); void ThrowNotOwned() { ThrowHelper.ThrowInvalidOperationException($"TryGetAllocatedNativeBuffer: {bufferRef} is not owned."); } }
public BufferRef Allocate(byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null) { BufferRef.EnsureBucketIndexInRange(bucketIndex); using (var txn = _env.BeginTransaction()) { try { BufferRef freeRef = Allocate(txn, bucketIndex, out fromFreeList, buckets); txn.Commit(); // Without packing and returning buffers to pool this kills performance // But it must be done. if (!fromFreeList) { _env.Sync(true); } return(freeRef); } catch { txn.Abort(); throw; } } }
internal unsafe SharedMemory RentNative(int bucketSize) { var bucketIndex = SelectBucketIndex(bucketSize); BufferRef.EnsureBucketIndexInRange(bucketIndex); // We must init the buffer header before allocating. // We know from inside BRA.Allocate txn the value or BufferRef // before committing. For new we must init the header, // for one from the free list we must assert the buffer is disposed // and not owned (it may be in Releasing state though). var br = _bra.Allocate((byte)bucketIndex, out _, _buckets); var db = _buckets.DangerousGet(br); var sm = SharedMemory.Create(db, br, this); Debug.Assert(sm.IsDisposed); Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == (HeaderFlags.Releasing | HeaderFlags.IsDisposed)); sm.FromReleasingDisposedToOwned(); Debug.Assert(!sm.IsDisposed); Debug.Assert(sm.ReferenceCount == 0); Debug.Assert(Unsafe.Read <uint>(sm.HeaderPointer) == HeaderFlags.IsOwned); return(sm); }
public SharedMemoryBucket(DirectFile directFile, int bucketIndex, int pageSize = DefaultPageSize) { BufferRef.EnsureBucketIndexInRange(bucketIndex); DirectFile = directFile; var directBuffer = directFile.DirectBuffer; var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, pageSize); BufferCount = directBuffer.Length / bufferSize; DirectBuffer = directBuffer; BucketIndex = bucketIndex; PageSize = pageSize; BufferSize = bufferSize; }
public void Free(BufferRef bufferRef) //, Wpid wpid = default) { using (var txn = _env.BeginTransaction()) { try { Free(txn, bufferRef); txn.Commit(); } catch { txn.Abort(); throw; } } }
public bool IsInFreeList(BufferRef bufferRef) { using (var txn = _env.BeginReadOnlyTransaction()) { using (var fC = _freeDb.OpenReadOnlyCursor(txn)) { byte bucketIndex = checked ((byte)bufferRef.BucketIndex); BufferRef value = bufferRef; if (fC.TryGet(ref bucketIndex, ref value, CursorGetOption.GetBoth)) { if (bufferRef != value) { ThrowHelper.FailFast("Wrong BRA implementation"); } return(true); } return(false); } } }
protected void Free(Transaction txn, BufferRef bufferRef) //, Wpid wpid = default) { if (bufferRef.Flag) { ThrowHelper.FailFast($"bufferRef.Flag is true in BRA.Free"); } using (var aC = _allocatedDb.OpenCursor(txn)) { byte bucketIndex = checked ((byte)bufferRef.BucketIndex); var ar = bufferRef; if (aC.TryFindDup(Lookup.EQ, ref bucketIndex, ref ar)) { _allocatedDb.Delete(txn, bucketIndex, ar); _freeDb.Put(txn, bucketIndex, bufferRef); } else { ThrowFreeBufferRefNotAllocated(); } } }
public long GetTotalAllocated(int bucketIndex = -1) { using (var txn = _env.BeginReadOnlyTransaction()) using (var aC = _allocatedDb.OpenReadOnlyCursor(txn)) { BufferRef lastAllocated = default; var totalAllocated = 0L; if (bucketIndex == -1) { byte bucketIndexByte = 0; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); totalAllocated += dataSize * (long)aC.Count(); while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); totalAllocated += dataSize * (long)aC.Count(); } } } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); var bucketIndexByte = (byte)bucketIndex; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); totalAllocated += dataSize * (long)aC.Count(); } } _lastTotalSize = totalAllocated; return(totalAllocated); } }
private SharedMemoryBucket CreateBucket(ref SharedMemoryBucket[] bucketFiles, BufferRef bufferRef, int bufferFileIndex, int bucketIndex) { SharedMemoryBucket bucket; lock (_buckets) { if (bucketFiles == null) { bucketFiles = new SharedMemoryBucket[4]; _buckets[bufferRef.BucketIndex] = bucketFiles; } if (bufferFileIndex >= bucketFiles.Length) { var newSize = BitUtil.FindNextPositivePowerOfTwo(bufferFileIndex + 1); var newbucketFiles = new SharedMemoryBucket[newSize]; Array.Copy(bucketFiles, 0, newbucketFiles, 0, bucketFiles.Length); bucketFiles = newbucketFiles; _buckets[bufferRef.BucketIndex] = newbucketFiles; } bucket = bucketFiles[bufferFileIndex]; if (bucket.DirectFile == null) { var bucketDir = Path.Combine(_directoryPath, bucketIndex.ToString()); Directory.CreateDirectory(bucketDir); var buffersPerBucketFile = (1 << 15) >> bucketIndex; var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); var allocationSize = buffersPerBucketFile * bufferSize; // Since we are using sparse files check available free space var available = GetAvailableFreeSpace(); // 2x just in case, maybe we need to monitor this and stop any allocations everywhere when below c.200MB or 1GB to be sure if (available >= 0 && allocationSize * 2 > available) { throw new NotEnoughSpaceException(true); } // Console.WriteLine($"allocationSize for bucket {bucketIndex}: {allocationSize}"); var df = new DirectFile(Path.Combine(bucketDir, bufferFileIndex + ".smbkt"), allocationSize, true, FileOptions.RandomAccess // Note that we cannot use async with sparse without using valid NativeOverlapped struct in DeviceIoControl function | FileOptions.WriteThrough, // TODO review if we need this true); DirectFile.PrefetchMemory(df.DirectBuffer); Trace.TraceInformation($"Allocated new file in a bucket {bucketIndex}. Total # of files: {bucketFiles.Count(x => x.DirectFile != null)}"); bucket = new SharedMemoryBucket(df, bucketIndex, _pageSize); // Console.WriteLine($"File capacity for bucket {bucketIndex}: {bucket.BufferCount}"); bucketFiles[bufferFileIndex] = bucket; } } return(bucket); }
public void PrintFree(int bucketIndex = -1, bool printBuffers = false, Func <BufferRef, string> additionalInfo = null) { using (var txn = _env.BeginReadOnlyTransaction()) using (var fC = _freeDb.OpenReadOnlyCursor(txn)) { BufferRef lastFree = default; var totalFree = 0L; if (bucketIndex == -1) { Console.WriteLine("------------------- FREE -------------------"); byte bucketIndexByte = 0; if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)fC.Count(); var bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); count = (long)fC.Count(); bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } } Console.WriteLine($"------ FREE {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------"); } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); Console.WriteLine($"------------------- FREE [{bucketIndex}] -------------------"); var bucketIndexByte = (byte)bucketIndex; if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)fC.Count(); var bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count}], buffer size [{dataSize}], bucket size: [{bucketSize}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } Console.WriteLine($"------ FREE [{bucketIndex}] {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------"); } void PrintBuffers(byte bucketIndexByte) { if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.SetKey) && fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.FirstDuplicate)) { var info = String.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastFree); } Console.WriteLine($" [{lastFree.BufferIndex}]{info}"); while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextDuplicate)) { info = String.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastFree); } Console.WriteLine($" [{lastFree.BufferIndex}]{info}"); } } } } }
public void PrintAllocated(int bucketIndex = -1, bool printBuffers = false, Func <BufferRef, string> additionalInfo = null) { using (var txn = _env.BeginReadOnlyTransaction()) using (var aC = _allocatedDb.OpenReadOnlyCursor(txn)) { BufferRef lastAllocated = default; var totalAllocated = 0L; if (bucketIndex == -1) { Console.WriteLine("------------------- ALLOCATED -------------------"); byte bucketIndexByte = 0; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)aC.Count(); var bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); count = (long)aC.Count(); bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } } Console.WriteLine($"------ ALLOCATED {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------"); } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); Console.WriteLine($"------------------- ALLOCATED [{bucketIndex}] -------------------"); var bucketIndexByte = (byte)bucketIndex; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)aC.Count(); var bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } Console.WriteLine($"------ ALLOCATED [{bucketIndex}] {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------"); } void PrintBuffers(byte bucketIndexByte) { if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.SetKey) && aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.FirstDuplicate)) { var info = string.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastAllocated); } Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}"); while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextDuplicate)) { info = string.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastAllocated); } Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}"); } } } _lastTotalSize = totalAllocated; } }
protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null) { var originalBucketIndex = bucketIndex; fromFreeList = false; BufferRef.EnsureBucketIndexInRange(bucketIndex); BufferRef freeRef = default; using (var fC = _freeDb.OpenCursor(txn)) using (var aC = _allocatedDb.OpenCursor(txn)) { // If free list is not empty then everything is simple - just move an item from FL to AL if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) && fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate) ) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer { ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); _freeDb.Delete(txn, bucketIndex, freeRef); var ar = freeRef; aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None); fromFreeList = true; if (buckets != null) { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); unchecked { if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed)) { ThrowHelper.ThrowInvalidOperationException( "Buffer in free list must be (Releasing | IsDisposed)."); } // TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK //if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask) //{ // ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed."); //} //var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing); //if (ignoreCountAndReleasing != 0) //{ // ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header."); //} var newHeader = header; // newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing); newHeader.AllocatorInstanceId = _wpid.InstanceId; // TODO check existing header values // It must be zeros if the buffer was disposed normally //if ((long)header != 0) //{ // Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}"); //} if ((long)header != directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader, (long)header)) { // This should only happen if someone if touching the header concurrently // But we are inside write txn now so this should never happen. // TODO could remove expensive CAS for simple write. ThrowHelper.FailFast("Header changed during allocating from free list."); } } } } else { // Free list is empty. // Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are // the same then the last allocated value has the maximum buffer index. int bufferIndex = 1; // if is empty then start from 1 BufferRef lastAllocated = default; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) && aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate)) { ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based"); bufferIndex += lastAllocated.BufferIndex; } ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); freeRef = BufferRef.Create(bucketIndex, bufferIndex); lastAllocated = freeRef; // _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData); aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData); // We are done, freeRef is set. Reuse cursors and locals. var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); _allocationsWithoutSizeCheckCount++; if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation && CheckTotalSizeEveryNAllocation > 0 || _lastTotalSize / (double)_maxTotalSize > 0.9 || dataSize / (double)MaxTotalSize > 0.01 // 1% ) { _allocationsWithoutSizeCheckCount = 0; var totalAllocated = 0L; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First)) { totalAllocated += dataSize * (long)aC.Count(); } while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { totalAllocated += dataSize * (long)aC.Count(); } _lastTotalSize = totalAllocated; if (totalAllocated > _maxTotalSize) { ThrowBufferRefAllocatorFullException(); } } if (buckets != null) { unchecked { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); if (header.FlagsCounter != default) { throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}"); } if (header.AllocatorInstanceId != default) { throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}"); } header.FlagsCounter = HeaderFlags.Releasing | HeaderFlags.IsDisposed; header.AllocatorInstanceId = _wpid.InstanceId; // No need to check previous value because it is a new buffer // that was never touched from this BRA perspective. directBuffer.VolatileWriteInt64(0, (long)header); } } } } return(freeRef); }