private SharedMemoryBucket CreateBucket(ref SharedMemoryBucket[] bucketFiles, BufferRef bufferRef, int bufferFileIndex, int bucketIndex) { SharedMemoryBucket bucket; lock (_buckets) { if (bucketFiles == null) { bucketFiles = new SharedMemoryBucket[4]; _buckets[bufferRef.BucketIndex] = bucketFiles; } if (bufferFileIndex >= bucketFiles.Length) { var newSize = BitUtil.FindNextPositivePowerOfTwo(bufferFileIndex + 1); var newbucketFiles = new SharedMemoryBucket[newSize]; Array.Copy(bucketFiles, 0, newbucketFiles, 0, bucketFiles.Length); bucketFiles = newbucketFiles; _buckets[bufferRef.BucketIndex] = newbucketFiles; } bucket = bucketFiles[bufferFileIndex]; if (bucket.DirectFile == null) { var bucketDir = Path.Combine(_directoryPath, bucketIndex.ToString()); Directory.CreateDirectory(bucketDir); var buffersPerBucketFile = (1 << 15) >> bucketIndex; var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); var allocationSize = buffersPerBucketFile * bufferSize; // Since we are using sparse files check available free space var available = GetAvailableFreeSpace(); // 2x just in case, maybe we need to monitor this and stop any allocations everywhere when below c.200MB or 1GB to be sure if (available >= 0 && allocationSize * 2 > available) { throw new NotEnoughSpaceException(true); } // Console.WriteLine($"allocationSize for bucket {bucketIndex}: {allocationSize}"); var df = new DirectFile(Path.Combine(bucketDir, bufferFileIndex + ".smbkt"), allocationSize, true, FileOptions.RandomAccess // Note that we cannot use async with sparse without using valid NativeOverlapped struct in DeviceIoControl function | FileOptions.WriteThrough, // TODO review if we need this true); DirectFile.PrefetchMemory(df.DirectBuffer); Trace.TraceInformation($"Allocated new file in a bucket {bucketIndex}. Total # of files: {bucketFiles.Count(x => x.DirectFile != null)}"); bucket = new SharedMemoryBucket(df, bucketIndex, _pageSize); // Console.WriteLine($"File capacity for bucket {bucketIndex}: {bucket.BufferCount}"); bucketFiles[bufferFileIndex] = bucket; } } return(bucket); }
public SharedMemoryBucket(DirectFile directFile, int bucketIndex, int pageSize = DefaultPageSize) { BufferRef.EnsureBucketIndexInRange(bucketIndex); DirectFile = directFile; var directBuffer = directFile.DirectBuffer; var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, pageSize); BufferCount = directBuffer.Length / bufferSize; DirectBuffer = directBuffer; BucketIndex = bucketIndex; PageSize = pageSize; BufferSize = bufferSize; }
public long GetTotalAllocated(int bucketIndex = -1) { using (var txn = _env.BeginReadOnlyTransaction()) using (var aC = _allocatedDb.OpenReadOnlyCursor(txn)) { BufferRef lastAllocated = default; var totalAllocated = 0L; if (bucketIndex == -1) { byte bucketIndexByte = 0; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); totalAllocated += dataSize * (long)aC.Count(); while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); totalAllocated += dataSize * (long)aC.Count(); } } } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); var bucketIndexByte = (byte)bucketIndex; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); totalAllocated += dataSize * (long)aC.Count(); } } _lastTotalSize = totalAllocated; return(totalAllocated); } }
public void PrintFree(int bucketIndex = -1, bool printBuffers = false, Func <BufferRef, string> additionalInfo = null) { using (var txn = _env.BeginReadOnlyTransaction()) using (var fC = _freeDb.OpenReadOnlyCursor(txn)) { BufferRef lastFree = default; var totalFree = 0L; if (bucketIndex == -1) { Console.WriteLine("------------------- FREE -------------------"); byte bucketIndexByte = 0; if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)fC.Count(); var bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); count = (long)fC.Count(); bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } } Console.WriteLine($"------ FREE {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------"); } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); Console.WriteLine($"------------------- FREE [{bucketIndex}] -------------------"); var bucketIndexByte = (byte)bucketIndex; if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)fC.Count(); var bucketSize = dataSize * count; totalFree += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count}], buffer size [{dataSize}], bucket size: [{bucketSize}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } Console.WriteLine($"------ FREE [{bucketIndex}] {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------"); } void PrintBuffers(byte bucketIndexByte) { if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.SetKey) && fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.FirstDuplicate)) { var info = String.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastFree); } Console.WriteLine($" [{lastFree.BufferIndex}]{info}"); while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextDuplicate)) { info = String.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastFree); } Console.WriteLine($" [{lastFree.BufferIndex}]{info}"); } } } } }
public void PrintAllocated(int bucketIndex = -1, bool printBuffers = false, Func <BufferRef, string> additionalInfo = null) { using (var txn = _env.BeginReadOnlyTransaction()) using (var aC = _allocatedDb.OpenReadOnlyCursor(txn)) { BufferRef lastAllocated = default; var totalAllocated = 0L; if (bucketIndex == -1) { Console.WriteLine("------------------- ALLOCATED -------------------"); byte bucketIndexByte = 0; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)aC.Count(); var bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); count = (long)aC.Count(); bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } } Console.WriteLine($"------ ALLOCATED {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------"); } else { BufferRef.EnsureBucketIndexInRange(bucketIndex); Console.WriteLine($"------------------- ALLOCATED [{bucketIndex}] -------------------"); var bucketIndexByte = (byte)bucketIndex; if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set)) { var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize); var count = (long)aC.Count(); var bucketSize = dataSize * count; totalAllocated += bucketSize; Console.WriteLine( $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]"); if (printBuffers) { PrintBuffers(bucketIndexByte); } } Console.WriteLine($"------ ALLOCATED [{bucketIndex}] {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------"); } void PrintBuffers(byte bucketIndexByte) { if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.SetKey) && aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.FirstDuplicate)) { var info = string.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastAllocated); } Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}"); while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextDuplicate)) { info = string.Empty; if (additionalInfo != null) { info = ": " + additionalInfo(lastAllocated); } Console.WriteLine($" [{lastAllocated.BufferIndex}]{info}"); } } } _lastTotalSize = totalAllocated; } }
protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null) { var originalBucketIndex = bucketIndex; fromFreeList = false; BufferRef.EnsureBucketIndexInRange(bucketIndex); BufferRef freeRef = default; using (var fC = _freeDb.OpenCursor(txn)) using (var aC = _allocatedDb.OpenCursor(txn)) { // If free list is not empty then everything is simple - just move an item from FL to AL if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) && fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate) ) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer { ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); _freeDb.Delete(txn, bucketIndex, freeRef); var ar = freeRef; aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None); fromFreeList = true; if (buckets != null) { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); unchecked { if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed)) { ThrowHelper.ThrowInvalidOperationException( "Buffer in free list must be (Releasing | IsDisposed)."); } // TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK //if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask) //{ // ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed."); //} //var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing); //if (ignoreCountAndReleasing != 0) //{ // ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header."); //} var newHeader = header; // newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing); newHeader.AllocatorInstanceId = _wpid.InstanceId; // TODO check existing header values // It must be zeros if the buffer was disposed normally //if ((long)header != 0) //{ // Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}"); //} if ((long)header != directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader, (long)header)) { // This should only happen if someone if touching the header concurrently // But we are inside write txn now so this should never happen. // TODO could remove expensive CAS for simple write. ThrowHelper.FailFast("Header changed during allocating from free list."); } } } } else { // Free list is empty. // Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are // the same then the last allocated value has the maximum buffer index. int bufferIndex = 1; // if is empty then start from 1 BufferRef lastAllocated = default; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) && aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate)) { ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based"); bufferIndex += lastAllocated.BufferIndex; } ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}"); freeRef = BufferRef.Create(bucketIndex, bufferIndex); lastAllocated = freeRef; // _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData); aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData); // We are done, freeRef is set. Reuse cursors and locals. var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize); _allocationsWithoutSizeCheckCount++; if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation && CheckTotalSizeEveryNAllocation > 0 || _lastTotalSize / (double)_maxTotalSize > 0.9 || dataSize / (double)MaxTotalSize > 0.01 // 1% ) { _allocationsWithoutSizeCheckCount = 0; var totalAllocated = 0L; if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First)) { totalAllocated += dataSize * (long)aC.Count(); } while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate)) { totalAllocated += dataSize * (long)aC.Count(); } _lastTotalSize = totalAllocated; if (totalAllocated > _maxTotalSize) { ThrowBufferRefAllocatorFullException(); } } if (buckets != null) { unchecked { #pragma warning disable 618 var directBuffer = buckets.DangerousGet(freeRef); #pragma warning restore 618 var header = directBuffer.Read <SharedMemory.BufferHeader>(0); if (header.FlagsCounter != default) { throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}"); } if (header.AllocatorInstanceId != default) { throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}"); } header.FlagsCounter = HeaderFlags.Releasing | HeaderFlags.IsDisposed; header.AllocatorInstanceId = _wpid.InstanceId; // No need to check previous value because it is a new buffer // that was never touched from this BRA perspective. directBuffer.VolatileWriteInt64(0, (long)header); } } } } return(freeRef); }