Ejemplo n.º 1
0
        private SharedMemoryBucket CreateBucket(ref SharedMemoryBucket[] bucketFiles, BufferRef bufferRef, int bufferFileIndex, int bucketIndex)
        {
            SharedMemoryBucket bucket;

            lock (_buckets)
            {
                if (bucketFiles == null)
                {
                    bucketFiles = new SharedMemoryBucket[4];
                    _buckets[bufferRef.BucketIndex] = bucketFiles;
                }

                if (bufferFileIndex >= bucketFiles.Length)
                {
                    var newSize        = BitUtil.FindNextPositivePowerOfTwo(bufferFileIndex + 1);
                    var newbucketFiles = new SharedMemoryBucket[newSize];
                    Array.Copy(bucketFiles, 0, newbucketFiles, 0, bucketFiles.Length);
                    bucketFiles = newbucketFiles;
                    _buckets[bufferRef.BucketIndex] = newbucketFiles;
                }

                bucket = bucketFiles[bufferFileIndex];
                if (bucket.DirectFile == null)
                {
                    var bucketDir = Path.Combine(_directoryPath, bucketIndex.ToString());
                    Directory.CreateDirectory(bucketDir);

                    var buffersPerBucketFile = (1 << 15) >> bucketIndex;
                    var bufferSize           = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
                    var allocationSize       = buffersPerBucketFile * bufferSize;

                    // Since we are using sparse files check available free space

                    var available = GetAvailableFreeSpace();

                    // 2x just in case, maybe we need to monitor this and stop any allocations everywhere when below c.200MB or 1GB to be sure
                    if (available >= 0 && allocationSize * 2 > available)
                    {
                        throw new NotEnoughSpaceException(true);
                    }

                    // Console.WriteLine($"allocationSize for bucket {bucketIndex}: {allocationSize}");
                    var df = new DirectFile(Path.Combine(bucketDir, bufferFileIndex + ".smbkt"), allocationSize, true,
                                            FileOptions.RandomAccess    // Note that we cannot use async with sparse without using valid NativeOverlapped struct in DeviceIoControl function
                                            | FileOptions.WriteThrough, // TODO review if we need this
                                            true);

                    DirectFile.PrefetchMemory(df.DirectBuffer);

                    Trace.TraceInformation($"Allocated new file in a bucket {bucketIndex}. Total # of files: {bucketFiles.Count(x => x.DirectFile != null)}");
                    bucket = new SharedMemoryBucket(df, bucketIndex, _pageSize);
                    // Console.WriteLine($"File capacity for bucket {bucketIndex}: {bucket.BufferCount}");
                    bucketFiles[bufferFileIndex] = bucket;
                }
            }

            return(bucket);
        }
Ejemplo n.º 2
0
            public SharedMemoryBucket(DirectFile directFile, int bucketIndex, int pageSize = DefaultPageSize)
            {
                BufferRef.EnsureBucketIndexInRange(bucketIndex);

                DirectFile = directFile;

                var directBuffer = directFile.DirectBuffer;

                var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, pageSize);

                BufferCount = directBuffer.Length / bufferSize;

                DirectBuffer = directBuffer;
                BucketIndex  = bucketIndex;
                PageSize     = pageSize;
                BufferSize   = bufferSize;
            }
Ejemplo n.º 3
0
        public long GetTotalAllocated(int bucketIndex = -1)
        {
            using (var txn = _env.BeginReadOnlyTransaction())
                using (var aC = _allocatedDb.OpenReadOnlyCursor(txn))
                {
                    BufferRef lastAllocated  = default;
                    var       totalAllocated = 0L;

                    if (bucketIndex == -1)
                    {
                        byte bucketIndexByte = 0;
                        if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First))
                        {
                            var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
                            totalAllocated += dataSize * (long)aC.Count();

                            while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate))
                            {
                                dataSize        = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
                                totalAllocated += dataSize * (long)aC.Count();
                            }
                        }
                    }
                    else
                    {
                        BufferRef.EnsureBucketIndexInRange(bucketIndex);
                        var bucketIndexByte = (byte)bucketIndex;
                        if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set))
                        {
                            var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
                            totalAllocated += dataSize * (long)aC.Count();
                        }
                    }

                    _lastTotalSize = totalAllocated;
                    return(totalAllocated);
                }
        }
Ejemplo n.º 4
0
        public void PrintFree(int bucketIndex = -1, bool printBuffers = false,
                              Func <BufferRef, string> additionalInfo = null)
        {
            using (var txn = _env.BeginReadOnlyTransaction())
                using (var fC = _freeDb.OpenReadOnlyCursor(txn))
                {
                    BufferRef lastFree  = default;
                    var       totalFree = 0L;

                    if (bucketIndex == -1)
                    {
                        Console.WriteLine("------------------- FREE -------------------");
                        byte bucketIndexByte = 0;
                        if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.First))
                        {
                            var dataSize   = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
                            var count      = (long)fC.Count();
                            var bucketSize = dataSize * count;
                            totalFree += bucketSize;
                            Console.WriteLine(
                                $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
                            if (printBuffers)
                            {
                                PrintBuffers(bucketIndexByte);
                            }

                            while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextNoDuplicate))
                            {
                                dataSize   = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
                                count      = (long)fC.Count();
                                bucketSize = dataSize * count;
                                totalFree += bucketSize;
                                Console.WriteLine(
                                    $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");

                                if (printBuffers)
                                {
                                    PrintBuffers(bucketIndexByte);
                                }
                            }
                        }
                        Console.WriteLine($"------ FREE {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------");
                    }
                    else
                    {
                        BufferRef.EnsureBucketIndexInRange(bucketIndex);
                        Console.WriteLine($"------------------- FREE [{bucketIndex}] -------------------");
                        var bucketIndexByte = (byte)bucketIndex;
                        if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.Set))
                        {
                            var dataSize   = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
                            var count      = (long)fC.Count();
                            var bucketSize = dataSize * count;
                            totalFree += bucketSize;
                            Console.WriteLine(
                                $"[{bucketIndexByte}]: count [{count}], buffer size [{dataSize}], bucket size: [{bucketSize}]");
                            if (printBuffers)
                            {
                                PrintBuffers(bucketIndexByte);
                            }
                        }
                        Console.WriteLine($"------ FREE [{bucketIndex}] {totalFree:N0}, {Math.Round(100 * (totalFree / (double)MaxTotalSize), 2)}% of Max {MaxTotalSize:N0} ------");
                    }

                    void PrintBuffers(byte bucketIndexByte)
                    {
                        if (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.SetKey) &&
                            fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.FirstDuplicate))
                        {
                            var info = String.Empty;
                            if (additionalInfo != null)
                            {
                                info = ": " + additionalInfo(lastFree);
                            }

                            Console.WriteLine($"   [{lastFree.BufferIndex}]{info}");

                            while (fC.TryGet(ref bucketIndexByte, ref lastFree, CursorGetOption.NextDuplicate))
                            {
                                info = String.Empty;
                                if (additionalInfo != null)
                                {
                                    info = ": " + additionalInfo(lastFree);
                                }

                                Console.WriteLine($"   [{lastFree.BufferIndex}]{info}");
                            }
                        }
                    }
                }
        }
Ejemplo n.º 5
0
        public void PrintAllocated(int bucketIndex = -1, bool printBuffers = false,
                                   Func <BufferRef, string> additionalInfo = null)
        {
            using (var txn = _env.BeginReadOnlyTransaction())
                using (var aC = _allocatedDb.OpenReadOnlyCursor(txn))
                {
                    BufferRef lastAllocated  = default;
                    var       totalAllocated = 0L;

                    if (bucketIndex == -1)
                    {
                        Console.WriteLine("------------------- ALLOCATED -------------------");
                        byte bucketIndexByte = 0;
                        if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.First))
                        {
                            var dataSize   = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
                            var count      = (long)aC.Count();
                            var bucketSize = dataSize * count;
                            totalAllocated += bucketSize;
                            Console.WriteLine(
                                $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
                            if (printBuffers)
                            {
                                PrintBuffers(bucketIndexByte);
                            }

                            while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextNoDuplicate))
                            {
                                dataSize        = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
                                count           = (long)aC.Count();
                                bucketSize      = dataSize * count;
                                totalAllocated += bucketSize;
                                Console.WriteLine(
                                    $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");

                                if (printBuffers)
                                {
                                    PrintBuffers(bucketIndexByte);
                                }
                            }
                        }
                        Console.WriteLine($"------ ALLOCATED {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------");
                    }
                    else
                    {
                        BufferRef.EnsureBucketIndexInRange(bucketIndex);
                        Console.WriteLine($"------------------- ALLOCATED [{bucketIndex}] -------------------");
                        var bucketIndexByte = (byte)bucketIndex;
                        if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.Set))
                        {
                            var dataSize   = SharedMemoryPool.BucketIndexToBufferSize(bucketIndexByte, _pageSize);
                            var count      = (long)aC.Count();
                            var bucketSize = dataSize * count;
                            totalAllocated += bucketSize;
                            Console.WriteLine(
                                $"[{bucketIndexByte}]: count [{count:N0}], buffer size [{dataSize:N0}], bucket size: [{bucketSize:N0}]");
                            if (printBuffers)
                            {
                                PrintBuffers(bucketIndexByte);
                            }
                        }
                        Console.WriteLine($"------ ALLOCATED [{bucketIndex}] {totalAllocated:N0}, {Utilization}% of Max {MaxTotalSize:N0} ------");
                    }

                    void PrintBuffers(byte bucketIndexByte)
                    {
                        if (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.SetKey) &&
                            aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.FirstDuplicate))
                        {
                            var info = string.Empty;
                            if (additionalInfo != null)
                            {
                                info = ": " + additionalInfo(lastAllocated);
                            }

                            Console.WriteLine($"   [{lastAllocated.BufferIndex}]{info}");

                            while (aC.TryGet(ref bucketIndexByte, ref lastAllocated, CursorGetOption.NextDuplicate))
                            {
                                info = string.Empty;
                                if (additionalInfo != null)
                                {
                                    info = ": " + additionalInfo(lastAllocated);
                                }
                                Console.WriteLine($"   [{lastAllocated.BufferIndex}]{info}");
                            }
                        }
                    }

                    _lastTotalSize = totalAllocated;
                }
        }
Ejemplo n.º 6
0
        protected BufferRef Allocate(Transaction txn, byte bucketIndex, out bool fromFreeList, SharedMemoryBuckets buckets = null)
        {
            var originalBucketIndex = bucketIndex;

            fromFreeList = false;
            BufferRef.EnsureBucketIndexInRange(bucketIndex);

            BufferRef freeRef = default;

            using (var fC = _freeDb.OpenCursor(txn))
                using (var aC = _allocatedDb.OpenCursor(txn))
                {
                    // If free list is not empty then everything is simple - just move an item from FL to AL

                    if (fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.Set) &&
                        fC.TryGet(ref bucketIndex, ref freeRef, CursorGetOption.FirstDuplicate)
                        ) // prefer closer to the bucket start, DRs are sorted by index // TODO test with first with high churn & packer
                    {
                        ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");

                        _freeDb.Delete(txn, bucketIndex, freeRef);
                        var ar = freeRef;

                        aC.Put(ref bucketIndex, ref ar, CursorPutOptions.None);

                        fromFreeList = true;
                        if (buckets != null)
                        {
#pragma warning disable 618
                            var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
                            var header = directBuffer.Read <SharedMemory.BufferHeader>(0);
                            unchecked
                            {
                                if (header.FlagsCounter != (HeaderFlags.Releasing | HeaderFlags.IsDisposed))
                                {
                                    ThrowHelper.ThrowInvalidOperationException(
                                        "Buffer in free list must be (Releasing | IsDisposed).");
                                }

                                // TODO review, we require (Releasing | IsDisposed) in FL. Remove commented code below if that is OK

                                //if (((int)header.FlagsCounter & (int)AtomicCounter.CountMask) != (int)AtomicCounter.CountMask)
                                //{
                                //    ThrowHelper.ThrowInvalidOperationException("Buffer in free list must be disposed.");
                                //}

                                //var ignoreCountAndReleasing = ((int)header.FlagsCounter & ~(int)AtomicCounter.CountMask) & ~((int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
                                //if (ignoreCountAndReleasing != 0)
                                //{
                                //    ThrowHelper.ThrowInvalidOperationException("Non-Releasing flags in free buffer header.");
                                //}

                                var newHeader = header;
                                // newHeader.FlagsCounter = (SharedMemory.BufferHeader.HeaderFlags)(uint)(AtomicCounter.CountMask | (int)SharedMemory.BufferHeader.HeaderFlags.Releasing);
                                newHeader.AllocatorInstanceId = _wpid.InstanceId;

                                // TODO check existing header values
                                // It must be zeros if the buffer was disposed normally
                                //if ((long)header != 0)
                                //{
                                //    Trace.TraceWarning($"Header of free list buffer is not zero: FlagsCounter {header.FlagsCounter}, InstanceId: {header.AllocatorInstanceId}");
                                //}

                                if ((long)header !=
                                    directBuffer.InterlockedCompareExchangeInt64(0, (long)newHeader,
                                                                                 (long)header))
                                {
                                    // This should only happen if someone if touching the header concurrently
                                    // But we are inside write txn now so this should never happen.
                                    // TODO could remove expensive CAS for simple write.
                                    ThrowHelper.FailFast("Header changed during allocating from free list.");
                                }
                            }
                        }
                    }
                    else
                    {
                        // Free list is empty.
                        // Allocated must be sorted by BufferRef as uint. Since in DUP_SORTED bucket bites are
                        // the same then the last allocated value has the maximum buffer index.

                        int bufferIndex = 1; // if is empty then start from 1

                        BufferRef lastAllocated = default;
                        if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.Set) &&
                            aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.LastDuplicate))
                        {
                            ThrowHelper.AssertFailFast(lastAllocated.BufferIndex > 0, "BufferRef.BufferIndex is one-based");
                            bufferIndex += lastAllocated.BufferIndex;
                        }

                        ThrowHelper.AssertFailFast(originalBucketIndex == bucketIndex, $"originalBucketIndex {originalBucketIndex} == bucketIndex {bucketIndex}");

                        freeRef = BufferRef.Create(bucketIndex, bufferIndex);

                        lastAllocated = freeRef;

                        // _allocatedDb.Put(txn, bucketIndex, lastAllocated, TransactionPutOptions.AppendDuplicateData);
                        aC.Put(ref bucketIndex, ref lastAllocated, CursorPutOptions.AppendDuplicateData);

                        // We are done, freeRef is set. Reuse cursors and locals.

                        var dataSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);

                        _allocationsWithoutSizeCheckCount++;
                        if (_allocationsWithoutSizeCheckCount == CheckTotalSizeEveryNAllocation &&
                            CheckTotalSizeEveryNAllocation > 0
                            ||
                            _lastTotalSize / (double)_maxTotalSize > 0.9
                            ||
                            dataSize / (double)MaxTotalSize > 0.01 // 1%
                            )
                        {
                            _allocationsWithoutSizeCheckCount = 0;

                            var totalAllocated = 0L;
                            if (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.First))
                            {
                                totalAllocated += dataSize * (long)aC.Count();
                            }

                            while (aC.TryGet(ref bucketIndex, ref lastAllocated, CursorGetOption.NextNoDuplicate))
                            {
                                totalAllocated += dataSize * (long)aC.Count();
                            }

                            _lastTotalSize = totalAllocated;

                            if (totalAllocated > _maxTotalSize)
                            {
                                ThrowBufferRefAllocatorFullException();
                            }
                        }

                        if (buckets != null)
                        {
                            unchecked
                            {
#pragma warning disable 618
                                var directBuffer = buckets.DangerousGet(freeRef);
#pragma warning restore 618
                                var header = directBuffer.Read <SharedMemory.BufferHeader>(0);

                                if (header.FlagsCounter != default)
                                {
                                    throw new InvalidOperationException($"Allocated buffer for {freeRef} has non-empty header.FlagsCounter: {header.FlagsCounter}");
                                }

                                if (header.AllocatorInstanceId != default)
                                {
                                    throw new InvalidOperationException($"Allocated buffer has non-empry header.AllocatorInstanceId: {header.AllocatorInstanceId}");
                                }

                                header.FlagsCounter        = HeaderFlags.Releasing | HeaderFlags.IsDisposed;
                                header.AllocatorInstanceId = _wpid.InstanceId;

                                // No need to check previous value because it is a new buffer
                                // that was never touched from this BRA perspective.
                                directBuffer.VolatileWriteInt64(0, (long)header);
                            }
                        }
                    }
                }

            return(freeRef);
        }