private SharedMemoryBucket CreateBucket(ref SharedMemoryBucket[] bucketFiles, BufferRef bufferRef, int bufferFileIndex, int bucketIndex)
        {
            SharedMemoryBucket bucket;

            lock (_buckets)
            {
                if (bucketFiles == null)
                {
                    bucketFiles = new SharedMemoryBucket[4];
                    _buckets[bufferRef.BucketIndex] = bucketFiles;
                }

                if (bufferFileIndex >= bucketFiles.Length)
                {
                    var newSize        = BitUtil.FindNextPositivePowerOfTwo(bufferFileIndex + 1);
                    var newbucketFiles = new SharedMemoryBucket[newSize];
                    Array.Copy(bucketFiles, 0, newbucketFiles, 0, bucketFiles.Length);
                    bucketFiles = newbucketFiles;
                    _buckets[bufferRef.BucketIndex] = newbucketFiles;
                }

                bucket = bucketFiles[bufferFileIndex];
                if (bucket.DirectFile == null)
                {
                    var bucketDir = Path.Combine(_directoryPath, bucketIndex.ToString());
                    Directory.CreateDirectory(bucketDir);

                    var buffersPerBucketFile = (1 << 15) >> bucketIndex;
                    var bufferSize           = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, _pageSize);
                    var allocationSize       = buffersPerBucketFile * bufferSize;

                    // Since we are using sparse files check available free space

                    var available = GetAvailableFreeSpace();

                    // 2x just in case, maybe we need to monitor this and stop any allocations everywhere when below c.200MB or 1GB to be sure
                    if (available >= 0 && allocationSize * 2 > available)
                    {
                        throw new NotEnoughSpaceException(true);
                    }

                    // Console.WriteLine($"allocationSize for bucket {bucketIndex}: {allocationSize}");
                    var df = new DirectFile(Path.Combine(bucketDir, bufferFileIndex + ".smbkt"), allocationSize, true,
                                            FileOptions.RandomAccess    // Note that we cannot use async with sparse without using valid NativeOverlapped struct in DeviceIoControl function
                                            | FileOptions.WriteThrough, // TODO review if we need this
                                            true);

                    DirectFile.PrefetchMemory(df.DirectBuffer);

                    Trace.TraceInformation($"Allocated new file in a bucket {bucketIndex}. Total # of files: {bucketFiles.Count(x => x.DirectFile != null)}");
                    bucket = new SharedMemoryBucket(df, bucketIndex, _pageSize);
                    // Console.WriteLine($"File capacity for bucket {bucketIndex}: {bucket.BufferCount}");
                    bucketFiles[bufferFileIndex] = bucket;
                }
            }

            return(bucket);
        }
            public SharedMemoryBucket(DirectFile directFile, int bucketIndex, int pageSize = DefaultPageSize)
            {
                BufferRef.EnsureBucketIndexInRange(bucketIndex);

                DirectFile = directFile;

                var directBuffer = directFile.DirectBuffer;

                var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(bucketIndex, pageSize);

                BufferCount = directBuffer.Length / bufferSize;

                DirectBuffer = directBuffer;
                BucketIndex  = bucketIndex;
                PageSize     = pageSize;
                BufferSize   = bufferSize;
            }