Exemplo n.º 1
0
        public void clear_all_buffers_from_current_generation_on_low_memory(LowMemorySeverity lowMemorySeverity)
        {
            var encryptionBuffersPool = new EncryptionBuffersPool();
            var generation            = encryptionBuffersPool.Generation;
            var i      = 1;
            var toFree = new List <(IntPtr, long)>();

            while (i <= new Size(8, SizeUnit.Megabytes).GetValue(SizeUnit.Bytes))
            {
                var ptr = encryptionBuffersPool.Get(i, out _);
                toFree.Add(((IntPtr)ptr, i));

                i *= 2;
            }

            var stats = encryptionBuffersPool.GetStats();

            Assert.Equal(0, stats.TotalSize);

            encryptionBuffersPool.LowMemory(lowMemorySeverity);
            stats = encryptionBuffersPool.GetStats();
            Assert.Equal(0, stats.TotalSize);

            foreach (var o in toFree)
            {
                encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, generation);
            }

            stats = encryptionBuffersPool.GetStats();
            Assert.Equal(0, stats.TotalSize);

            var size    = 8 * 1024;
            var pointer = encryptionBuffersPool.Get(size, out _);

            encryptionBuffersPool.Return(pointer, size, NativeMemory.ThreadAllocations.Value, encryptionBuffersPool.Generation);

            // will cache the buffer
            stats = encryptionBuffersPool.GetStats();
            Assert.Equal(size, stats.TotalSize);

            // will continue to cache the buffer
            encryptionBuffersPool.LowMemory(lowMemorySeverity);
            stats = encryptionBuffersPool.GetStats();
            Assert.Equal(size, stats.TotalSize);

            encryptionBuffersPool.LowMemoryOver();
            ClearMemory(encryptionBuffersPool);
        }
Exemplo n.º 2
0
        public void clear_all_buffers_on_extremely_low_memory()
        {
            var encryptionBuffersPool = new EncryptionBuffersPool();
            var generation            = encryptionBuffersPool.Generation;
            var i      = 1;
            var toFree = new List <(IntPtr, long)>();

            while (i <= new Size(8, SizeUnit.Megabytes).GetValue(SizeUnit.Bytes))
            {
                var ptr = encryptionBuffersPool.Get(i, out _);
                toFree.Add(((IntPtr)ptr, i));

                i *= 2;
            }

            var stats = encryptionBuffersPool.GetStats();

            Assert.Equal(0, stats.TotalSize);

            foreach (var o in toFree)
            {
                encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, generation);
            }

            stats = encryptionBuffersPool.GetStats();
            var allocated = toFree.Sum(x => x.Item2);

            Assert.Equal(allocated, stats.TotalSize);

            ClearMemory(encryptionBuffersPool);
        }
Exemplo n.º 3
0
        public void properly_calculate_thread_total_allocations_when_we_cant_put_buffer_in_pool()
        {
            var encryptionBuffersPool = new EncryptionBuffersPool(registerLowMemory: false, registerCleanup: false);

            var ptr = encryptionBuffersPool.Get(1, out var initialSize, out var threadStats);

            var size = initialSize;
            var free4KbAlignedMemoryCount       = 0;
            var updateMemoryStatsForThreadCount = 0;
            var testingStuff = encryptionBuffersPool.ForTestingPurposesOnly();

            testingStuff.CanAddToPerCorePool    = false;
            testingStuff.CanAddToGlobalPool     = false;
            testingStuff.OnFree4KbAlignedMemory = s =>
            {
                free4KbAlignedMemoryCount++;
                size -= s;
            };
            testingStuff.OnUpdateMemoryStatsForThread = s =>
            {
                updateMemoryStatsForThreadCount++;
            };

            encryptionBuffersPool.Return(ptr, initialSize, threadStats, encryptionBuffersPool.Generation);

            Assert.Equal(1, free4KbAlignedMemoryCount);
            Assert.Equal(0, size);
            Assert.Equal(0, updateMemoryStatsForThreadCount);
        }
Exemplo n.º 4
0
        public override void BreakLargeAllocationToSeparatePages(IPagerLevelTransactionState tx, long pageNumber)
        {
            if (tx == null)
            {
                throw new NotSupportedException("Cannot use crypto pager without a transaction");
            }

            var state = GetTransactionState(tx);

            if (state.LoadedBuffers.TryGetValue(pageNumber, out var encBuffer) == false)
            {
                throw new InvalidOperationException("Tried to break buffer that wasn't allocated in this tx");
            }

            for (int i = 1; i < encBuffer.Size / Constants.Storage.PageSize; i++)
            {
                var buffer = new EncryptionBuffer
                {
                    Pointer      = encBuffer.Pointer + i * Constants.Storage.PageSize,
                    Size         = Constants.Storage.PageSize,
                    OriginalSize = 0,
                };

                buffer.Hash             = _encryptionBuffersPool.Get(EncryptionBuffer.HashSizeInt, out var thread);
                buffer.AllocatingThread = thread;

                // here we _intentionally_ copy the old hash from the large page, so when we commit
                // the tx, the pager will realize that we need to write this page
                Memory.Copy(buffer.Hash, encBuffer.Hash, EncryptionBuffer.HashSizeInt);

                state.LoadedBuffers[pageNumber + i] = buffer;
            }

            encBuffer.OriginalSize = encBuffer.Size;
            encBuffer.Size         = Constants.Storage.PageSize;

            // here we _intentionally_ don't modify the hash of the page, even though its size was
            // changed, because we need the pager to recognize that it was modified on tx commit
            // encBuffer.Hash = remains the same
        }
Exemplo n.º 5
0
        private EncryptionBuffer GetBufferAndAddToTxState(long pageNumber, CryptoTransactionState state, int size)
        {
            var ptr    = _encryptionBuffersPool.Get(size, out var thread);
            var buffer = new EncryptionBuffer
            {
                Size             = size,
                Pointer          = ptr,
                AllocatingThread = thread
            };

            state.LoadedBuffers[pageNumber] = buffer;
            return(buffer);
        }
Exemplo n.º 6
0
        public void clear_buffers_only_when_in_extremely_low_memory()
        {
            var encryptionBuffersPool = new EncryptionBuffersPool();

            var ptr   = encryptionBuffersPool.Get(1, out _);
            var stats = encryptionBuffersPool.GetStats();

            Assert.Equal(0, stats.TotalSize);

            encryptionBuffersPool.Return(ptr, 1, NativeMemory.ThreadAllocations.Value, encryptionBuffersPool.Generation);
            stats = encryptionBuffersPool.GetStats();
            Assert.Equal(1, stats.TotalSize);

            encryptionBuffersPool.LowMemory(LowMemorySeverity.Low);
            stats = encryptionBuffersPool.GetStats();
            Assert.Equal(1, stats.TotalSize);

            ClearMemory(encryptionBuffersPool);
        }
Exemplo n.º 7
0
        public void dont_pool_buffers_larger_than_8Mb()
        {
            var encryptionBuffersPool = new EncryptionBuffersPool();
            var i      = 1;
            var toFree = new List <(IntPtr, long)>();

            while (i < new Size(64, SizeUnit.Megabytes).GetDoubleValue(SizeUnit.Bytes))
            {
                var ptr = encryptionBuffersPool.Get(i, out _);
                toFree.Add(((IntPtr)ptr, i));

                i *= 2;
            }

            var maxSize   = new Size(8, SizeUnit.Megabytes).GetValue(SizeUnit.Bytes);
            var totalSize = 0L;

            foreach (var o in toFree)
            {
                if (o.Item2 <= maxSize)
                {
                    totalSize += o.Item2;
                }

                encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, encryptionBuffersPool.Generation);
            }

            var stats = encryptionBuffersPool.GetStats();

            Assert.Equal(totalSize, stats.TotalSize);

            i = 1;
            foreach (var allocationInfo in stats.Details)
            {
                Assert.Equal(1, allocationInfo.NumberOfItems);
                Assert.Equal(i, allocationInfo.AllocationSize);
                i *= 2;
            }

            ClearMemory(encryptionBuffersPool);
        }
Exemplo n.º 8
0
        public void can_save_buffers_after_low_memory()
        {
            var encryptionBuffersPool = new EncryptionBuffersPool();

            encryptionBuffersPool.LowMemory(LowMemorySeverity.ExtremelyLow);
            encryptionBuffersPool.LowMemoryOver();

            var i      = 1;
            var toFree = new List <(IntPtr, long)>();

            var generation = encryptionBuffersPool.Generation;

            while (i <= 1024)
            {
                var ptr = encryptionBuffersPool.Get(i, out var size, out _);
                toFree.Add(((IntPtr)ptr, i));

                i *= 2;
            }

            var stats = encryptionBuffersPool.GetStats();

            Assert.Equal(0, stats.TotalPoolSize);

            foreach (var o in toFree)
            {
                encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, generation);
            }

            stats = encryptionBuffersPool.GetStats();
            var allocated = toFree.Sum(x => x.Item2);

            Assert.Equal(allocated, stats.TotalPoolSize);

            ClearMemory(encryptionBuffersPool);
        }