public void clear_all_buffers_from_current_generation_on_low_memory(LowMemorySeverity lowMemorySeverity) { var encryptionBuffersPool = new EncryptionBuffersPool(); var generation = encryptionBuffersPool.Generation; var i = 1; var toFree = new List <(IntPtr, long)>(); while (i <= new Size(8, SizeUnit.Megabytes).GetValue(SizeUnit.Bytes)) { var ptr = encryptionBuffersPool.Get(i, out _); toFree.Add(((IntPtr)ptr, i)); i *= 2; } var stats = encryptionBuffersPool.GetStats(); Assert.Equal(0, stats.TotalSize); encryptionBuffersPool.LowMemory(lowMemorySeverity); stats = encryptionBuffersPool.GetStats(); Assert.Equal(0, stats.TotalSize); foreach (var o in toFree) { encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, generation); } stats = encryptionBuffersPool.GetStats(); Assert.Equal(0, stats.TotalSize); var size = 8 * 1024; var pointer = encryptionBuffersPool.Get(size, out _); encryptionBuffersPool.Return(pointer, size, NativeMemory.ThreadAllocations.Value, encryptionBuffersPool.Generation); // will cache the buffer stats = encryptionBuffersPool.GetStats(); Assert.Equal(size, stats.TotalSize); // will continue to cache the buffer encryptionBuffersPool.LowMemory(lowMemorySeverity); stats = encryptionBuffersPool.GetStats(); Assert.Equal(size, stats.TotalSize); encryptionBuffersPool.LowMemoryOver(); ClearMemory(encryptionBuffersPool); }
public void clear_all_buffers_on_extremely_low_memory() { var encryptionBuffersPool = new EncryptionBuffersPool(); var generation = encryptionBuffersPool.Generation; var i = 1; var toFree = new List <(IntPtr, long)>(); while (i <= new Size(8, SizeUnit.Megabytes).GetValue(SizeUnit.Bytes)) { var ptr = encryptionBuffersPool.Get(i, out _); toFree.Add(((IntPtr)ptr, i)); i *= 2; } var stats = encryptionBuffersPool.GetStats(); Assert.Equal(0, stats.TotalSize); foreach (var o in toFree) { encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, generation); } stats = encryptionBuffersPool.GetStats(); var allocated = toFree.Sum(x => x.Item2); Assert.Equal(allocated, stats.TotalSize); ClearMemory(encryptionBuffersPool); }
public void properly_calculate_thread_total_allocations_when_we_cant_put_buffer_in_pool() { var encryptionBuffersPool = new EncryptionBuffersPool(registerLowMemory: false, registerCleanup: false); var ptr = encryptionBuffersPool.Get(1, out var initialSize, out var threadStats); var size = initialSize; var free4KbAlignedMemoryCount = 0; var updateMemoryStatsForThreadCount = 0; var testingStuff = encryptionBuffersPool.ForTestingPurposesOnly(); testingStuff.CanAddToPerCorePool = false; testingStuff.CanAddToGlobalPool = false; testingStuff.OnFree4KbAlignedMemory = s => { free4KbAlignedMemoryCount++; size -= s; }; testingStuff.OnUpdateMemoryStatsForThread = s => { updateMemoryStatsForThreadCount++; }; encryptionBuffersPool.Return(ptr, initialSize, threadStats, encryptionBuffersPool.Generation); Assert.Equal(1, free4KbAlignedMemoryCount); Assert.Equal(0, size); Assert.Equal(0, updateMemoryStatsForThreadCount); }
public override void BreakLargeAllocationToSeparatePages(IPagerLevelTransactionState tx, long pageNumber) { if (tx == null) { throw new NotSupportedException("Cannot use crypto pager without a transaction"); } var state = GetTransactionState(tx); if (state.LoadedBuffers.TryGetValue(pageNumber, out var encBuffer) == false) { throw new InvalidOperationException("Tried to break buffer that wasn't allocated in this tx"); } for (int i = 1; i < encBuffer.Size / Constants.Storage.PageSize; i++) { var buffer = new EncryptionBuffer { Pointer = encBuffer.Pointer + i * Constants.Storage.PageSize, Size = Constants.Storage.PageSize, OriginalSize = 0, }; buffer.Hash = _encryptionBuffersPool.Get(EncryptionBuffer.HashSizeInt, out var thread); buffer.AllocatingThread = thread; // here we _intentionally_ copy the old hash from the large page, so when we commit // the tx, the pager will realize that we need to write this page Memory.Copy(buffer.Hash, encBuffer.Hash, EncryptionBuffer.HashSizeInt); state.LoadedBuffers[pageNumber + i] = buffer; } encBuffer.OriginalSize = encBuffer.Size; encBuffer.Size = Constants.Storage.PageSize; // here we _intentionally_ don't modify the hash of the page, even though its size was // changed, because we need the pager to recognize that it was modified on tx commit // encBuffer.Hash = remains the same }
private EncryptionBuffer GetBufferAndAddToTxState(long pageNumber, CryptoTransactionState state, int size) { var ptr = _encryptionBuffersPool.Get(size, out var thread); var buffer = new EncryptionBuffer { Size = size, Pointer = ptr, AllocatingThread = thread }; state.LoadedBuffers[pageNumber] = buffer; return(buffer); }
public void clear_buffers_only_when_in_extremely_low_memory() { var encryptionBuffersPool = new EncryptionBuffersPool(); var ptr = encryptionBuffersPool.Get(1, out _); var stats = encryptionBuffersPool.GetStats(); Assert.Equal(0, stats.TotalSize); encryptionBuffersPool.Return(ptr, 1, NativeMemory.ThreadAllocations.Value, encryptionBuffersPool.Generation); stats = encryptionBuffersPool.GetStats(); Assert.Equal(1, stats.TotalSize); encryptionBuffersPool.LowMemory(LowMemorySeverity.Low); stats = encryptionBuffersPool.GetStats(); Assert.Equal(1, stats.TotalSize); ClearMemory(encryptionBuffersPool); }
public void dont_pool_buffers_larger_than_8Mb() { var encryptionBuffersPool = new EncryptionBuffersPool(); var i = 1; var toFree = new List <(IntPtr, long)>(); while (i < new Size(64, SizeUnit.Megabytes).GetDoubleValue(SizeUnit.Bytes)) { var ptr = encryptionBuffersPool.Get(i, out _); toFree.Add(((IntPtr)ptr, i)); i *= 2; } var maxSize = new Size(8, SizeUnit.Megabytes).GetValue(SizeUnit.Bytes); var totalSize = 0L; foreach (var o in toFree) { if (o.Item2 <= maxSize) { totalSize += o.Item2; } encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, encryptionBuffersPool.Generation); } var stats = encryptionBuffersPool.GetStats(); Assert.Equal(totalSize, stats.TotalSize); i = 1; foreach (var allocationInfo in stats.Details) { Assert.Equal(1, allocationInfo.NumberOfItems); Assert.Equal(i, allocationInfo.AllocationSize); i *= 2; } ClearMemory(encryptionBuffersPool); }
public void can_save_buffers_after_low_memory() { var encryptionBuffersPool = new EncryptionBuffersPool(); encryptionBuffersPool.LowMemory(LowMemorySeverity.ExtremelyLow); encryptionBuffersPool.LowMemoryOver(); var i = 1; var toFree = new List <(IntPtr, long)>(); var generation = encryptionBuffersPool.Generation; while (i <= 1024) { var ptr = encryptionBuffersPool.Get(i, out var size, out _); toFree.Add(((IntPtr)ptr, i)); i *= 2; } var stats = encryptionBuffersPool.GetStats(); Assert.Equal(0, stats.TotalPoolSize); foreach (var o in toFree) { encryptionBuffersPool.Return((byte *)o.Item1, o.Item2, NativeMemory.ThreadAllocations.Value, generation); } stats = encryptionBuffersPool.GetStats(); var allocated = toFree.Sum(x => x.Item2); Assert.Equal(allocated, stats.TotalPoolSize); ClearMemory(encryptionBuffersPool); }