private static void ThrowUnableToRemoveScratch(ScratchBufferItem scratchBufferItem)
{
throw new InvalidOperationException(
$"Could not remove a scratch file from the scratch buffers collection. Number: {scratchBufferItem.Number}");
}
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages)
{
if (tx == null)
{
throw new ArgumentNullException("tx");
}
var size = Utils.NearestPowerOfTwo(numberOfPages);
PageFromScratchBuffer result;
var current = _current;
var currentFile = current.File;
if (currentFile.TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
long sizeAfterAllocation;
long oldestActiveTransaction = tx.Environment.OldestTransaction;
if (_scratchBuffers.Count == 1)
{
sizeAfterAllocation = currentFile.SizeAfterAllocation(size);
}
else
{
sizeAfterAllocation = size * AbstractPager.PageSize;
var scratchesToDelete = new List <int>();
// determine how many bytes of older scratches are still in use (at least when this snapshot is taken)
foreach (var scratch in _scratchBuffers.Values)
{
var bytesInUse = scratch.File.ActivelyUsedBytes(oldestActiveTransaction);
if (bytesInUse > 0)
{
sizeAfterAllocation += bytesInUse;
}
else
{
if (scratch != current)
{
scratchesToDelete.Add(scratch.Number);
}
}
}
// delete inactive scratches
foreach (var scratchNumber in scratchesToDelete)
{
ScratchBufferItem scratchBufferToRemove;
if (_scratchBuffers.TryRemove(scratchNumber, out scratchBufferToRemove))
{
scratchBufferToRemove.File.Dispose();
}
}
}
if (sizeAfterAllocation >= (_sizeLimit * 3) / 4 && oldestActiveTransaction > current.OldestTransactionWhenFlushWasForced)
{
// we may get recursive flushing, so we want to avoid it
if (tx.Environment.Journal.Applicator.IsCurrentThreadInFlushOperation == false)
{
// We are starting to force a flush to free scratch pages. We are doing it at this point (80% of the max scratch size)
// to make sure that next transactions will be able to allocate pages that we are going to free in the current transaction.
// Important notice: all pages freed by this run will get ValidAfterTransactionId == tx.Id (so only next ones can use it)
bool flushLockTaken = false;
using (tx.Environment.Journal.Applicator.TryTakeFlushingLock(ref flushLockTaken))
{
if (flushLockTaken) // if we are already flushing, we don't need to force a flush
{
try
{
tx.Environment.ForceLogFlushToDataFile(tx, allowToFlushOverwrittenPages: true);
current.OldestTransactionWhenFlushWasForced = oldestActiveTransaction;
}
catch (TimeoutException)
{
// we'll try next time
}
catch (InvalidJournalFlushRequest)
{
// journals flushing already in progress
}
}
}
}
}
if (sizeAfterAllocation > _sizeLimit)
{
var sp = Stopwatch.StartNew();
// Our problem is that we don't have any available free pages, probably because
// there are read transactions that are holding things open. We are going to see if
// there are any free pages that _might_ be freed for us if we wait for a bit. The idea
// is that we let the read transactions time to complete and do their work, at which point
// we can continue running. It is possible that a long running read transaction
// would in fact generate enough work for us to timeout, but hopefully we can avoid that.
while (sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout)
{
if (currentFile.TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
Thread.Sleep(32);
}
sp.Stop();
bool createNextFile = false;
if (currentFile.HasDiscontinuousSpaceFor(tx, size, _scratchBuffers.Count))
{
// there is enough space for the requested allocation but the problem is its fragmentation
// so we will create a new scratch file and will allow to allocate new continuous range from there
createNextFile = true;
}
else if (_scratchBuffers.Count == 1 && currentFile.Size < _sizeLimit &&
(currentFile.ActivelyUsedBytes(oldestActiveTransaction) + size * AbstractPager.PageSize) < _sizeLimit)
{
// there is only one scratch file that hasn't reach the size limit yet and
// the number of bytes being in active use allows to allocate the requested size
// let it create a new file
createNextFile = true;
}
if (createNextFile)
{
// We need to ensure that _current stays constant through the codepath until return.
current = NextFile();
try
{
tx.EnsurePagerStateReference(_current.File.PagerState);
return(current.File.Allocate(tx, numberOfPages, size));
}
finally
{
// That's why we update only after exiting.
_current = current;
}
}
var debugInfoBuilder = new StringBuilder();
debugInfoBuilder.AppendFormat("Current transaction id: {0}\r\n", tx.Id);
debugInfoBuilder.AppendFormat("Requested number of pages: {0} (adjusted size: {1} == {2:#,#;;0} KB)\r\n", numberOfPages, size, size * AbstractPager.PageSize / 1024);
debugInfoBuilder.AppendFormat("Oldest active transaction: {0} (snapshot: {1})\r\n", tx.Environment.OldestTransaction, oldestActiveTransaction);
debugInfoBuilder.AppendFormat("Oldest active transaction when flush was forced: {0}\r\n", current.OldestTransactionWhenFlushWasForced);
debugInfoBuilder.AppendFormat("Next write transaction id: {0}\r\n", tx.Environment.NextWriteTransactionId + 1);
debugInfoBuilder.AppendLine("Active transactions:");
foreach (var activeTransaction in tx.Environment.ActiveTransactions)
{
debugInfoBuilder.AppendFormat("\tId: {0} - {1}\r\n", activeTransaction.Id, activeTransaction.Flags);
}
debugInfoBuilder.AppendLine("Scratch files usage:");
foreach (var scratchBufferItem in _scratchBuffers.Values.OrderBy(x => x.Number))
{
debugInfoBuilder.AppendFormat("\t{0} - size: {1:#,#;;0} KB, in active use: {2:#,#;;0} KB\r\n", StorageEnvironmentOptions.ScratchBufferName(scratchBufferItem.File.Number), scratchBufferItem.File.Size / 1024, scratchBufferItem.File.ActivelyUsedBytes(oldestActiveTransaction) / 1024);
}
debugInfoBuilder.AppendLine("Most available free pages:");
foreach (var scratchBufferFile in _scratchBuffers.OrderBy(x => x.Key))
{
debugInfoBuilder.AppendFormat("\t{0}\r\n", StorageEnvironmentOptions.ScratchBufferName(scratchBufferFile.Value.Number));
foreach (var freePage in scratchBufferFile.Value.File.GetMostAvailableFreePagesBySize())
{
debugInfoBuilder.AppendFormat("\t\tSize:{0}, ValidAfterTransactionId: {1}\r\n", freePage.Key, freePage.Value);
}
}
debugInfoBuilder.AppendFormat("Compression buffer size: {0:#,#;;0} KB\r\n", tx.Environment.Journal.CompressionBufferSize / 1024);
string debugInfo = debugInfoBuilder.ToString();
string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" +
"Current file size is:\t{0:#,#;;0} KB.\r\n" +
"Requested size for current file:\t{1:#,#;;0} KB.\r\n" +
"Requested total size for all files:\t{2:#,#;;0} KB.\r\n" +
"Limit:\t\t\t{3:#,#;;0} KB.\r\n" +
"Already flushed and waited for {4:#,#;;0} ms for read transactions to complete.\r\n" +
"Do you have a long running read transaction executing?\r\n" +
"Debug info:\r\n{5}",
currentFile.Size / 1024L,
currentFile.SizeAfterAllocation(size) / 1024L,
sizeAfterAllocation / 1024L,
_sizeLimit / 1024L,
sp.ElapsedMilliseconds,
debugInfo
);
throw new ScratchBufferSizeLimitException(message);
}
// we don't have free pages to give out, need to allocate some
result = currentFile.Allocate(tx, numberOfPages, size);
_options.OnScratchBufferSizeChanged(sizeAfterAllocation);
return(result);
}
private void AddScratchBufferFile(ScratchBufferItem scratch)
{
RemoveInactiveScratches(scratch);
_scratchBuffers.AddOrUpdate(scratch.Number, scratch, (_, __) => scratch);
}
public ScratchBufferPool(StorageEnvironment env)
{
_options = env.Options;
_sizeLimit = env.Options.MaxScratchBufferSize;
_current = NextFile();
}
public PageFromScratchBuffer Allocate(LowLevelTransaction tx, int numberOfPages)
{
if (tx == null)
{
throw new ArgumentNullException(nameof(tx));
}
var size = Bits.NextPowerOf2(numberOfPages);
PageFromScratchBuffer result;
var current = _current;
if (current.File.TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
long sizeAfterAllocation;
long oldestActiveTransaction = tx.Environment.OldestTransaction;
if (_scratchBuffers.Count == 1)
{
sizeAfterAllocation = current.File.SizeAfterAllocation(size);
}
else
{
sizeAfterAllocation = size * tx.Environment.Options.PageSize;
var scratchesToDelete = new List <int>();
sizeAfterAllocation += current.File.ActivelyUsedBytes(oldestActiveTransaction);
// determine how many bytes of older scratches are still in use
foreach (var scratch in _scratchBuffers.Values)
{
var bytesInUse = scratch.File.ActivelyUsedBytes(oldestActiveTransaction);
if (bytesInUse <= 0)
{
if (scratch != current)
{
scratchesToDelete.Add(scratch.Number);
}
}
}
// delete inactive scratches
foreach (var scratchNumber in scratchesToDelete)
{
ScratchBufferItem scratchBufferToRemove;
if (_scratchBuffers.TryRemove(scratchNumber, out scratchBufferToRemove))
{
scratchBufferToRemove.File.Dispose();
}
}
}
if (sizeAfterAllocation >= (_sizeLimit * 3) / 4 && oldestActiveTransaction > current.OldestTransactionWhenFlushWasForced)
{
// we may get recursive flushing, so we want to avoid it
if (tx.Environment.Journal.Applicator.IsCurrentThreadInFlushOperation == false)
{
// We are starting to force a flush to free scratch pages. We are doing it at this point (80% of the max scratch size)
// to make sure that next transactions will be able to allocate pages that we are going to free in the current transaction.
// Important notice: all pages freed by this run will get ValidAfterTransactionId == tx.Id (so only next ones can use it)
bool flushLockTaken = false;
using (tx.Environment.Journal.Applicator.TryTakeFlushingLock(ref flushLockTaken))
{
if (flushLockTaken) // if we are already flushing, we don't need to force a flush
{
try
{
tx.Environment.ForceLogFlushToDataFile(tx, allowToFlushOverwrittenPages: true);
current.OldestTransactionWhenFlushWasForced = oldestActiveTransaction;
}
catch (TimeoutException)
{
// we'll try next time
}
catch (InvalidJournalFlushRequestException)
{
// journals flushing already in progress
}
}
}
}
}
if (sizeAfterAllocation > _sizeLimit)
{
var sp = Stopwatch.StartNew();
// Our problem is that we don't have any available free pages, probably because
// there are read transactions that are holding things open. We are going to see if
// there are any free pages that _might_ be freed for us if we wait for a bit. The idea
// is that we let the read transactions time to complete and do their work, at which point
// we can continue running. It is possible that a long running read transaction
// would in fact generate enough work for us to timeout, but hopefully we can avoid that.
while (tx.IsLazyTransaction == false && // lazy transaction is holding a read tx that will stop this, nothing to do here
tx.Environment.Options.ManualFlushing == false &&
sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout)
{
if (current.File.TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
Thread.Sleep(32);
}
sp.Stop();
bool createNextFile = false;
if (tx.IsLazyTransaction)
{
// in lazy transaction when reaching full scratch buffer - we might still have continuous space, but because
// of high insertion rate - we reach scratch buffer full, so we will create new scratch anyhow
createNextFile = true;
}
else if (current.File.HasDiscontinuousSpaceFor(tx, size, _scratchBuffers.Count))
{
// there is enough space for the requested allocation but the problem is its fragmentation
// so we will create a new scratch file and will allow to allocate new continuous range from there
createNextFile = true;
}
else if (_scratchBuffers.Count == 1 && current.File.Size < _sizeLimit &&
(current.File.ActivelyUsedBytes(oldestActiveTransaction) + size * tx.Environment.Options.PageSize) < _sizeLimit)
{
// there is only one scratch file that hasn't reach the size limit yet and
// the number of bytes being in active use allows to allocate the requested size
// let it create a new file
createNextFile = true;
}
if (createNextFile)
{
// We need to ensure that _current stays constant through the codepath until return.
current = NextFile();
try
{
current.File.PagerState.AddRef();
tx.EnsurePagerStateReference(current.File.PagerState);
return(current.File.Allocate(tx, numberOfPages, size));
}
finally
{
// That's why we update only after exiting.
_current = current;
}
}
ThrowScratchBufferTooBig(tx, numberOfPages, size, oldestActiveTransaction, sizeAfterAllocation, sp, current);
}
// we don't have free pages to give out, need to allocate some
result = current.File.Allocate(tx, numberOfPages, size);
_options.OnScratchBufferSizeChanged(sizeAfterAllocation);
return(result);
}
private void ThrowScratchBufferTooBig(LowLevelTransaction tx, int numberOfPages, long size, long oldestActiveTransaction,
long sizeAfterAllocation, Stopwatch sp, ScratchBufferItem current)
{
var debugInfoBuilder = new StringBuilder();
var totalPages = tx.GetTransactionPages().Count;
debugInfoBuilder.AppendFormat("Current transaction id: {0}\r\n", tx.Id);
if ((totalPages + numberOfPages) * tx.Environment.Options.PageSize >= _sizeLimit / 2)
{
debugInfoBuilder.Append("- - - - - - - - - - - - -\r\n");
debugInfoBuilder.AppendFormat(
"This transaction is VERY big, and requires {0:##,###;;0} kb out of {1:##,###;;0} kb allows!\r\n",
((totalPages + numberOfPages) * tx.Environment.Options.PageSize) / 1024,
_sizeLimit / 1024
);
debugInfoBuilder.Append("- - - - - - - - - - - - -\r\n");
}
debugInfoBuilder.AppendFormat("Requested number of pages: {0} (adjusted size: {1} == {2:#,#;;0} KB)\r\n", numberOfPages,
size, size * tx.Environment.Options.PageSize / 1024);
debugInfoBuilder.AppendFormat("Total number of pages in tx: {0} (adjusted size: {1} == {2:#,#;;0} KB)\r\n", totalPages,
totalPages, totalPages * tx.Environment.Options.PageSize / 1024);
debugInfoBuilder.AppendFormat("Oldest active transaction: {0} (snapshot: {1})\r\n", tx.Environment.OldestTransaction,
oldestActiveTransaction);
debugInfoBuilder.AppendFormat("Oldest active transaction when flush was forced: {0}\r\n",
current.OldestTransactionWhenFlushWasForced);
debugInfoBuilder.AppendFormat("Next write transaction id: {0}\r\n", tx.Environment.NextWriteTransactionId + 1);
debugInfoBuilder.AppendLine("Active transactions:");
foreach (var activeTransaction in tx.Environment.ActiveTransactions)
{
debugInfoBuilder.AppendFormat("\tId: {0} - {1}\r\n", activeTransaction.Id, activeTransaction.Flags);
}
debugInfoBuilder.AppendLine("Scratch files usage:");
foreach (var scratchBufferFile in _scratchBuffers.OrderBy(x => x.Key))
{
debugInfoBuilder.AppendFormat("\t{0} - size: {1:#,#;;0} KB, in active use: {2:#,#;;0} KB\r\n",
StorageEnvironmentOptions.ScratchBufferName(scratchBufferFile.Value.Number), scratchBufferFile.Value.File.Size / 1024,
scratchBufferFile.Value.File.ActivelyUsedBytes(oldestActiveTransaction) / 1024);
}
debugInfoBuilder.AppendLine("Most available free pages:");
foreach (var scratchBufferFile in _scratchBuffers.OrderBy(x => x.Key))
{
debugInfoBuilder.AppendFormat("\t{0}\r\n", StorageEnvironmentOptions.ScratchBufferName(scratchBufferFile.Value.Number));
foreach (var freePage in scratchBufferFile.Value.File.GetMostAvailableFreePagesBySize())
{
debugInfoBuilder.AppendFormat("\t\tSize:{0}, ValidAfterTransactionId: {1}\r\n", freePage.Key, freePage.Value);
}
}
debugInfoBuilder.AppendFormat("Compression buffer size: {0:#,#;;0} KB\r\n",
tx.Environment.Journal.CompressionBufferSize / 1024);
string debugInfo = debugInfoBuilder.ToString();
string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" +
"Current file size is:\t{0:#,#;;0} KB.\r\n" +
"Requested size for current file:\t{1:#,#;;0} KB.\r\n" +
"Requested total size for all files:\t{2:#,#;;0} KB.\r\n" +
"Limit:\t\t\t{3:#,#;;0} KB.\r\n" +
"Already flushed and waited for {4:#,#;;0} ms for read transactions to complete.\r\n" +
"Do you have a long running read transaction executing?\r\n" +
"Debug info:\r\n{5}",
current.File.Size / 1024L,
current.File.SizeAfterAllocation(size) / 1024L,
sizeAfterAllocation / 1024L,
_sizeLimit / 1024L,
sp.ElapsedMilliseconds,
debugInfo
);
throw new ScratchBufferSizeLimitException(message);
}
private void AddScratchBufferFile(ScratchBufferItem scratch)
{
RemoveInactiveScratches(scratch, updateCacheBeforeDisposingScratch: true);
_scratchBuffers.AddOrUpdate(scratch.Number, scratch, (_, __) => scratch);
}
public void Free(LowLevelTransaction tx, int scratchNumber, long page, long?txId)
{
var scratch = _scratchBuffers[scratchNumber];
scratch.File.Free(page, txId);
if (scratch.File.AllocatedPagesCount != 0)
{
return;
}
List <ScratchBufferFile> recycledScratchesToDispose = null;
while (_recycleArea.First != null)
{
var recycledScratch = _recycleArea.First.Value;
if (IsLowMemory() == false &&
DateTime.UtcNow - recycledScratch.RecycledAt <= RecycledScratchFileTimeout)
{
break;
}
_recycleArea.RemoveFirst();
if (recycledScratch.File.HasActivelyUsedBytes(_env.PossibleOldestReadTransaction(tx)))
{
// even though this was in the recycle area, there might still be some transactions looking at it
// so we cannot dispose it right now, the disposal will happen in RemoveInactiveScratches
// when we are sure it's really no longer in use
continue;
}
_scratchBuffers.TryRemove(recycledScratch.Number, out var _);
if (recycledScratchesToDispose == null)
{
recycledScratchesToDispose = new List <ScratchBufferFile>();
}
recycledScratchesToDispose.Add(recycledScratch.File);
}
if (recycledScratchesToDispose != null)
{
using (_env.IsInPreventNewTransactionsMode == false ? _env.PreventNewTransactions() : (IDisposable)null)
{
// we're about to dispose recycled scratch pagers, we need to update the cache so next transactions won't attempt to EnsurePagerStateReference() on them
UpdateCacheForPagerStatesOfAllScratches();
}
foreach (var recycledScratch in recycledScratchesToDispose)
{
recycledScratch.Dispose();
_scratchSpaceMonitor.Decrease(recycledScratch.NumberOfAllocatedPages * Constants.Storage.PageSize);
}
_forTestingPurposes?.ActionToCallDuringRemovalsOfRecycledScratchesRightAfterDisposingScratches?.Invoke();
}
if (scratch == _current)
{
if (scratch.File.Size <= _options.MaxScratchBufferSize)
{
// we'll take the chance that no one is using us to reset the memory allocations
// and avoid fragmentation, we can only do that if no transaction is looking at us
if (scratch.File.HasActivelyUsedBytes(_env.PossibleOldestReadTransaction(tx)) == false)
{
scratch.File.Reset();
}
return;
}
// this is the current one, but the size is too big, let us trim it
var newCurrent = NextFile(_options.InitialLogFileSize, _options.MaxScratchBufferSize);
newCurrent.File.PagerState.AddRef();
_current = newCurrent;
}
TryRecycleScratchFile(scratch);
}
