public PageFromScratchBuffer Allocate(LowLevelTransaction tx, int numberOfPages)
{
if (tx == null)
{
throw new ArgumentNullException(nameof(tx));
}
var size = Bits.PowerOf2(numberOfPages);
var current = _current;
PageFromScratchBuffer result;
if (current.File.TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
// we can allocate from the end of the file directly
if (current.File.LastUsedPage + size <= current.File.NumberOfAllocatedPages)
{
return(current.File.Allocate(tx, numberOfPages, size));
}
if (current.File.Size < _options.MaxScratchBufferSize)
{
var numberOfPagesBeforeAllocate = current.File.NumberOfAllocatedPages;
var page = current.File.Allocate(tx, numberOfPages, size);
if (current.File.NumberOfAllocatedPages > numberOfPagesBeforeAllocate)
{
_scratchSpaceMonitor.Increase((current.File.NumberOfAllocatedPages - numberOfPagesBeforeAllocate) * Constants.Storage.PageSize);
}
return(page);
}
var minSize = numberOfPages * Constants.Storage.PageSize;
var requestedSize = Math.Max(minSize, Math.Min(_current.File.Size * 2, _options.MaxScratchBufferSize));
// We need to ensure that _current stays constant through the codepath until return.
current = NextFile(minSize, requestedSize);
try
{
var scratchPagerState = current.File.PagerState;
tx.EnsurePagerStateReference(ref scratchPagerState);
return(current.File.Allocate(tx, numberOfPages, size));
}
finally
{
// That's why we update only after exiting.
_current = current;
}
}
public PageFromScratchBuffer Allocate(LowLevelTransaction tx, int numberOfPages, int sizeToAllocate)
{
var pagerState = _scratchPager.EnsureContinuous(_lastUsedPage, sizeToAllocate);
tx?.EnsurePagerStateReference(pagerState);
var result = new PageFromScratchBuffer(_scratchNumber, _lastUsedPage, sizeToAllocate, numberOfPages);
_allocatedPagesCount += numberOfPages;
_allocatedPages.Add(_lastUsedPage, result);
_lastUsedPage += sizeToAllocate;
return result;
}
public virtual byte *AcquirePagePointer(LowLevelTransaction tx, long pageNumber, PagerState pagerState = null)
{
if (Disposed)
{
ThrowAlreadyDisposedException();
}
if (pageNumber > NumberOfAllocatedPages)
{
ThrowOnInvalidPageNumber(pageNumber);
}
var state = pagerState ?? _pagerState;
tx?.EnsurePagerStateReference(state);
return(state.MapBase + pageNumber * _pageSize);
}
private void EnsureDataPagerSpacing(LowLevelTransaction transaction, Page last, int numberOfPagesInLastPage,
bool alreadyInWriteTx)
{
if (_waj._dataPager.WillRequireExtension(last.PageNumber, numberOfPagesInLastPage) == false)
{
return;
}
if (alreadyInWriteTx)
{
var pagerState = _waj._dataPager.EnsureContinuous(last.PageNumber, numberOfPagesInLastPage);
transaction.EnsurePagerStateReference(pagerState);
}
else
{
using (var tx = _waj._env.NewLowLevelTransaction(TransactionFlags.ReadWrite).JournalApplicatorTransaction())
{
var pagerState = _waj._dataPager.EnsureContinuous(last.PageNumber, numberOfPagesInLastPage);
tx.EnsurePagerStateReference(pagerState);
tx.Commit();
}
}
}
private IntPtr[] CompressPages(LowLevelTransaction tx, int numberOfPages, AbstractPager compressionPager)
{
// numberOfPages include the tx header page, which we don't compress
var dataPagesCount = numberOfPages - 1;
int pageSize = tx.Environment.Options.PageSize;
var sizeInBytes = dataPagesCount * pageSize;
// We want to include the Transaction Header straight into the compression buffer.
var outputBufferSize = LZ4.MaximumOutputLength(sizeInBytes) + sizeof(TransactionHeader);
var outputBufferInPages = outputBufferSize / pageSize +
(outputBufferSize % pageSize == 0 ? 0 : 1);
// The pages required includes the intermediate pages and the required output pages.
var pagesRequired = (dataPagesCount + outputBufferInPages);
var pagerState = compressionPager.EnsureContinuous(0, pagesRequired);
tx.EnsurePagerStateReference(pagerState);
// We get the pointer to the compression buffer, which will be the buffer that will hold the whole thing.
var outputBuffer = compressionPager.AcquirePagePointer(tx, dataPagesCount);
// Where we are going to store the input data continously to compress it afterwards.
var tempBuffer = compressionPager.AcquirePagePointer(tx, 0);
var txPages = tx.GetTransactionPages();
var write = tempBuffer;
foreach (var txPage in txPages)
{
var scratchPage = tx.Environment.ScratchBufferPool.AcquirePagePointer(tx, txPage.ScratchFileNumber, txPage.PositionInScratchBuffer);
var count = txPage.NumberOfPages * pageSize;
Memory.Copy(write, scratchPage, count);
write += count;
}
var compressionBuffer = outputBuffer + sizeof(TransactionHeader);
var len = DoCompression(tempBuffer, compressionBuffer, sizeInBytes, outputBufferSize);
int totalLength = len + sizeof(TransactionHeader); // We need to account for the transaction header as part of the total length.
var remainder = totalLength % pageSize;
var compressedPages = (totalLength / pageSize) + (remainder == 0 ? 0 : 1);
if (remainder != 0)
{
// zero the remainder of the page
UnmanagedMemory.Set(outputBuffer + totalLength, 0, pageSize - remainder);
}
var txHeaderPage = tx.GetTransactionHeaderPage();
var txHeaderBase = tx.Environment.ScratchBufferPool.AcquirePagePointer(tx, txHeaderPage.ScratchFileNumber, txHeaderPage.PositionInScratchBuffer);
var txHeader = (TransactionHeader *)txHeaderBase;
txHeader->Compressed = true;
txHeader->CompressedSize = len;
txHeader->UncompressedSize = sizeInBytes;
txHeader->Hash = Hashing.XXHash64.Calculate(compressionBuffer, len);
// Copy the transaction header to the output buffer.
Memory.Copy(outputBuffer, txHeaderBase, sizeof(TransactionHeader));
var pages = new IntPtr[compressedPages];
for (int index = 0; index < compressedPages; index++)
{
pages[index] = new IntPtr(outputBuffer + (index * pageSize));
}
return(pages);
}
internal LowLevelTransaction NewLowLevelTransaction(TransactionPersistentContext transactionPersistentContext, TransactionFlags flags, ByteStringContext context = null, TimeSpan?timeout = null)
{
_cancellationTokenSource.Token.ThrowIfCancellationRequested();
bool txLockTaken = false;
bool flushInProgressReadLockTaken = false;
try
{
IncrementUsageOnNewTransaction();
if (flags == TransactionFlags.ReadWrite)
{
var wait = timeout ?? (Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30));
if (FlushInProgressLock.IsWriteLockHeld == false)
{
flushInProgressReadLockTaken = FlushInProgressLock.TryEnterReadLock(wait);
}
txLockTaken = _transactionWriter.Wait(wait);
if (txLockTaken == false || (flushInProgressReadLockTaken == false &&
FlushInProgressLock.IsWriteLockHeld == false))
{
GlobalFlushingBehavior.GlobalFlusher.Value.MaybeFlushEnvironment(this);
ThrowOnTimeoutWaitingForWriteTxLock(wait);
}
_cancellationTokenSource.Token.ThrowIfCancellationRequested();
_currentTransactionHolder = NativeMemory.ThreadAllocations.Value;
WriteTransactionStarted();
if (_endOfDiskSpace != null)
{
_endOfDiskSpace.AssertCanContinueWriting();
_endOfDiskSpace = null;
Task.Run(IdleFlushTimer);
GlobalFlushingBehavior.GlobalFlusher.Value.MaybeFlushEnvironment(this);
}
}
LowLevelTransaction tx;
_txCommit.EnterReadLock();
try
{
_cancellationTokenSource.Token.ThrowIfCancellationRequested();
if (_currentTransactionHolder == null)
{
_currentTransactionHolder = NativeMemory.ThreadAllocations.Value;
}
long txId = flags == TransactionFlags.ReadWrite ? NextWriteTransactionId : CurrentReadTransactionId;
tx = new LowLevelTransaction(this, txId, transactionPersistentContext, flags, _freeSpaceHandling,
context)
{
FlushInProgressLockTaken = flushInProgressReadLockTaken,
CurrentTransactionHolder = _currentTransactionHolder
};
ActiveTransactions.Add(tx);
}
finally
{
_txCommit.ExitReadLock();
}
var state = _dataPager.PagerState;
tx.EnsurePagerStateReference(state);
return(tx);
}
catch (Exception)
{
try
{
if (txLockTaken)
{
_transactionWriter.Release();
}
if (flushInProgressReadLockTaken)
{
FlushInProgressLock.ExitReadLock();
}
}
finally
{
DecrementUsageOnTransactionCreationFailure();
}
throw;
}
}
public PageFromScratchBuffer Allocate(LowLevelTransaction tx, int numberOfPages)
{
if (tx == null)
{
throw new ArgumentNullException(nameof(tx));
}
var size = Bits.NextPowerOf2(numberOfPages);
PageFromScratchBuffer result;
var current = _current;
if (current.File.TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
long sizeAfterAllocation;
long oldestActiveTransaction = tx.Environment.OldestTransaction;
if (_scratchBuffers.Count == 1)
{
sizeAfterAllocation = current.File.SizeAfterAllocation(size);
}
else
{
sizeAfterAllocation = size * tx.Environment.Options.PageSize;
var scratchesToDelete = new List <int>();
sizeAfterAllocation += current.File.ActivelyUsedBytes(oldestActiveTransaction);
// determine how many bytes of older scratches are still in use
foreach (var scratch in _scratchBuffers.Values)
{
var bytesInUse = scratch.File.ActivelyUsedBytes(oldestActiveTransaction);
if (bytesInUse <= 0)
{
if (scratch != current)
{
scratchesToDelete.Add(scratch.Number);
}
}
}
// delete inactive scratches
foreach (var scratchNumber in scratchesToDelete)
{
ScratchBufferItem scratchBufferToRemove;
if (_scratchBuffers.TryRemove(scratchNumber, out scratchBufferToRemove))
{
scratchBufferToRemove.File.Dispose();
}
}
}
if (sizeAfterAllocation >= (_sizeLimit * 3) / 4 && oldestActiveTransaction > current.OldestTransactionWhenFlushWasForced)
{
// we may get recursive flushing, so we want to avoid it
if (tx.Environment.Journal.Applicator.IsCurrentThreadInFlushOperation == false)
{
// We are starting to force a flush to free scratch pages. We are doing it at this point (80% of the max scratch size)
// to make sure that next transactions will be able to allocate pages that we are going to free in the current transaction.
// Important notice: all pages freed by this run will get ValidAfterTransactionId == tx.Id (so only next ones can use it)
bool flushLockTaken = false;
using (tx.Environment.Journal.Applicator.TryTakeFlushingLock(ref flushLockTaken))
{
if (flushLockTaken) // if we are already flushing, we don't need to force a flush
{
try
{
tx.Environment.ForceLogFlushToDataFile(tx, allowToFlushOverwrittenPages: true);
current.OldestTransactionWhenFlushWasForced = oldestActiveTransaction;
}
catch (TimeoutException)
{
// we'll try next time
}
catch (InvalidJournalFlushRequestException)
{
// journals flushing already in progress
}
}
}
}
}
if (sizeAfterAllocation > _sizeLimit)
{
var sp = Stopwatch.StartNew();
// Our problem is that we don't have any available free pages, probably because
// there are read transactions that are holding things open. We are going to see if
// there are any free pages that _might_ be freed for us if we wait for a bit. The idea
// is that we let the read transactions time to complete and do their work, at which point
// we can continue running. It is possible that a long running read transaction
// would in fact generate enough work for us to timeout, but hopefully we can avoid that.
while (tx.IsLazyTransaction == false && // lazy transaction is holding a read tx that will stop this, nothing to do here
tx.Environment.Options.ManualFlushing == false &&
sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout)
{
if (current.File.TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
Thread.Sleep(32);
}
sp.Stop();
bool createNextFile = false;
if (tx.IsLazyTransaction)
{
// in lazy transaction when reaching full scratch buffer - we might still have continuous space, but because
// of high insertion rate - we reach scratch buffer full, so we will create new scratch anyhow
createNextFile = true;
}
else if (current.File.HasDiscontinuousSpaceFor(tx, size, _scratchBuffers.Count))
{
// there is enough space for the requested allocation but the problem is its fragmentation
// so we will create a new scratch file and will allow to allocate new continuous range from there
createNextFile = true;
}
else if (_scratchBuffers.Count == 1 && current.File.Size < _sizeLimit &&
(current.File.ActivelyUsedBytes(oldestActiveTransaction) + size * tx.Environment.Options.PageSize) < _sizeLimit)
{
// there is only one scratch file that hasn't reach the size limit yet and
// the number of bytes being in active use allows to allocate the requested size
// let it create a new file
createNextFile = true;
}
if (createNextFile)
{
// We need to ensure that _current stays constant through the codepath until return.
current = NextFile();
try
{
current.File.PagerState.AddRef();
tx.EnsurePagerStateReference(current.File.PagerState);
return(current.File.Allocate(tx, numberOfPages, size));
}
finally
{
// That's why we update only after exiting.
_current = current;
}
}
ThrowScratchBufferTooBig(tx, numberOfPages, size, oldestActiveTransaction, sizeAfterAllocation, sp, current);
}
// we don't have free pages to give out, need to allocate some
result = current.File.Allocate(tx, numberOfPages, size);
_options.OnScratchBufferSizeChanged(sizeAfterAllocation);
return(result);
}
internal LowLevelTransaction NewLowLevelTransaction(TransactionPersistentContext transactionPersistentContext, TransactionFlags flags, ByteStringContext context = null, TimeSpan?timeout = null)
{
bool txLockTaken = false;
bool flushInProgressReadLockTaken = false;
try
{
if (flags == TransactionFlags.ReadWrite)
{
var wait = timeout ?? (Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30));
if (FlushInProgressLock.IsWriteLockHeld == false)
{
flushInProgressReadLockTaken = FlushInProgressLock.TryEnterReadLock(wait);
}
if (Monitor.IsEntered(_txWriter))
{
ThrowOnRecursiveWriteTransaction();
}
Monitor.TryEnter(_txWriter, wait, ref txLockTaken);
if (txLockTaken == false || (flushInProgressReadLockTaken == false && FlushInProgressLock.IsWriteLockHeld == false))
{
GlobalFlushingBehavior.GlobalFlusher.Value.MaybeFlushEnvironment(this);
ThrowOnTimeoutWaitingForWriteTxLock(wait);
}
_writeTransactionRunning.SetByAsyncCompletion();
if (_endOfDiskSpace != null)
{
if (_endOfDiskSpace.CanContinueWriting)
{
CatastrophicFailure = null;
_endOfDiskSpace = null;
_cancellationTokenSource = new CancellationTokenSource();
Task.Run(IdleFlushTimer);
GlobalFlushingBehavior.GlobalFlusher.Value.MaybeFlushEnvironment(this);
}
}
}
LowLevelTransaction tx;
_txCommit.EnterReadLock();
try
{
long txId = flags == TransactionFlags.ReadWrite ? _transactionsCounter + 1 : _transactionsCounter;
tx = new LowLevelTransaction(this, txId, transactionPersistentContext, flags, _freeSpaceHandling,
context)
{
FlushInProgressLockTaken = flushInProgressReadLockTaken
};
ActiveTransactions.Add(tx);
}
finally
{
_txCommit.ExitReadLock();
}
var state = _dataPager.PagerState;
tx.EnsurePagerStateReference(state);
return(tx);
}
catch (Exception)
{
if (txLockTaken)
{
Monitor.Exit(_txWriter);
}
if (flushInProgressReadLockTaken)
{
FlushInProgressLock.ExitReadLock();
}
throw;
}
}
internal LowLevelTransaction NewLowLevelTransaction(TransactionFlags flags, ByteStringContext context = null, TimeSpan?timeout = null)
{
bool txLockTaken = false;
bool flushInProgressReadLockTaken = false;
try
{
if (flags == TransactionFlags.ReadWrite)
{
var wait = timeout ?? (Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30));
if (FlushInProgressLock.IsWriteLockHeld == false)
{
flushInProgressReadLockTaken = FlushInProgressLock.TryEnterReadLock(wait);
}
Monitor.TryEnter(_txWriter, wait, ref txLockTaken);
if (txLockTaken == false || (flushInProgressReadLockTaken == false && FlushInProgressLock.IsWriteLockHeld == false))
{
GlobalFlusher.Value.MaybeFlushEnvironment(this);
throw new TimeoutException("Waited for " + wait +
" for transaction write lock, but could not get it");
}
if (_endOfDiskSpace != null)
{
if (_endOfDiskSpace.CanContinueWriting)
{
_flushingTaskFailure = null;
_endOfDiskSpace = null;
_cancellationTokenSource = new CancellationTokenSource();
Task.Run(IdleFlushTimer);
GlobalFlusher.Value.MaybeFlushEnvironment(this);
}
}
}
LowLevelTransaction tx;
_txCommit.EnterReadLock();
try
{
long txId = flags == TransactionFlags.ReadWrite ? _transactionsCounter + 1 : _transactionsCounter;
tx = new LowLevelTransaction(this, txId, flags, _freeSpaceHandling, context);
}
finally
{
_txCommit.ExitReadLock();
}
_activeTransactions.Add(tx);
var state = _dataPager.PagerState;
tx.EnsurePagerStateReference(state);
return(tx);
}
catch (Exception)
{
if (txLockTaken)
{
Monitor.Exit(_txWriter);
}
if (flushInProgressReadLockTaken)
{
FlushInProgressLock.ExitReadLock();
}
throw;
}
}