private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result)
{
result = null;
LinkedList <PendingPage> list;
if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0)
{
return(false);
}
var val = list.First.Value;
if (val.ValidAfterTransactionId >= tx.Environment.OldestTransaction)
{
return(false);
}
list.RemoveFirst();
var pageFromScratchBuffer = new PageFromScratchBuffer
{
PositionInScratchBuffer = val.Page,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(val.Page, pageFromScratchBuffer);
{
result = pageFromScratchBuffer;
return(true);
}
}
private void InitTransactionHeader()
{
var allocation = _env.ScratchBufferPool.Allocate(this, 1);
var page = _env.ScratchBufferPool.ReadPage(allocation.ScratchFileNumber, allocation.PositionInScratchBuffer);
_transactionHeaderPage = allocation;
StdLib.memset(page.Base, 0, AbstractPager.PageSize);
_txHeader = (TransactionHeader *)page.Base;
_txHeader->HeaderMarker = Constants.TransactionHeaderMarker;
_txHeader->TransactionId = _id;
_txHeader->NextPageNumber = _state.NextPageNumber;
_txHeader->LastPageNumber = -1;
_txHeader->PageCount = -1;
_txHeader->Crc = 0;
_txHeader->TxMarker = TransactionMarker.None;
_txHeader->Compressed = false;
_txHeader->CompressedSize = 0;
_txHeader->UncompressedSize = 0;
_allocatedPagesInTransaction = 0;
_overflowPagesInTransaction = 0;
_scratchPagesTable.Clear();
}
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages)
{
var size = Utils.NearestPowerOfTwo(numberOfPages);
PageFromScratchBuffer result;
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
// we don't have free pages to give out, need to allocate some
_scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size);
result = new PageFromScratchBuffer
{
PositionInScratchBuffer = _lastUsedPage,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(_lastUsedPage, result);
_lastUsedPage += size;
return(result);
}
private void InitTransactionHeader()
{
var allocation = _env.ScratchBufferPool.Allocate(this, 1);
var page = _env.ScratchBufferPool.ReadPage(this, allocation.ScratchFileNumber, allocation.PositionInScratchBuffer);
_transactionHeaderPage = allocation;
UnmanagedMemory.Set(page.Pointer, 0, Environment.Options.PageSize);
_txHeader = (TransactionHeader *)page.Pointer;
_txHeader->HeaderMarker = Constants.TransactionHeaderMarker;
_txHeader->TransactionId = _id;
_txHeader->NextPageNumber = _state.NextPageNumber;
_txHeader->LastPageNumber = -1;
_txHeader->PageCount = -1;
_txHeader->Hash = 0;
_txHeader->TxMarker = TransactionMarker.None;
_txHeader->Compressed = false;
_txHeader->CompressedSize = 0;
_txHeader->UncompressedSize = 0;
_allocatedPagesInTransaction = 0;
_overflowPagesInTransaction = 0;
_scratchPagesTable.Clear();
}
internal void WriteDirect(TransactionHeader *transactionHeader, PageFromScratchBuffer pages)
{
for (int i = 0; i < pages.NumberOfPages; i++)
{
var page = _env.ScratchBufferPool.ReadPage(pages.ScratchFileNumber, pages.PositionInScratchBuffer + i);
int numberOfPages = 1;
if (page.IsOverflow)
{
numberOfPages = (page.OverflowSize / AbstractPager.PageSize) + (page.OverflowSize % AbstractPager.PageSize == 0 ? 0 : 1);
i += numberOfPages;
_overflowPagesInTransaction += (numberOfPages - 1);
}
var pageFromScratchBuffer = _env.ScratchBufferPool.Allocate(this, numberOfPages);
var dest = _env.ScratchBufferPool.AcquirePagePointer(pageFromScratchBuffer.ScratchFileNumber, pageFromScratchBuffer.PositionInScratchBuffer);
StdLib.memcpy(dest, page.Base, numberOfPages * AbstractPager.PageSize);
_allocatedPagesInTransaction++;
_dirtyPages.Add(page.PageNumber);
page.Dirty = true;
if (numberOfPages > 1)
{
_dirtyOverflowPages.Add(page.PageNumber + 1, numberOfPages - 1);
}
_scratchPagesTable[page.PageNumber] = pageFromScratchBuffer;
_transactionPages.Add(pageFromScratchBuffer);
_state.NextPageNumber = transactionHeader->NextPageNumber;
}
}
internal void BreakLargeAllocationToSeparatePages(long pageNumber)
{
if (_disposed != TxState.None)
{
ThrowObjectDisposed();
}
PageFromScratchBuffer value;
if (_scratchPagesTable.TryGetValue(pageNumber, out value) == false)
{
throw new InvalidOperationException("The page " + pageNumber + " was not previous allocated in this transaction");
}
if (value.NumberOfPages == 1)
{
return;
}
_transactionPages.Remove(value);
_env.ScratchBufferPool.BreakLargeAllocationToSeparatePages(this, value);
for (int i = 0; i < value.NumberOfPages; i++)
{
var pageFromScratchBuffer = new PageFromScratchBuffer(value.ScratchFileNumber, value.PositionInScratchBuffer + i, 1, 1);
_transactionPages.Add(pageFromScratchBuffer);
_scratchPagesTable[pageNumber + i] = pageFromScratchBuffer;
_dirtyPages.Add(pageNumber + i);
TrackDirtyPage(pageNumber + i);
var newPage = _env.ScratchBufferPool.ReadPage(this, value.ScratchFileNumber, value.PositionInScratchBuffer + i);
newPage.PageNumber = pageNumber + i;
newPage.Flags = PageFlags.Single;
TrackWritablePage(newPage);
}
}
private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result)
{
result = null;
LinkedList <PendingPage> list;
if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0)
{
return(false);
}
var val = list.Last.Value;
var oldestTransaction = tx.Environment.OldestTransaction;
if (oldestTransaction != 0 && val.ValidAfterTransactionId >= oldestTransaction) // OldestTransaction can be 0 when there are none other transactions and we are in process of new transaction header allocation
{
return(false);
}
list.RemoveLast();
var pageFromScratchBuffer = new PageFromScratchBuffer
{
PositionInScratchBuffer = val.Page,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(val.Page, pageFromScratchBuffer);
{
result = pageFromScratchBuffer;
return(true);
}
}
internal void WriteDirect(TransactionHeader *transactionHeader, PageFromScratchBuffer pages)
{
for (int i = 0; i < pages.NumberOfPages; i++)
{
var page = _env.ScratchBufferPool.ReadPage(pages.ScratchFileNumber, pages.PositionInScratchBuffer + i);
int numberOfPages = 1;
if (page.IsOverflow)
{
numberOfPages = (page.OverflowSize / AbstractPager.PageSize) + (page.OverflowSize % AbstractPager.PageSize == 0 ? 0 : 1);
i += numberOfPages;
_overflowPagesInTransaction += (numberOfPages - 1);
}
WritePageDirect(page, numberOfPages);
_state.NextPageNumber = transactionHeader->NextPageNumber;
}
}
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages)
{
var size = Utils.NearestPowerOfTwo(numberOfPages);
PageFromScratchBuffer result;
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
return result;
// we don't have free pages to give out, need to allocate some
_scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size);
result = new PageFromScratchBuffer
{
PositionInScratchBuffer = _lastUsedPage,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(_lastUsedPage, result);
_lastUsedPage += size;
return result;
}
private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result)
{
result = null;
LinkedList<PendingPage> list;
if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0)
return false;
var val = list.First.Value;
if (val.ValidAfterTransactionId >= tx.Environment.OldestTransaction)
return false;
list.RemoveFirst();
var pageFromScratchBuffer = new PageFromScratchBuffer
{
PositionInScratchBuffer = val.Page,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(val.Page, pageFromScratchBuffer);
{
result = pageFromScratchBuffer;
return true;
}
}
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages)
{
if (tx == null) throw new ArgumentNullException("tx");
var size = Utils.NearestPowerOfTwo(numberOfPages);
PageFromScratchBuffer result;
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
return result;
if ((_lastUsedPage + size)*AbstractPager.PageSize > _sizeLimit)
{
var sp = Stopwatch.StartNew();
// Our problem is that we don't have any available free pages, probably because
// there are read transactions that are holding things open. We are going to see if
// there are any free pages that _might_ be freed for us if we wait for a bit. The idea
// is that we let the read transactions time to complete and do their work, at which point
// we can continue running.
// We start this by forcing a flush, then we are waiting up to the timeout for we are waiting
// for the read transactions to complete. It is possible that a long running read transaction
// would in fact generate enough work for us to timeout, but hopefully we can avoid that.
tx.Environment.ForceLogFlushToDataFile(tx);
while (sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout)
{
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
return result;
Thread.Sleep(32);
}
string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" +
"Current size is:\t{0:#,#;;0} kb.\r\n" +
"Limit:\t\t\t{1:#,#;;0} kb.\r\n" +
"Requested Size:\t{2:#,#;;0} kb.\r\n" +
"Already flushed and waited for {3:#,#;;0} ms for read transactions to complete.\r\n" +
"Do you have a long running read transaction executing?",
(_scratchPager.NumberOfAllocatedPages*AbstractPager.PageSize)/1024,
_sizeLimit/1024,
((_lastUsedPage + size)*AbstractPager.PageSize)/1024,
sp.ElapsedMilliseconds);
throw new ScratchBufferSizeLimitException(message);
}
// we don't have free pages to give out, need to allocate some
_scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size);
result = new PageFromScratchBuffer
{
PositionInScratchBuffer = _lastUsedPage,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(_lastUsedPage, result);
_lastUsedPage += size;
return result;
}
private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result)
{
result = null;
LinkedList<PendingPage> list;
if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0)
return false;
var val = list.Last.Value;
var oldestTransaction = tx.Environment.OldestTransaction;
if (oldestTransaction != 0 && val.ValidAfterTransactionId >= oldestTransaction) // OldestTransaction can be 0 when there are none other transactions and we are in process of new transaction header allocation
return false;
list.RemoveLast();
var pageFromScratchBuffer = new PageFromScratchBuffer
{
PositionInScratchBuffer = val.Page,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(val.Page, pageFromScratchBuffer);
{
result = pageFromScratchBuffer;
return true;
}
}
internal void WriteDirect(TransactionHeader* transactionHeader, PageFromScratchBuffer pages)
{
for (int i = 0; i < pages.NumberOfPages; i++)
{
var page = _env.ScratchBufferPool.ReadPage(pages.PositionInScratchBuffer+i);
int numberOfPages = 1;
if (page.IsOverflow)
{
numberOfPages = (page.OverflowSize / AbstractPager.PageSize) + (page.OverflowSize % AbstractPager.PageSize == 0 ? 0 : 1);
i += numberOfPages;
_overflowPagesInTransaction += (numberOfPages - 1);
}
var pageFromScratchBuffer = _env.ScratchBufferPool.Allocate(this, numberOfPages);
var dest = _env.ScratchBufferPool.AcquirePagePointer(pageFromScratchBuffer.PositionInScratchBuffer);
NativeMethods.memcpy(dest, page.Base, numberOfPages*AbstractPager.PageSize);
_allocatedPagesInTransaction++;
_dirtyPages.Add(page.PageNumber);
page.Dirty = true;
if (numberOfPages > 1)
_dirtyOverflowPages.Add(page.PageNumber + 1, numberOfPages - 1);
_scratchPagesTable[page.PageNumber] = pageFromScratchBuffer;
_transactionPages.Add(pageFromScratchBuffer);
_state.NextPageNumber = transactionHeader->NextPageNumber;
}
}
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages)
{
if (tx == null)
{
throw new ArgumentNullException("tx");
}
var size = Utils.NearestPowerOfTwo(numberOfPages);
PageFromScratchBuffer result;
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
if ((_lastUsedPage + size) * AbstractPager.PageSize > _sizeLimit)
{
var sp = Stopwatch.StartNew();
// Our problem is that we don't have any available free pages, probably because
// there are read transactions that are holding things open. We are going to see if
// there are any free pages that _might_ be freed for us if we wait for a bit. The idea
// is that we let the read transactions time to complete and do their work, at which point
// we can continue running.
// We start this by forcing a flush, then we are waiting up to the timeout for we are waiting
// for the read transactions to complete. It is possible that a long running read transaction
// would in fact generate enough work for us to timeout, but hopefully we can avoid that.
tx.Environment.ForceLogFlushToDataFile(tx);
while (sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout)
{
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
{
return(result);
}
Thread.Sleep(32);
}
string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" +
"Current size is:\t{0:#,#;;0} kb.\r\n" +
"Limit:\t\t\t{1:#,#;;0} kb.\r\n" +
"Requested Size:\t{2:#,#;;0} kb.\r\n" +
"Already flushed and waited for {3:#,#;;0} ms for read transactions to complete.\r\n" +
"Do you have a long running read transaction executing?",
(_scratchPager.NumberOfAllocatedPages * AbstractPager.PageSize) / 1024,
_sizeLimit / 1024,
((_lastUsedPage + size) * AbstractPager.PageSize) / 1024,
sp.ElapsedMilliseconds);
throw new ScratchBufferSizeLimitException(message);
}
// we don't have free pages to give out, need to allocate some
_scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size);
result = new PageFromScratchBuffer
{
PositionInScratchBuffer = _lastUsedPage,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(_lastUsedPage, result);
_lastUsedPage += size;
return(result);
}
