private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result) { result = null; LinkedList <PendingPage> list; if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0) { return(false); } var val = list.First.Value; if (val.ValidAfterTransactionId >= tx.Environment.OldestTransaction) { return(false); } list.RemoveFirst(); var pageFromScratchBuffer = new PageFromScratchBuffer { PositionInScratchBuffer = val.Page, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(val.Page, pageFromScratchBuffer); { result = pageFromScratchBuffer; return(true); } }
private void InitTransactionHeader() { var allocation = _env.ScratchBufferPool.Allocate(this, 1); var page = _env.ScratchBufferPool.ReadPage(allocation.ScratchFileNumber, allocation.PositionInScratchBuffer); _transactionHeaderPage = allocation; StdLib.memset(page.Base, 0, AbstractPager.PageSize); _txHeader = (TransactionHeader *)page.Base; _txHeader->HeaderMarker = Constants.TransactionHeaderMarker; _txHeader->TransactionId = _id; _txHeader->NextPageNumber = _state.NextPageNumber; _txHeader->LastPageNumber = -1; _txHeader->PageCount = -1; _txHeader->Crc = 0; _txHeader->TxMarker = TransactionMarker.None; _txHeader->Compressed = false; _txHeader->CompressedSize = 0; _txHeader->UncompressedSize = 0; _allocatedPagesInTransaction = 0; _overflowPagesInTransaction = 0; _scratchPagesTable.Clear(); }
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages) { var size = Utils.NearestPowerOfTwo(numberOfPages); PageFromScratchBuffer result; if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) { return(result); } // we don't have free pages to give out, need to allocate some _scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size); result = new PageFromScratchBuffer { PositionInScratchBuffer = _lastUsedPage, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(_lastUsedPage, result); _lastUsedPage += size; return(result); }
private void InitTransactionHeader() { var allocation = _env.ScratchBufferPool.Allocate(this, 1); var page = _env.ScratchBufferPool.ReadPage(this, allocation.ScratchFileNumber, allocation.PositionInScratchBuffer); _transactionHeaderPage = allocation; UnmanagedMemory.Set(page.Pointer, 0, Environment.Options.PageSize); _txHeader = (TransactionHeader *)page.Pointer; _txHeader->HeaderMarker = Constants.TransactionHeaderMarker; _txHeader->TransactionId = _id; _txHeader->NextPageNumber = _state.NextPageNumber; _txHeader->LastPageNumber = -1; _txHeader->PageCount = -1; _txHeader->Hash = 0; _txHeader->TxMarker = TransactionMarker.None; _txHeader->Compressed = false; _txHeader->CompressedSize = 0; _txHeader->UncompressedSize = 0; _allocatedPagesInTransaction = 0; _overflowPagesInTransaction = 0; _scratchPagesTable.Clear(); }
internal void WriteDirect(TransactionHeader *transactionHeader, PageFromScratchBuffer pages) { for (int i = 0; i < pages.NumberOfPages; i++) { var page = _env.ScratchBufferPool.ReadPage(pages.ScratchFileNumber, pages.PositionInScratchBuffer + i); int numberOfPages = 1; if (page.IsOverflow) { numberOfPages = (page.OverflowSize / AbstractPager.PageSize) + (page.OverflowSize % AbstractPager.PageSize == 0 ? 0 : 1); i += numberOfPages; _overflowPagesInTransaction += (numberOfPages - 1); } var pageFromScratchBuffer = _env.ScratchBufferPool.Allocate(this, numberOfPages); var dest = _env.ScratchBufferPool.AcquirePagePointer(pageFromScratchBuffer.ScratchFileNumber, pageFromScratchBuffer.PositionInScratchBuffer); StdLib.memcpy(dest, page.Base, numberOfPages * AbstractPager.PageSize); _allocatedPagesInTransaction++; _dirtyPages.Add(page.PageNumber); page.Dirty = true; if (numberOfPages > 1) { _dirtyOverflowPages.Add(page.PageNumber + 1, numberOfPages - 1); } _scratchPagesTable[page.PageNumber] = pageFromScratchBuffer; _transactionPages.Add(pageFromScratchBuffer); _state.NextPageNumber = transactionHeader->NextPageNumber; } }
internal void BreakLargeAllocationToSeparatePages(long pageNumber) { if (_disposed != TxState.None) { ThrowObjectDisposed(); } PageFromScratchBuffer value; if (_scratchPagesTable.TryGetValue(pageNumber, out value) == false) { throw new InvalidOperationException("The page " + pageNumber + " was not previous allocated in this transaction"); } if (value.NumberOfPages == 1) { return; } _transactionPages.Remove(value); _env.ScratchBufferPool.BreakLargeAllocationToSeparatePages(this, value); for (int i = 0; i < value.NumberOfPages; i++) { var pageFromScratchBuffer = new PageFromScratchBuffer(value.ScratchFileNumber, value.PositionInScratchBuffer + i, 1, 1); _transactionPages.Add(pageFromScratchBuffer); _scratchPagesTable[pageNumber + i] = pageFromScratchBuffer; _dirtyPages.Add(pageNumber + i); TrackDirtyPage(pageNumber + i); var newPage = _env.ScratchBufferPool.ReadPage(this, value.ScratchFileNumber, value.PositionInScratchBuffer + i); newPage.PageNumber = pageNumber + i; newPage.Flags = PageFlags.Single; TrackWritablePage(newPage); } }
private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result) { result = null; LinkedList <PendingPage> list; if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0) { return(false); } var val = list.Last.Value; var oldestTransaction = tx.Environment.OldestTransaction; if (oldestTransaction != 0 && val.ValidAfterTransactionId >= oldestTransaction) // OldestTransaction can be 0 when there are none other transactions and we are in process of new transaction header allocation { return(false); } list.RemoveLast(); var pageFromScratchBuffer = new PageFromScratchBuffer { PositionInScratchBuffer = val.Page, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(val.Page, pageFromScratchBuffer); { result = pageFromScratchBuffer; return(true); } }
internal void WriteDirect(TransactionHeader *transactionHeader, PageFromScratchBuffer pages) { for (int i = 0; i < pages.NumberOfPages; i++) { var page = _env.ScratchBufferPool.ReadPage(pages.ScratchFileNumber, pages.PositionInScratchBuffer + i); int numberOfPages = 1; if (page.IsOverflow) { numberOfPages = (page.OverflowSize / AbstractPager.PageSize) + (page.OverflowSize % AbstractPager.PageSize == 0 ? 0 : 1); i += numberOfPages; _overflowPagesInTransaction += (numberOfPages - 1); } WritePageDirect(page, numberOfPages); _state.NextPageNumber = transactionHeader->NextPageNumber; } }
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages) { var size = Utils.NearestPowerOfTwo(numberOfPages); PageFromScratchBuffer result; if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) return result; // we don't have free pages to give out, need to allocate some _scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size); result = new PageFromScratchBuffer { PositionInScratchBuffer = _lastUsedPage, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(_lastUsedPage, result); _lastUsedPage += size; return result; }
private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result) { result = null; LinkedList<PendingPage> list; if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0) return false; var val = list.First.Value; if (val.ValidAfterTransactionId >= tx.Environment.OldestTransaction) return false; list.RemoveFirst(); var pageFromScratchBuffer = new PageFromScratchBuffer { PositionInScratchBuffer = val.Page, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(val.Page, pageFromScratchBuffer); { result = pageFromScratchBuffer; return true; } }
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages) { if (tx == null) throw new ArgumentNullException("tx"); var size = Utils.NearestPowerOfTwo(numberOfPages); PageFromScratchBuffer result; if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) return result; if ((_lastUsedPage + size)*AbstractPager.PageSize > _sizeLimit) { var sp = Stopwatch.StartNew(); // Our problem is that we don't have any available free pages, probably because // there are read transactions that are holding things open. We are going to see if // there are any free pages that _might_ be freed for us if we wait for a bit. The idea // is that we let the read transactions time to complete and do their work, at which point // we can continue running. // We start this by forcing a flush, then we are waiting up to the timeout for we are waiting // for the read transactions to complete. It is possible that a long running read transaction // would in fact generate enough work for us to timeout, but hopefully we can avoid that. tx.Environment.ForceLogFlushToDataFile(tx); while (sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout) { if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) return result; Thread.Sleep(32); } string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" + "Current size is:\t{0:#,#;;0} kb.\r\n" + "Limit:\t\t\t{1:#,#;;0} kb.\r\n" + "Requested Size:\t{2:#,#;;0} kb.\r\n" + "Already flushed and waited for {3:#,#;;0} ms for read transactions to complete.\r\n" + "Do you have a long running read transaction executing?", (_scratchPager.NumberOfAllocatedPages*AbstractPager.PageSize)/1024, _sizeLimit/1024, ((_lastUsedPage + size)*AbstractPager.PageSize)/1024, sp.ElapsedMilliseconds); throw new ScratchBufferSizeLimitException(message); } // we don't have free pages to give out, need to allocate some _scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size); result = new PageFromScratchBuffer { PositionInScratchBuffer = _lastUsedPage, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(_lastUsedPage, result); _lastUsedPage += size; return result; }
private bool TryGettingFromAllocatedBuffer(Transaction tx, int numberOfPages, long size, out PageFromScratchBuffer result) { result = null; LinkedList<PendingPage> list; if (!_freePagesBySize.TryGetValue(size, out list) || list.Count <= 0) return false; var val = list.Last.Value; var oldestTransaction = tx.Environment.OldestTransaction; if (oldestTransaction != 0 && val.ValidAfterTransactionId >= oldestTransaction) // OldestTransaction can be 0 when there are none other transactions and we are in process of new transaction header allocation return false; list.RemoveLast(); var pageFromScratchBuffer = new PageFromScratchBuffer { PositionInScratchBuffer = val.Page, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(val.Page, pageFromScratchBuffer); { result = pageFromScratchBuffer; return true; } }
internal void WriteDirect(TransactionHeader* transactionHeader, PageFromScratchBuffer pages) { for (int i = 0; i < pages.NumberOfPages; i++) { var page = _env.ScratchBufferPool.ReadPage(pages.PositionInScratchBuffer+i); int numberOfPages = 1; if (page.IsOverflow) { numberOfPages = (page.OverflowSize / AbstractPager.PageSize) + (page.OverflowSize % AbstractPager.PageSize == 0 ? 0 : 1); i += numberOfPages; _overflowPagesInTransaction += (numberOfPages - 1); } var pageFromScratchBuffer = _env.ScratchBufferPool.Allocate(this, numberOfPages); var dest = _env.ScratchBufferPool.AcquirePagePointer(pageFromScratchBuffer.PositionInScratchBuffer); NativeMethods.memcpy(dest, page.Base, numberOfPages*AbstractPager.PageSize); _allocatedPagesInTransaction++; _dirtyPages.Add(page.PageNumber); page.Dirty = true; if (numberOfPages > 1) _dirtyOverflowPages.Add(page.PageNumber + 1, numberOfPages - 1); _scratchPagesTable[page.PageNumber] = pageFromScratchBuffer; _transactionPages.Add(pageFromScratchBuffer); _state.NextPageNumber = transactionHeader->NextPageNumber; } }
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages) { if (tx == null) { throw new ArgumentNullException("tx"); } var size = Utils.NearestPowerOfTwo(numberOfPages); PageFromScratchBuffer result; if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) { return(result); } if ((_lastUsedPage + size) * AbstractPager.PageSize > _sizeLimit) { var sp = Stopwatch.StartNew(); // Our problem is that we don't have any available free pages, probably because // there are read transactions that are holding things open. We are going to see if // there are any free pages that _might_ be freed for us if we wait for a bit. The idea // is that we let the read transactions time to complete and do their work, at which point // we can continue running. // We start this by forcing a flush, then we are waiting up to the timeout for we are waiting // for the read transactions to complete. It is possible that a long running read transaction // would in fact generate enough work for us to timeout, but hopefully we can avoid that. tx.Environment.ForceLogFlushToDataFile(tx); while (sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout) { if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) { return(result); } Thread.Sleep(32); } string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" + "Current size is:\t{0:#,#;;0} kb.\r\n" + "Limit:\t\t\t{1:#,#;;0} kb.\r\n" + "Requested Size:\t{2:#,#;;0} kb.\r\n" + "Already flushed and waited for {3:#,#;;0} ms for read transactions to complete.\r\n" + "Do you have a long running read transaction executing?", (_scratchPager.NumberOfAllocatedPages * AbstractPager.PageSize) / 1024, _sizeLimit / 1024, ((_lastUsedPage + size) * AbstractPager.PageSize) / 1024, sp.ElapsedMilliseconds); throw new ScratchBufferSizeLimitException(message); } // we don't have free pages to give out, need to allocate some _scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size); result = new PageFromScratchBuffer { PositionInScratchBuffer = _lastUsedPage, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(_lastUsedPage, result); _lastUsedPage += size; return(result); }