private IBPlusTreeNode <TKey> PageToNode(IPage page)
{
var node = page.BackingObject as IBPlusTreeNode <TKey>;
if (node != null)
{
return(node);
}
var header = (BPlusTreeNodePageHeader)PageFormatter.GetPageHeader(page);
List <DbItem> items = PageFormatter.ReadFixedSizeItems(page);
var result = new BPlusTreeNode <TKey>(page.Index, items.Count)
{
IsLeaf = header.IsLeaf,
ParentNodeIndex = header.ParentPageIndex,
PreviousNodeIndex = header.PreviousPageIndex,
NextNodeIndex = header.NextPageIndex
};
int cnt = items.Count;
for (int i = 0; i < cnt; i++)
{
result.Entries.Add(IndexEntryFromBytes(items[i].RawData));
}
page.BackingObject = result;
if (_nodeEntrySizeRange != header.SizeRange)
{
throw new DataTankerException("Mismatch key size"); // TODO: specify possible size range
}
return(result);
}
private void Init()
{
IPage headingPage = _pageManager.FetchPage(0);
var header = PageFormatter.GetPageHeader(headingPage) as HeadingPageHeader;
if (header == null)
{
throw new StorageFormatException("Heading page not found");
}
_firstFsmPageIndex = header.FsmPageIndex;
_fsmPageIndexes.Add(_firstFsmPageIndex);
IPage firstFsmPage = _pageManager.FetchPage(_firstFsmPageIndex);
var fsmHeader = PageFormatter.GetPageHeader(firstFsmPage) as FreeSpaceMapPageHeader;
if (fsmHeader == null)
{
throw new StorageFormatException("Free space map page not found");
}
_entryPerPage = PageFormatter.GetFsmEntryCount(firstFsmPage);
_isInitialized = true;
}
/// <summary>
/// Gets the segment of binary representation of db item.
/// </summary>
/// <param name="reference">Reference to the db item</param>
/// <param name="startIndex">The start index in binary representation</param>
/// <param name="endIndex">The end index in binary representation</param>
/// <returns>The array of bytes containing specified segment of db item</returns>
public byte[] GetItemSegment(DbItemReference reference, long startIndex, long endIndex)
{
if (startIndex < 0)
{
throw new ArgumentOutOfRangeException(nameof(startIndex));
}
if (endIndex < 0)
{
throw new ArgumentOutOfRangeException(nameof(endIndex));
}
if (endIndex < startIndex)
{
throw new ArgumentException("End index should be qreater than or equal to start index");
}
if (!_pageManager.PageExists(reference.PageIndex))
{
return new byte[] {}
}
;
IPage page = _pageManager.FetchPage(reference.PageIndex);
var header = PageFormatter.GetPageHeader(page);
if (header.SizeRange == SizeRange.MultiPage)
{
return(GetLargeItemSegment(header, page, startIndex, endIndex));
}
return(GetFixedSizeItemSegment(page, reference, startIndex, endIndex));
}
/// <summary>
/// Releases db item by its reference
/// </summary>
/// <param name="reference">Reference to item to release</param>
public void Free(DbItemReference reference)
{
IPage page = _pageManager.FetchPage(reference.PageIndex);
var header = PageFormatter.GetPageHeader(page);
if (header.SizeRange == SizeRange.MultiPage)
{
while (page != null)
{
_pageManager.RemovePage(page.Index);
var nextPageIndex = ((MultipageItemPageHeader)PageFormatter.GetPageHeader(page)).NextPageIndex;
page = nextPageIndex == -1
? null
: _pageManager.FetchPage(nextPageIndex);
}
}
else
{
if (PageFormatter.ReadFixedSizeItemsCount(page) == 1)
{
_pageManager.RemovePage(page.Index);
_fsm.Set(page.Index, FsmValue.Full);
}
else
{
PageFormatter.DeleteFixedSizeItem(page, reference.ItemIndex);
_pageManager.UpdatePage(page);
_fsm.Set(page.Index, EnumHelper.FsmValueFromSizeRange(header.SizeRange));
}
}
}
/// <summary>
/// Opens an existing Storage.
/// </summary>
/// <param name="path">A string containing information about storage location</param>
public void OpenExisting(string path)
{
CheckDisposed();
if (PageManager == null)
{
throw new InvalidOperationException("Page manager is not set");
}
if (_isOpen)
{
throw new InvalidOperationException("Storage is already open");
}
_path = path;
ReadInfo();
CheckInfo();
_isOpen = true;
PageManager.OpenExistingPageSpace();
IPage headingPage = PageManager.FetchPage(0);
var header = PageFormatter.GetPageHeader(headingPage);
var headingHeader = (HeadingPageHeader)header;
if (headingHeader == null)
{
throw new StorageFormatException("Heading page not found");
}
if (headingHeader.PageSize != PageSize)
{
var pageSize = PageSize;
_isOpen = false;
Close();
throw new StorageFormatException($"Page size: {pageSize} bytes is set. But pages of the opening storage is {headingHeader.PageSize} bytes length");
}
if (headingHeader.OnDiskStructureVersion != OnDiskStructureVersion)
{
_isOpen = false;
Close();
throw new NotSupportedException($"On-disk structure version {headingHeader.OnDiskStructureVersion} is not supported.");
}
if (headingHeader.AccessMethod != (short)AccessMethod)
{
_isOpen = false;
Close();
throw new NotSupportedException($"Access method {headingHeader.AccessMethod} is not supported by this instance of storage.");
}
PageManager.EnterAtomicOperation();
}
/// <summary>
/// Sets the node with the specified index as new root.
/// </summary>
/// <param name="nodeIndex">An index of new root node</param>
public void SetRoot(long nodeIndex)
{
var headerPage = _pageManager.FetchPage(0);
var header = (HeadingPageHeader)PageFormatter.GetPageHeader(headerPage);
header.AccessMethodPageIndex = nodeIndex;
header.WriteToPage(headerPage);
_rootIndex = nodeIndex;
_pageManager.UpdatePage(headerPage);
}
/// <summary>
/// Fetches a root node of the tree from storage.
/// </summary>
/// <returns>The root node of tree</returns>
public IBPlusTreeNode <TKey> FetchRoot()
{
var headerPage = _pageManager.FetchPage(0);
if (_rootIndex == null)
{
_rootIndex = ((HeadingPageHeader)PageFormatter.GetPageHeader(headerPage)).AccessMethodPageIndex;
}
return(Fetch(_rootIndex.Value));
}
private DbItemReference AllocateMultiPage(DbItem item)
{
long bytesWritten = 0;
long startPageIndex = -1;
DbItemReference result = null;
IPage page = null;
IPage previousPage = null;
while (bytesWritten < item.RawData.LongLength)
{
page = _pageManager.CreatePage();
if (startPageIndex == -1)
{
startPageIndex = page.Index;
result = new DbItemReference(page.Index, 0);
}
var header = new MultipageItemPageHeader
{
StartPageIndex = startPageIndex,
PreviousPageIndex = previousPage?.Index ?? -1,
NextPageIndex = -1,
SizeRange = SizeRange.MultiPage
};
PageFormatter.InitPage(page, header);
bytesWritten += PageFormatter.WriteMultipageItemBlock(page, item, bytesWritten);
if (previousPage != null)
{
header = (MultipageItemPageHeader)PageFormatter.GetPageHeader(previousPage);
header.NextPageIndex = page.Index;
header.WriteToPage(previousPage);
_pageManager.UpdatePage(previousPage);
}
previousPage = page;
}
if (page != null)
{
_pageManager.UpdatePage(page);
}
return(result);
}
private byte[] GetLargeItemSegment(PageHeaderBase header, IPage page, long startIndex, long endIndex)
{
if (PageFormatter.ReadMultipageItemLength(page) <= endIndex)
{
throw new ArgumentOutOfRangeException(nameof(endIndex));
}
var length = endIndex - startIndex + 1;
var result = new byte[length];
long sourceOffset = 0;
long destOffset = 0;
// navigate through large item
while (true)
{
var readBytes = PageFormatter.ReadMultipageItemBlock(page, Math.Min(_pageManager.PageSize, (int)(endIndex + 1 - sourceOffset)));
sourceOffset += readBytes.Length;
if (sourceOffset > startIndex)
{
var ssi = sourceOffset - startIndex - 1 < readBytes.Length
? startIndex + readBytes.Length - sourceOffset
: 0;
var l = readBytes.Length - Math.Max(0, sourceOffset - endIndex - 1) - ssi;
Array.Copy(readBytes, ssi, result, destOffset, l);
destOffset += l;
if (sourceOffset >= endIndex)
{
return(result);
}
}
var nextPageIndex = ((MultipageItemPageHeader)header).NextPageIndex;
if (nextPageIndex != -1)
{
page = _pageManager.FetchPage(nextPageIndex);
header = PageFormatter.GetPageHeader(page);
}
}
}
/// <summary>
/// Gets DbItem instance by reference.
/// </summary>
/// <param name="reference">Reference to the requested db item</param>
/// <returns></returns>
public DbItem Get(DbItemReference reference)
{
if (!_pageManager.PageExists(reference.PageIndex))
{
return(null);
}
IPage page = _pageManager.FetchPage(reference.PageIndex);
var header = PageFormatter.GetPageHeader(page);
if (header.SizeRange == SizeRange.MultiPage)
{
var length = PageFormatter.ReadMultipageItemLength(page);
var content = new byte[length];
long offset = 0;
while (page != null)
{
var readBytes = PageFormatter.ReadMultipageItemBlock(page, Math.Min(_pageManager.PageSize, (int)(length - offset)));
readBytes.CopyTo(content, offset);
offset += readBytes.Length;
var nextPageIndex = ((MultipageItemPageHeader)header).NextPageIndex;
if (nextPageIndex != -1)
{
page = _pageManager.FetchPage(nextPageIndex);
header = PageFormatter.GetPageHeader(page);
}
else
{
page = null;
}
}
return(new DbItem(content));
}
return(PageFormatter.IsFixedSizeItemAllocated(page, reference.ItemIndex)
? PageFormatter.ReadFixedSizeItem(page, reference.ItemIndex)
: null);
}
/// <summary>
/// Gets the db item length (in bytes).
/// </summary>
/// <param name="reference">Reference to the requested db item</param>
/// <returns>The length of requested db item</returns>
public long GetLength(DbItemReference reference)
{
if (!_pageManager.PageExists(reference.PageIndex))
{
return(0);
}
IPage page = _pageManager.FetchPage(reference.PageIndex);
var header = PageFormatter.GetPageHeader(page);
if (header.SizeRange == SizeRange.MultiPage)
{
return(PageFormatter.ReadMultipageItemLength(page));
}
return(PageFormatter.IsFixedSizeItemAllocated(page, reference.ItemIndex)
? PageFormatter.ReadFixedSizeItemLength(page, reference.ItemIndex)
: 0);
}
private IPage GetFsmPageByTargetPageIndex(long pageIndex)
{
var fsmPageNumber = (int)pageIndex / _entryPerPage;
// try to find appropriate fsm-page in the index list
if (_fsmPageIndexes.Count >= fsmPageNumber + 1)
{
return(_pageManager.FetchPage(_fsmPageIndexes[fsmPageNumber]));
}
// iterate fsm sequence from the last found fsm-page
_fsmPageCount = _fsmPageIndexes.Count;
IPage fsmPage = _pageManager.FetchPage(_fsmPageIndexes.Last());
while (true)
{
var header = PageFormatter.GetPageHeader(fsmPage) as FreeSpaceMapPageHeader;
if (header == null)
{
throw new StorageFormatException("Free space map page not found");
}
_lastFsmPage = fsmPage;
_fsmPageCount++;
if (header.NextPageIndex == -1)
{
return(null);
}
fsmPage = _pageManager.FetchPage(header.NextPageIndex);
if (_fsmPageCount < fsmPageNumber)
{
return(fsmPage);
}
_fsmPageIndexes.Add(fsmPage.Index);
}
}
/// <summary>
/// Reallocates already allocated db item with specified content and produces reference to it.
/// </summary>
/// <param name="reference">Reference to already allocated item</param>
/// <param name="newContent">Content of item to reallocate</param>
/// <returns>Reference to the reallocated item</returns>
public DbItemReference Reallocate(DbItemReference reference, byte[] newContent)
{
var item = new DbItem(newContent);
if (item.GetAllocationType(_pageManager.PageSize) == AllocationType.SinglePage)
{
IPage page = _pageManager.FetchPage(reference.PageIndex);
if (PageFormatter.GetPageHeader(page).SizeRange == item.SizeRange)
{
PageFormatter.RewriteFixedSizeItem(page, reference.ItemIndex, item);
_pageManager.UpdatePage(page);
return(reference);
}
Free(reference);
return(AllocateSinglePage(item));
}
Free(reference);
return(AllocateMultiPage(item));
}
private void CheckRoot()
{
if (_rootNodeReference == null)
{
var headerPage = _pageManager.FetchPage(0);
_rootPageIndex = ((HeadingPageHeader)PageFormatter.GetPageHeader(headerPage)).AccessMethodPageIndex;
var page = _pageManager.FetchPage(_rootPageIndex);
var items = PageFormatter.ReadVariableSizeItems(page);
if (items.Any())
{
// root page already has an item
_rootNodeReference = new DbItemReference(_rootPageIndex, 0);
}
else
{
var node = new RadixTreeNode(256);
PageFormatter.AddVariableSizeItem(page, GetNodeBytes(node, GetNodeSize(0, 256)));
_rootNodeReference = new DbItemReference(_rootPageIndex, 0);
_pageManager.UpdatePage(page);
}
}
}
public void CorrectWriteAndReadAllHeaders()
{
int pageSize = 4096;
var dummyPageManager = new FileSystemPageManager(pageSize);
var p = new Page(dummyPageManager, 0, new byte[pageSize]);
var hph = new HeadingPageHeader();
PageFormatter.InitPage(p, hph);
PageHeaderBase phb = PageFormatter.GetPageHeader(p);
Assert.IsInstanceOf <HeadingPageHeader>(phb);
Assert.AreEqual(hph.Length, phb.Length);
Assert.AreEqual(hph.PageType, phb.PageType);
Assert.AreEqual(hph.SizeRange, phb.SizeRange);
var fsiph = new FixedSizeItemsPageHeader();
PageFormatter.InitPage(p, fsiph);
phb = PageFormatter.GetPageHeader(p);
Assert.IsInstanceOf <FixedSizeItemsPageHeader>(phb);
Assert.AreEqual(fsiph.Length, phb.Length);
Assert.AreEqual(fsiph.PageType, phb.PageType);
Assert.AreEqual(fsiph.SizeRange, phb.SizeRange);
var mpph = new MultipageItemPageHeader();
PageFormatter.InitPage(p, mpph);
phb = PageFormatter.GetPageHeader(p);
Assert.IsInstanceOf <MultipageItemPageHeader>(phb);
Assert.AreEqual(mpph.Length, phb.Length);
Assert.AreEqual(mpph.PageType, phb.PageType);
Assert.AreEqual(mpph.SizeRange, phb.SizeRange);
Assert.AreEqual(mpph.StartPageIndex, ((MultipageItemPageHeader)phb).StartPageIndex);
Assert.AreEqual(mpph.PreviousPageIndex, ((MultipageItemPageHeader)phb).PreviousPageIndex);
Assert.AreEqual(mpph.NextPageIndex, ((MultipageItemPageHeader)phb).NextPageIndex);
var fsmph = new FreeSpaceMapPageHeader();
PageFormatter.InitPage(p, fsmph);
phb = PageFormatter.GetPageHeader(p);
Assert.IsInstanceOf <FreeSpaceMapPageHeader>(phb);
Assert.AreEqual(fsmph.Length, phb.Length);
Assert.AreEqual(fsmph.PageType, phb.PageType);
Assert.AreEqual(fsmph.SizeRange, phb.SizeRange);
Assert.AreEqual(fsmph.BasePageIndex, ((FreeSpaceMapPageHeader)phb).BasePageIndex);
var tnph = new BPlusTreeNodePageHeader();
PageFormatter.InitPage(p, tnph);
phb = PageFormatter.GetPageHeader(p);
Assert.IsInstanceOf <BPlusTreeNodePageHeader>(phb);
Assert.AreEqual(tnph.Length, phb.Length);
Assert.AreEqual(tnph.PageType, phb.PageType);
Assert.AreEqual(tnph.SizeRange, phb.SizeRange);
Assert.AreEqual(tnph.ParentPageIndex, ((BPlusTreeNodePageHeader)phb).ParentPageIndex);
Assert.AreEqual(tnph.PreviousPageIndex, ((BPlusTreeNodePageHeader)phb).PreviousPageIndex);
Assert.AreEqual(tnph.NextPageIndex, ((BPlusTreeNodePageHeader)phb).NextPageIndex);
var rtnph = new RadixTreeNodesPageHeader();
PageFormatter.InitPage(p, rtnph);
phb = PageFormatter.GetPageHeader(p);
Assert.IsInstanceOf <RadixTreeNodesPageHeader>(phb);
Assert.AreEqual(rtnph.Length, phb.Length);
Assert.AreEqual(rtnph.PageType, phb.PageType);
Assert.AreEqual(rtnph.SizeRange, phb.SizeRange);
Assert.AreEqual(rtnph.FreeSpace, ((RadixTreeNodesPageHeader)phb).FreeSpace);
}
public long GetFreePageIndex(FsmValue value)
{
if (!_isInitialized)
{
Init();
}
// try to find in lucky pages
if (_luckyFsmPages.ContainsKey(value))
{
IPage page = _luckyFsmPages[value].Page;
var requestedFsmIndex = (int)(_luckyFsmPages[value].LastGoodIndex % _entryPerPage);
int matchingFsmIndex;
if (PageFormatter.GetFsmValue(page, requestedFsmIndex) == value)
{
matchingFsmIndex = requestedFsmIndex;
}
else
{
matchingFsmIndex = PageFormatter.GetIndexOfFirstMatchingFsmValue(page, value);
_luckyFsmPages[value].LastGoodIndex = matchingFsmIndex;
}
if (matchingFsmIndex == -1)
{
// page becomes unlucky
_luckyFsmPages.Remove(value);
}
else
{
return(PageFormatter.GetBasePageIndex(page) + matchingFsmIndex);
}
}
var currentPageIndex = _firstFsmPageIndex;
long index = 0;
if (_scanned.Contains(value))
{
return(-1);
}
while (true)
{
// sequential scan all fsm pages for specified fsm-value
var currentPage = _pageManager.FetchPage(currentPageIndex);
int matchingFsmIndex = PageFormatter.GetIndexOfFirstMatchingFsmValue(currentPage, value);
if (matchingFsmIndex == -1)
{
index += _entryPerPage;
var header = (FreeSpaceMapPageHeader)PageFormatter.GetPageHeader(currentPage);
if (header.NextPageIndex == -1)
{
_scanned.Add(value);
return(-1);
}
currentPageIndex = header.NextPageIndex;
}
else
{
// make found page lucky
_luckyFsmPages[value] = new LuckyPage {
Page = currentPage, LastGoodIndex = matchingFsmIndex + index
};
return(index + matchingFsmIndex);
}
}
}
public void Set(long pageIndex, FsmValue value)
{
if (!_isInitialized)
{
Init();
}
IPage page = GetFsmPageByTargetPageIndex(pageIndex);
if (page == null)
{
// fsm-page is missing for requested page index
long previousPageIndex = _lastFsmPage.Index;
var missingPageCount = (int)((pageIndex - (_fsmPageCount - _fsmPageIndexes.Count) * _entryPerPage) / _entryPerPage + 1);
var pages = new List <IPage>(missingPageCount);
// allocate new pages
for (int i = 0; i < missingPageCount; i++)
{
pages.Add(_pageManager.CreatePage());
}
var baseIndex = PageFormatter.GetBasePageIndex(_lastFsmPage);
// initialize them
for (int i = 0; i < missingPageCount; i++)
{
baseIndex += _entryPerPage;
InitFsmPage(pages[i],
previousPageIndex,
i == missingPageCount - 1 ? -1 : pages[i + 1].Index,
baseIndex);
previousPageIndex = pages[i].Index;
}
// and update
pages.ForEach(_pageManager.UpdatePage);
// save reference to added pages
var lastPageHeader = (FreeSpaceMapPageHeader)PageFormatter.GetPageHeader(_lastFsmPage);
lastPageHeader.NextPageIndex = pages[0].Index;
lastPageHeader.WriteToPage(_lastFsmPage);
_pageManager.UpdatePage(_lastFsmPage);
_lastFsmPage = null;
_fsmPageCount = -1;
Set(pageIndex, value);
}
else
{
PageFormatter.SetFsmValue(page, (int)pageIndex % _entryPerPage, value);
_pageManager.UpdatePage(page);
var fsmValuesToUpdate = _luckyFsmPages.Where(item => item.Value.Page.Index == page.Index).Select(item => item.Key).ToList();
foreach (var fsmValue in fsmValuesToUpdate)
{
_luckyFsmPages[fsmValue].Page = page;
}
if (!_luckyFsmPages.ContainsKey(value))
{
_luckyFsmPages[value] = new LuckyPage {
Page = page, LastGoodIndex = pageIndex
}
}
;
else
{
if (_scanned.Contains(value))
{
_scanned.Remove(value);
}
}
}
}
