private void Init() { IPage headingPage = _pageManager.FetchPage(0); var header = PageFormatter.GetPageHeader(headingPage) as HeadingPageHeader; if (header == null) { throw new StorageFormatException("Heading page not found"); } _firstFsmPageIndex = header.FsmPageIndex; _fsmPageIndexes.Add(_firstFsmPageIndex); IPage firstFsmPage = _pageManager.FetchPage(_firstFsmPageIndex); var fsmHeader = PageFormatter.GetPageHeader(firstFsmPage) as FreeSpaceMapPageHeader; if (fsmHeader == null) { throw new StorageFormatException("Free space map page not found"); } _entryPerPage = PageFormatter.GetFsmEntryCount(firstFsmPage); _isInitialized = true; }
private IBPlusTreeNode <TKey> PageToNode(IPage page) { var node = page.BackingObject as IBPlusTreeNode <TKey>; if (node != null) { return(node); } var header = (BPlusTreeNodePageHeader)PageFormatter.GetPageHeader(page); List <DbItem> items = PageFormatter.ReadFixedSizeItems(page); var result = new BPlusTreeNode <TKey>(page.Index, items.Count) { IsLeaf = header.IsLeaf, ParentNodeIndex = header.ParentPageIndex, PreviousNodeIndex = header.PreviousPageIndex, NextNodeIndex = header.NextPageIndex }; int cnt = items.Count; for (int i = 0; i < cnt; i++) { result.Entries.Add(IndexEntryFromBytes(items[i].RawData)); } page.BackingObject = result; if (_nodeEntrySizeRange != header.SizeRange) { throw new DataTankerException("Mismatch key size"); // TODO: specify possible size range } return(result); }
private byte[] Serialize(IBPlusTreeNode <TKey> node) { var header = new BPlusTreeNodePageHeader { IsLeaf = node.IsLeaf, NextPageIndex = node.NextNodeIndex, PreviousPageIndex = node.PreviousNodeIndex, ParentPageIndex = node.ParentNodeIndex, SizeRange = _nodeEntrySizeRange }; var page = new Page(_pageManager, node.Index, new byte[_pageManager.PageSize]); int cnt = node.Entries.Count; lock (_locker) { for (int i = 0; i < cnt; i++) { var entry = node.Entries[i]; _dbItems[i].RawData = GetIndexEntryBytes(entry); } PageFormatter.FormatFixedSizeItemsPage(page, header, _dbItems.Take(cnt).ToArray()); } return(page.Content); }
/// <summary> /// Instantiates a new <see cref="ArrayPageProvider{T}"/> wtih the given <see cref="System.Array"/> of data /// and optionally the <see cref="PageFormatter{T}"/> and a number of items per <see cref="Page"/>. /// </summary> /// <param name="array"> The <see cref="System.Array"/> of data. </param> /// <param name="formatter"> The <see cref="PageFormatter{T}"/>. </param> /// <param name="itemsPerPage"> The number of items per <see cref="Page"/>. </param> public ArrayPageProvider(T[] array, PageFormatter <ArraySegment <T> > formatter = null, int itemsPerPage = 10) { if (array == null) { throw new ArgumentNullException(nameof(array)); } if (itemsPerPage <= 0 || itemsPerPage > array.Length) { throw new ArgumentOutOfRangeException(nameof(itemsPerPage)); } Array = array; ItemsPerPage = itemsPerPage; Formatter = formatter ?? ((menu, segment) => new LocalEmbedBuilder() .WithDescription(string.Join('\n', segment.Select((x, i) => { var itemPrefix = $"{i + segment.Offset + 1}. "; var maxItemLength = (int)Math.Floor((double)LocalEmbedBuilder.MAX_DESCRIPTION_LENGTH / ItemsPerPage) - itemPrefix.Length - 2; if (maxItemLength <= 0) { throw new InvalidOperationException("There are too many items per-page. Set a lower amount or provide a custom page formatter."); } var item = x.ToString(); if (item.Length > maxItemLength) { item = $"{item[0..maxItemLength]}…"; }
/// <summary> /// Gets the segment of binary representation of db item. /// </summary> /// <param name="reference">Reference to the db item</param> /// <param name="startIndex">The start index in binary representation</param> /// <param name="endIndex">The end index in binary representation</param> /// <returns>The array of bytes containing specified segment of db item</returns> public byte[] GetItemSegment(DbItemReference reference, long startIndex, long endIndex) { if (startIndex < 0) { throw new ArgumentOutOfRangeException(nameof(startIndex)); } if (endIndex < 0) { throw new ArgumentOutOfRangeException(nameof(endIndex)); } if (endIndex < startIndex) { throw new ArgumentException("End index should be qreater than or equal to start index"); } if (!_pageManager.PageExists(reference.PageIndex)) { return new byte[] {} } ; IPage page = _pageManager.FetchPage(reference.PageIndex); var header = PageFormatter.GetPageHeader(page); if (header.SizeRange == SizeRange.MultiPage) { return(GetLargeItemSegment(header, page, startIndex, endIndex)); } return(GetFixedSizeItemSegment(page, reference, startIndex, endIndex)); }
public void TestExtractTextAndWash() { var pageNumber = 737; for (int i = pageNumber; i <= 970; i++) { var sourcePath = @"c:\code\PdfTranslator\Solution1\PageExtractor.Tests\bin\Debug\Extracted\"; var pdfFilename = "Page" + i + "_Iosephi_Scaligeri_Opus_de_emendatione_te.pdf"; var outFilename = "Page" + i.ToString("00#") + "_cleanedText_Iosephi_Scaligeri_Opus_de_emendatione_te.txt"; var outDirectory = @"c:\code\PdfTranslator\Solution1\Output\Latin\Raw\"; var te = new TextExtractor(); var textContents = te.Extract(Path.Combine(sourcePath, pdfFilename)); Assert.IsNotNull(textContents); var pe = new PageFormatter() { OriginalText = textContents.Text }; var correctedText = pe.FixCommonOCRErrors(); Assert.IsFalse(correctedText.Contains(" fed ")); File.WriteAllText(Path.Combine(outDirectory, outFilename), correctedText); } }
/// <summary> /// Releases db item by its reference /// </summary> /// <param name="reference">Reference to item to release</param> public void Free(DbItemReference reference) { IPage page = _pageManager.FetchPage(reference.PageIndex); var header = PageFormatter.GetPageHeader(page); if (header.SizeRange == SizeRange.MultiPage) { while (page != null) { _pageManager.RemovePage(page.Index); var nextPageIndex = ((MultipageItemPageHeader)PageFormatter.GetPageHeader(page)).NextPageIndex; page = nextPageIndex == -1 ? null : _pageManager.FetchPage(nextPageIndex); } } else { if (PageFormatter.ReadFixedSizeItemsCount(page) == 1) { _pageManager.RemovePage(page.Index); _fsm.Set(page.Index, FsmValue.Full); } else { PageFormatter.DeleteFixedSizeItem(page, reference.ItemIndex); _pageManager.UpdatePage(page); _fsm.Set(page.Index, EnumHelper.FsmValueFromSizeRange(header.SizeRange)); } } }
internal void assignToNew(string filename, PageFormatter fmtr) {//按照特定格式初始化buffer中的页,然后将这个页添加到指定文件中,若当前buffer为脏,先将页内容写回磁盘中 flush(); fmtr.format(contents); blk = contents.append(filename); pins = 0; }
public void InsertVariableSizeItemAfterDelete() { int pageSize = 4096; var dummyPageManager = new FileSystemPageManager(pageSize); var p = new Page(dummyPageManager, 0, new byte[pageSize]); var header = new RadixTreeNodesPageHeader { FreeSpace = (ushort)PageFormatter.GetMaximalFreeSpace((PageSize)p.Length) }; PageFormatter.InitPage(p, header); var r = new Random(); var items = new List <byte[]>(); var item = new byte[r.Next(100) + 1]; r.NextBytes(item); short itemIndex = 0; while (itemIndex != -1) { itemIndex = PageFormatter.AddVariableSizeItem(p, item); items.Add(item); item = new byte[r.Next(100) + 1]; r.NextBytes(item); } bool hasRemainingItems; // replace with the same PageFormatter.DeleteVariableSizeItem(p, 1, out hasRemainingItems); Assert.AreEqual(1, PageFormatter.AddVariableSizeItem(p, items[1])); item = PageFormatter.ReadVariableSizeItem(p, 1); Assert.IsTrue(AreEqualByteArrays(items[1], item)); // replace with smaller one PageFormatter.DeleteVariableSizeItem(p, 1, out hasRemainingItems); var smallItem = new byte[items[1].Length / 2 + 1]; r.NextBytes(smallItem); Assert.AreEqual(1, PageFormatter.AddVariableSizeItem(p, smallItem)); item = PageFormatter.ReadVariableSizeItem(p, 1); Assert.IsTrue(AreEqualByteArrays(smallItem, item)); // and put original again PageFormatter.DeleteVariableSizeItem(p, 1, out hasRemainingItems); PageFormatter.AddVariableSizeItem(p, items[1]); item = PageFormatter.ReadVariableSizeItem(p, 1); Assert.IsTrue(AreEqualByteArrays(items[1], item)); }
public Block pinNew(string filename, PageFormatter fmtr) {//将一个新的block添加到指定文件,先绑定文件,然后从文件中读一个块到缓冲区并绑定 Buffer buff = bufferMgr.pinNew(filename, fmtr); Block blk = buff.block(); buffers.Add(blk, buff); pins.Add(blk); return(blk); }
/// <summary> /// Opens an existing Storage. /// </summary> /// <param name="path">A string containing information about storage location</param> public void OpenExisting(string path) { CheckDisposed(); if (PageManager == null) { throw new InvalidOperationException("Page manager is not set"); } if (_isOpen) { throw new InvalidOperationException("Storage is already open"); } _path = path; ReadInfo(); CheckInfo(); _isOpen = true; PageManager.OpenExistingPageSpace(); IPage headingPage = PageManager.FetchPage(0); var header = PageFormatter.GetPageHeader(headingPage); var headingHeader = (HeadingPageHeader)header; if (headingHeader == null) { throw new StorageFormatException("Heading page not found"); } if (headingHeader.PageSize != PageSize) { var pageSize = PageSize; _isOpen = false; Close(); throw new StorageFormatException($"Page size: {pageSize} bytes is set. But pages of the opening storage is {headingHeader.PageSize} bytes length"); } if (headingHeader.OnDiskStructureVersion != OnDiskStructureVersion) { _isOpen = false; Close(); throw new NotSupportedException($"On-disk structure version {headingHeader.OnDiskStructureVersion} is not supported."); } if (headingHeader.AccessMethod != (short)AccessMethod) { _isOpen = false; Close(); throw new NotSupportedException($"Access method {headingHeader.AccessMethod} is not supported by this instance of storage."); } PageManager.EnterAtomicOperation(); }
public void GetAdditionalManualInfo(PageFormatter page) { foreach (TargetArchitecture architecture in Architectures) { foreach (TargetPlatform platform in Platforms) { page.AddRow(string.Format("-{0}", platform.GetTechnicalName(architecture)), string.Format("Enables {0} {1} platform handling", architecture, platform.Name)); } } }
/// <summary> /// Sets the node with the specified index as new root. /// </summary> /// <param name="nodeIndex">An index of new root node</param> public void SetRoot(long nodeIndex) { var headerPage = _pageManager.FetchPage(0); var header = (HeadingPageHeader)PageFormatter.GetPageHeader(headerPage); header.AccessMethodPageIndex = nodeIndex; header.WriteToPage(headerPage); _rootIndex = nodeIndex; _pageManager.UpdatePage(headerPage); }
public Block append(string filename, PageFormatter fmtr) {//向文件末尾添加一个新的块,返回这个块的引用,在“添加”操作之前先加 X 锁 Block dummyblk = new Block(filename, END_OF_FILE); concurMgr.xLock(dummyblk); Block blk = myBuffers.pinNew(filename, fmtr); unpin(blk); return(blk); }
/// <summary> /// Fetches a root node of the tree from storage. /// </summary> /// <returns>The root node of tree</returns> public IBPlusTreeNode <TKey> FetchRoot() { var headerPage = _pageManager.FetchPage(0); if (_rootIndex == null) { _rootIndex = ((HeadingPageHeader)PageFormatter.GetPageHeader(headerPage)).AccessMethodPageIndex; } return(Fetch(_rootIndex.Value)); }
public static RadixTreePageBackingObject FromPage(IPage page) { var result = new RadixTreePageBackingObject(page.Index); var items = PageFormatter.ReadVariableSizeItems(page); foreach (var item in items) { result.Items.Add(item); } return(result); }
public void PageHasNoRemainingItemsAfterDelete() { int pageSize = 4096; var dummyPageManager = new FileSystemPageManager(pageSize); var p = new Page(dummyPageManager, 0, new byte[pageSize]); var header = new RadixTreeNodesPageHeader { FreeSpace = (ushort)PageFormatter.GetMaximalFreeSpace((PageSize)p.Length) }; PageFormatter.InitPage(p, header); var r = new Random(); var items = new List <byte[]>(); var item = new byte[r.Next(100) + 1]; r.NextBytes(item); short itemIndex = 0; while (itemIndex != -1) { itemIndex = PageFormatter.AddVariableSizeItem(p, item); if (itemIndex == -1) { continue; } items.Add(item); item = new byte[r.Next(100) + 1]; r.NextBytes(item); } bool hasRemainingItems; for (short i = 0; i < items.Count; i++) { PageFormatter.DeleteVariableSizeItem(p, i, out hasRemainingItems); if (i < items.Count - 1) { Assert.IsTrue(hasRemainingItems); } else { Assert.IsFalse(hasRemainingItems); } } }
private void InitFsmPage(IPage page, long previousPageIndex, long nextPageIndex, long basePageIndex) { var header = new FreeSpaceMapPageHeader { StartPageIndex = _firstFsmPageIndex, NextPageIndex = nextPageIndex, PreviousPageIndex = previousPageIndex, BasePageIndex = basePageIndex }; PageFormatter.InitPage(page, header); PageFormatter.SetAllFsmValues(page, FsmValue.Full); }
public FsmValue Get(long pageIndex) { if (!_isInitialized) { Init(); } IPage page = GetFsmPageByTargetPageIndex(pageIndex); return(page == null ? FsmValue.Full : PageFormatter.GetFsmValue(page, (int)pageIndex % _entryPerPage)); }
public void FixedSizeItemsPage() { int pageSize = 32768; var dummyPageManager = new FileSystemPageManager(pageSize); var p = new Page(dummyPageManager, 0, new byte[pageSize]); var header = new FixedSizeItemsPageHeader(); var r = new Random(); foreach (var sizeRange in EnumHelper.FixedSizeItemsSizeRanges()) { header.SizeRange = sizeRange; PageFormatter.InitPage(p, header); var item = new DbItem(new byte[DbItem.GetMaxSize(header.SizeRange)]); r.NextBytes(item.RawData); // fill the page with the items short count = 0; bool spaceRemains = true; while (PageFormatter.HasFreeSpaceForFixedSizeItem(p)) { Assert.IsTrue(spaceRemains); PageFormatter.AddFixedSizeItem(p, item, out spaceRemains); count++; Assert.AreEqual(count, PageFormatter.ReadFixedSizeItemsCount(p)); } Assert.IsFalse(spaceRemains); // check if fetched objects are equal to originals for (short j = 0; j < PageFormatter.ReadFixedSizeItemsCount(p); j++) { DbItem readItem = PageFormatter.ReadFixedSizeItem(p, j); Assert.IsTrue(AreEqualByteArrays(item.RawData, readItem.RawData)); } // delete all added items short itemindex = 0; while (PageFormatter.ReadFixedSizeItemsCount(p) > 0) { PageFormatter.DeleteFixedSizeItem(p, itemindex); count--; itemindex++; Assert.AreEqual(count, PageFormatter.ReadFixedSizeItemsCount(p)); } } }
internal Buffer pinNew(string filename, PageFormatter fmtr) {//给指定文件分配一个新块,然后为其绑定一个buffer,若没有可用buffer返回空 lock (threadLock) { Buffer buff = chooseUnpinnedBuffer(); if (buff == null) { return(null); } buff.assignToNew(filename, fmtr); numAvailable--; buff.pin(); return(buff); } }
/// <summary> /// Instantiates a new <see cref="ArrayPageProvider{T}"/> with the specified array of items /// and optionally the <see cref="PageFormatter{T}"/> and a number of items per <see cref="Page"/>. /// </summary> /// <param name="array"> The array of items. </param> /// <param name="formatter"> The <see cref="PageFormatter{T}"/>. If <see langword="null"/>, defaults to <see cref="DefaultFormatter"/>. </param> /// <param name="itemsPerPage"> The number of items per-<see cref="Page"/>. </param> public ArrayPageProvider(T[] array, PageFormatter <ArraySegment <T> > formatter = null, int itemsPerPage = 10) { if (array == null) { throw new ArgumentNullException(nameof(array)); } if (itemsPerPage <= 0 || itemsPerPage > array.Length) { throw new ArgumentOutOfRangeException(nameof(itemsPerPage)); } Array = array; ItemsPerPage = itemsPerPage; Formatter = formatter ?? DefaultFormatter; }
private DbItemReference AllocateMultiPage(DbItem item) { long bytesWritten = 0; long startPageIndex = -1; DbItemReference result = null; IPage page = null; IPage previousPage = null; while (bytesWritten < item.RawData.LongLength) { page = _pageManager.CreatePage(); if (startPageIndex == -1) { startPageIndex = page.Index; result = new DbItemReference(page.Index, 0); } var header = new MultipageItemPageHeader { StartPageIndex = startPageIndex, PreviousPageIndex = previousPage?.Index ?? -1, NextPageIndex = -1, SizeRange = SizeRange.MultiPage }; PageFormatter.InitPage(page, header); bytesWritten += PageFormatter.WriteMultipageItemBlock(page, item, bytesWritten); if (previousPage != null) { header = (MultipageItemPageHeader)PageFormatter.GetPageHeader(previousPage); header.NextPageIndex = page.Index; header.WriteToPage(previousPage); _pageManager.UpdatePage(previousPage); } previousPage = page; } if (page != null) { _pageManager.UpdatePage(page); } return(result); }
public void OnEmptyStorageAllRequestedPagesShouldBeFull() { var manager = new FileSystemPageManager(4096); using (var storage = new Storage(manager)) { storage.CreateNew(StoragePath); var fsm = new FreeSpaceMap(manager); int entryCount = PageFormatter.GetFsmEntryCount(manager.FetchPage(1)); for (int i = 0; i < entryCount; i++) { Assert.AreEqual(FsmValue.Full, fsm.Get(i)); } } }
private byte[] GetLargeItemSegment(PageHeaderBase header, IPage page, long startIndex, long endIndex) { if (PageFormatter.ReadMultipageItemLength(page) <= endIndex) { throw new ArgumentOutOfRangeException(nameof(endIndex)); } var length = endIndex - startIndex + 1; var result = new byte[length]; long sourceOffset = 0; long destOffset = 0; // navigate through large item while (true) { var readBytes = PageFormatter.ReadMultipageItemBlock(page, Math.Min(_pageManager.PageSize, (int)(endIndex + 1 - sourceOffset))); sourceOffset += readBytes.Length; if (sourceOffset > startIndex) { var ssi = sourceOffset - startIndex - 1 < readBytes.Length ? startIndex + readBytes.Length - sourceOffset : 0; var l = readBytes.Length - Math.Max(0, sourceOffset - endIndex - 1) - ssi; Array.Copy(readBytes, ssi, result, destOffset, l); destOffset += l; if (sourceOffset >= endIndex) { return(result); } } var nextPageIndex = ((MultipageItemPageHeader)header).NextPageIndex; if (nextPageIndex != -1) { page = _pageManager.FetchPage(nextPageIndex); header = PageFormatter.GetPageHeader(page); } } }
public void ReadVariableSizeItems() { int pageSize = 4096; var dummyPageManager = new FileSystemPageManager(pageSize); var p = new Page(dummyPageManager, 0, new byte[pageSize]); var header = new RadixTreeNodesPageHeader { FreeSpace = (ushort)PageFormatter.GetMaximalFreeSpace((PageSize)p.Length) }; PageFormatter.InitPage(p, header); var r = new Random(); var items = new List <byte[]>(); var item = new byte[r.Next(100) + 1]; r.NextBytes(item); while (PageFormatter.AddVariableSizeItem(p, item) != -1) { items.Add(item); item = new byte[r.Next(100) + 1]; r.NextBytes(item); } for (short i = 0; i < items.Count; i++) { item = PageFormatter.ReadVariableSizeItem(p, i); Assert.IsTrue(AreEqualByteArrays(items[i], item)); } bool hasRemainingItems; PageFormatter.DeleteVariableSizeItem(p, 0, out hasRemainingItems); PageFormatter.DeleteVariableSizeItem(p, 2, out hasRemainingItems); PageFormatter.DeleteVariableSizeItem(p, (short)(items.Count - 1), out hasRemainingItems); Assert.Throws <PageFormatException>(() => PageFormatter.ReadVariableSizeItem(p, 0)); Assert.Throws <PageFormatException>(() => PageFormatter.ReadVariableSizeItem(p, 2)); Assert.Throws <PageFormatException>(() => PageFormatter.ReadVariableSizeItem(p, (short)(items.Count - 1))); }
protected override void Init() { base.Init(); //add access-method page IPage amPage = PageManager.CreatePage(); Debug.Assert(amPage.Index == 2, "The first access method page should have index 2"); var header = new RadixTreeNodesPageHeader { FreeSpace = (ushort)PageFormatter.GetMaximalFreeSpace((PageSize)PageManager.PageSize) }; PageFormatter.InitPage(amPage, header); PageManager.UpdatePage(amPage); }
public void MultiPageOperations() { var manager = new FileSystemPageManager(4096); using (var storage = new Storage(manager)) { storage.CreateNew(StoragePath); var fsm = new FreeSpaceMap(manager); int entryPerPage = PageFormatter.GetFsmEntryCount(manager.FetchPage(1)); int entryCount = entryPerPage * 3; // check writing to the last page var value = FsmValue.Class0; var index = entryCount - 1; fsm.Set(index, value); Assert.AreEqual(value, fsm.Get(index)); // middle page value = FsmValue.Class1; index = entryPerPage + 1; fsm.Set(index, value); Assert.AreEqual(value, fsm.Get(index)); // the first page value = FsmValue.Class2; index = entryPerPage - 1; fsm.Set(index, value); Assert.AreEqual(value, fsm.Get(index)); for (int i = 0; i < entryCount; i++) { fsm.Set(i, FsmValue.Class3); } for (int i = 0; i < entryCount; i++) { long pageIndex = fsm.GetFreePageIndex(FsmValue.Class3); Assert.AreNotEqual(-1, pageIndex); fsm.Set(i, FsmValue.Full); } } }
/// <summary> /// Gets DbItem instance by reference. /// </summary> /// <param name="reference">Reference to the requested db item</param> /// <returns></returns> public DbItem Get(DbItemReference reference) { if (!_pageManager.PageExists(reference.PageIndex)) { return(null); } IPage page = _pageManager.FetchPage(reference.PageIndex); var header = PageFormatter.GetPageHeader(page); if (header.SizeRange == SizeRange.MultiPage) { var length = PageFormatter.ReadMultipageItemLength(page); var content = new byte[length]; long offset = 0; while (page != null) { var readBytes = PageFormatter.ReadMultipageItemBlock(page, Math.Min(_pageManager.PageSize, (int)(length - offset))); readBytes.CopyTo(content, offset); offset += readBytes.Length; var nextPageIndex = ((MultipageItemPageHeader)header).NextPageIndex; if (nextPageIndex != -1) { page = _pageManager.FetchPage(nextPageIndex); header = PageFormatter.GetPageHeader(page); } else { page = null; } } return(new DbItem(content)); } return(PageFormatter.IsFixedSizeItemAllocated(page, reference.ItemIndex) ? PageFormatter.ReadFixedSizeItem(page, reference.ItemIndex) : null); }
public void GetAdditionalManualInfo(PageFormatter page) { List <UInt32> ids = new List <UInt32>(); foreach (ICppCompilerService service in CompilerServices.Services) { foreach (TargetArchitecture architecture in service.Architectures) { foreach (TargetPlatform platform in service.Platforms) { UInt32 id = platform.GetTechnicalName(architecture).Fnv32(); if (!ids.Contains(id)) { page.AddRow(string.Format("-{0}", platform.GetTechnicalName(architecture)), string.Format("Enables {0} {1} platform handling", architecture, platform.Name)); ids.Add(id); } } } } }
public Page(PageFormatter pageFormatter) { this.pageFormatter = pageFormatter; }