public GetReadOnlyPage ( long pageNumber ) : Voron.Trees.Page | ||
pageNumber | long | |
return | Voron.Trees.Page |
public static void DumpHumanReadable(Transaction tx, long startPageNumber,string filenamePrefix = null) { if (Debugger.IsAttached == false) return; var path = Path.Combine(Environment.CurrentDirectory, String.Format("{0}tree.hdump", filenamePrefix ?? String.Empty)); TreeDumper.DumpHumanReadable(tx, path, tx.GetReadOnlyPage(startPageNumber)); }
public static void DumpHumanReadable(Transaction tx, string path, Page start) { using (var writer = File.CreateText(path)) { var stack = new Stack<Page>(); stack.Push(start); writer.WriteLine("Root page #{0}",start.PageNumber); while (stack.Count > 0) { var currentPage = stack.Pop(); if (currentPage.IsLeaf) { writer.WriteLine(); writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Leaf), Used: {3} : {4}", currentPage.PageNumber,currentPage.NumberOfEntries,currentPage.Flags, currentPage.SizeUsed, currentPage.CalcSizeUsed()); if(currentPage.NumberOfEntries <= 0) writer.WriteLine("Empty page (tree corrupted?)"); for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries;nodeIndex++) { var node = currentPage.GetNode(nodeIndex); var key = currentPage.GetNodeKey(node); writer.WriteLine("Node #{0}, Flags = {1}, {4} = {2}, Key = {3}, Entry Size: {5}", nodeIndex, node->Flags, node->DataSize, MaxString(key.ToString(), 25), node->Flags == NodeFlags.Data ? "Size" : "Page", SizeOf.NodeEntry(node)); } writer.WriteLine(); } else if(currentPage.IsBranch) { writer.WriteLine(); writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Branch), Used: {3} : {4}", currentPage.PageNumber, currentPage.NumberOfEntries, currentPage.Flags, currentPage.SizeUsed, currentPage.SizeUsed); var key = new Slice(SliceOptions.Key); for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); writer.WriteLine("Node #{2}, {0} / to page #{1}, Entry Size: {3}", GetBranchNodeString(nodeIndex, key, currentPage, node), node->PageNumber, nodeIndex, SizeOf.NodeEntry(node)); } for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber) { writer.Write("Found invalid reference to page #{0}", currentPage.PageNumber); stack.Clear(); break; } var child = tx.GetReadOnlyPage(node->PageNumber); stack.Push(child); } writer.WriteLine(); } } } }
public static int GetDataSize(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); return overFlowPage.OverflowSize; } return node->DataSize; }
public unsafe static ValueReader Reader(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize); } return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize); }
public static byte* DirectAccess(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); return overFlowPage.Base + Constants.PageHeaderSize; } return (byte*) node + node->KeySize + Constants.NodeHeaderSize; }
public static void CopyTo(Transaction tx, NodeHeader* node, byte* dest) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); Memory.Copy(dest, overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize); } Memory.Copy(dest, (byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize); }
public static unsafe bool HasDuplicateBranchReferences(Transaction tx, Page start,out long pageNumberWithDuplicates) { var stack = new Stack<Page>(); var existingTreeReferences = new ConcurrentDictionary<long, List<long>>(); stack.Push(start); while (stack.Count > 0) { var currentPage = stack.Pop(); if (currentPage.IsBranch) { for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); existingTreeReferences.AddOrUpdate(currentPage.PageNumber, new List<long> { node->PageNumber }, (branchPageNumber, pageNumberReferences) => { pageNumberReferences.Add(node->PageNumber); return pageNumberReferences; }); } for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber) { throw new InvalidDataException("found invalid reference on branch - tree is corrupted"); } var child = tx.GetReadOnlyPage(node->PageNumber); stack.Push(child); } } } Func<long, HashSet<long>> relevantPageReferences = branchPageNumber => new HashSet<long>(existingTreeReferences .Where(kvp => kvp.Key != branchPageNumber) .SelectMany(kvp => kvp.Value)); // ReSharper disable once LoopCanBeConvertedToQuery foreach (var branchReferences in existingTreeReferences) { if ( branchReferences.Value.Any( referencePageNumber => relevantPageReferences(branchReferences.Key).Contains(referencePageNumber))) { pageNumberWithDuplicates = branchReferences.Key; return true; } } pageNumberWithDuplicates = -1; return false; }
public static Slice GetData(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); if (overFlowPage.OverflowSize > ushort.MaxValue) throw new InvalidOperationException("Cannot convert big data to a slice, too big"); return new Slice(overFlowPage.Base + Constants.PageHeaderSize, (ushort)overFlowPage.OverflowSize); } return new Slice((byte*)node + node->KeySize + Constants.NodeHeaderSize, (ushort) node->DataSize); }
public static ValueReader Reader(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); Debug.Assert(overFlowPage.IsOverflow, "Requested oveflow page but got " + overFlowPage.Flags); Debug.Assert(overFlowPage.OverflowSize > 0, "Overflow page cannot be size equal 0 bytes"); return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize); } return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize); }
public void DebugValidateTree(Transaction tx, long rootPageNumber) { var pages = new HashSet<long>(); var stack = new Stack<Page>(); var root = tx.GetReadOnlyPage(rootPageNumber); stack.Push(root); pages.Add(rootPageNumber); while (stack.Count > 0) { var p = stack.Pop(); if (p.NumberOfEntries == 0 && p != root) { DebugStuff.RenderAndShow(tx, rootPageNumber, 1); throw new InvalidOperationException("The page " + p.PageNumber + " is empty"); } p.DebugValidate(tx, _cmp, rootPageNumber); if (p.IsBranch == false) continue; for (int i = 0; i < p.NumberOfEntries; i++) { var page = p.GetNode(i)->PageNumber; if (pages.Add(page) == false) { DebugStuff.RenderAndShow(tx, rootPageNumber, 1); throw new InvalidOperationException("The page " + page + " already appeared in the tree!"); } stack.Push(tx.GetReadOnlyPage(page)); } } }
private bool TryOverwriteOverflowPages(Transaction tx, TreeMutableState treeState, NodeHeader* updatedNode, Slice key, int len, ushort? version, out byte* pos) { if (updatedNode->Flags == NodeFlags.PageRef && tx.Id <= tx.Environment.OldestTransaction) // ensure MVCC - do not overwrite if there is some older active transaction that might read those overflows { var overflowPage = tx.GetReadOnlyPage(updatedNode->PageNumber); if (len <= overflowPage.OverflowSize) { CheckConcurrency(key, version, updatedNode->Version, TreeActionType.Add); if (updatedNode->Version == ushort.MaxValue) updatedNode->Version = 0; updatedNode->Version++; var availableOverflows = tx.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize); var requestedOverflows = tx.DataPager.GetNumberOfOverflowPages(len); var overflowsToFree = availableOverflows - requestedOverflows; for (int i = 0; i < overflowsToFree; i++) { tx.FreePage(overflowPage.PageNumber + requestedOverflows + i); } treeState.OverflowPages -= overflowsToFree; treeState.PageCount -= overflowsToFree; overflowPage.OverflowSize = len; pos = overflowPage.Base + Constants.PageHeaderSize; return true; } } pos = null; return false; }
public List<long> AllPages(Transaction tx) { var results = new List<long>(); var stack = new Stack<Page>(); var root = tx.GetReadOnlyPage(State.RootPageNumber); stack.Push(root); while (stack.Count > 0) { var p = stack.Pop(); results.Add(p.PageNumber); for (int i = 0; i < p.NumberOfEntries; i++) { var node = p.GetNode(i); var pageNumber = node->PageNumber; if (p.IsBranch) { stack.Push(tx.GetReadOnlyPage(pageNumber)); } else if (node->Flags == NodeFlags.PageRef) { // This is an overflow page var overflowPage = tx.GetReadOnlyPage(pageNumber); var numberOfPages = tx.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize); for (long j = 0; j < numberOfPages; ++j) results.Add(overflowPage.PageNumber + j); } else if (node->Flags == NodeFlags.MultiValuePageRef) { var childTreeHeader = (TreeRootHeader*)((byte*)node + node->KeySize + Constants.NodeHeaderSize); results.Add(childTreeHeader->RootPageNumber); // this is a multi value var tree = OpenOrCreateMultiValueTree(tx, new Slice(node), node); results.AddRange(tree.AllPages(tx)); } } } return results; }
internal byte* DirectRead(Transaction tx, Slice key) { Lazy<Cursor> lazy; var p = FindPageFor(tx, key, out lazy); var node = p.Search(key, _cmp); if (node == null) return null; var item1 = new Slice(node); if (item1.Compare(key, _cmp) != 0) return null; if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); return overFlowPage.Base + Constants.PageHeaderSize; } return (byte*) node + node->KeySize + Constants.NodeHeaderSize; }
private bool TryUseRecentTransactionPage(Transaction tx, Slice key, out Lazy<Cursor> cursor, out Page page) { page = null; cursor = null; var recentPages = tx.GetRecentlyFoundPages(this); if (recentPages == null) return false; var foundPage = recentPages.Find(key); if (foundPage == null) return false; var lastFoundPageNumber = foundPage.Number; page = tx.GetReadOnlyPage(lastFoundPageNumber); if (page.IsLeaf == false) throw new DataException("Index points to a non leaf page"); page.NodePositionFor(key, _cmp); // will set the LastSearchPosition var cursorPath = foundPage.CursorPath; var pageCopy = page; cursor = new Lazy<Cursor>(() => { var c = new Cursor(); foreach (var p in cursorPath) { if (p == lastFoundPageNumber) c.Push(pageCopy); else { var cursorPage = tx.GetReadOnlyPage(p); if (key.Options == SliceOptions.BeforeAllKeys) { cursorPage.LastSearchPosition = 0; } else if (key.Options == SliceOptions.AfterAllKeys) { cursorPage.LastSearchPosition = (ushort)(cursorPage.NumberOfEntries - 1); } else if (cursorPage.Search(key, _cmp) != null) { if (cursorPage.LastMatch != 0) { cursorPage.LastSearchPosition--; } } c.Push(cursorPage); } } return c; }); return true; }
public static void Dump(Transaction tx, string path, Page start, int showNodesEvery = 25) { using (var writer = File.CreateText(path)) { writer.WriteLine(@" digraph structs { node [shape=Mrecord] rankdir=LR; bgcolor=transparent; "); var stack = new Stack<Page>(); stack.Push(start); var references = new StringBuilder(); while (stack.Count > 0) { var p = stack.Pop(); writer.WriteLine(@" subgraph cluster_p_{0} {{ label=""Page #{0}""; color={3}; p_{0} [label=""Page: {0}|{1}|Entries: {2:#,#} | {4:p} : {5:p} utilization""]; ", p.PageNumber, p.Flags, p.NumberOfEntries, p.IsLeaf ? "black" : "blue", Math.Round(((AbstractPager.PageSize - p.SizeLeft) / (double)AbstractPager.PageSize), 2), Math.Round(((AbstractPager.PageSize - p.CalcSizeLeft()) / (double)AbstractPager.PageSize), 2)); MemorySlice key = new Slice(SliceOptions.Key); if (p.IsLeaf && showNodesEvery > 0) { writer.WriteLine(" p_{0}_nodes [label=\" Entries:", p.PageNumber); for (int i = 0; i < p.NumberOfEntries; i += showNodesEvery) { if (i != 0 && showNodesEvery >= 5) { writer.WriteLine(" ... {0:#,#} keys redacted ...", showNodesEvery - 1); } var node = p.GetNode(i); key = p.GetNodeKey(node); writer.WriteLine("{0} - {2} {1:#,#}", MaxString(key.ToString(), 25), node->DataSize, node->Flags == NodeFlags.Data ? "Size" : "Page"); } if (p.NumberOfEntries < showNodesEvery) { writer.WriteLine(" ... {0:#,#} keys redacted ...", p.NumberOfEntries - 1); } writer.WriteLine("\"];"); } else if (p.IsBranch) { writer.Write(" p_{0}_refs [label=\"", p.PageNumber); for (int i = 0; i < p.NumberOfEntries; i++) { var node = p.GetNode(i); writer.Write("{3}<{2}> {0} / to page {1}", GetBranchNodeString(i, key, p, node), node->PageNumber, i, i == 0 ? "" : "|"); } writer.WriteLine("\"];"); var prev = -1L; for (int i = 0; i < p.NumberOfEntries; i++) { var node = p.GetNode(i); if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber) { writer.Write(" p_{0}_refs [label=\"CORRUPTED\"; Color=RED];", p.PageNumber); stack.Clear(); break; } var child = tx.GetReadOnlyPage(node->PageNumber); stack.Push(child); references.AppendFormat(" p_{0}_refs:{3} -> p_{1} [label=\"{2}\"];", p.PageNumber, child.PageNumber, GetBranchNodeString(i, key, p, node), i).AppendLine(); if (prev > -1) references.AppendFormat(" p_{0} -> p_{1} [style=\"invis\"];", child.PageNumber, prev); prev = child.PageNumber; } } writer.WriteLine(" }"); } writer.WriteLine(references.ToString()); writer.WriteLine("}"); } }
private void RemoveLeafNode(Transaction tx, Page page, out ushort nodeVersion) { var node = page.GetNode(page.LastSearchPosition); nodeVersion = node->Version; if (node->Flags == (NodeFlags.PageRef)) // this is an overflow pointer { var overflowPage = tx.GetReadOnlyPage(node->PageNumber); var numberOfPages = tx.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize); for (int i = 0; i < numberOfPages; i++) { tx.FreePage(overflowPage.PageNumber + i); } State.OverflowPages -= numberOfPages; State.PageCount -= numberOfPages; } page.RemoveNode(page.LastSearchPosition); }
protected void RenderAndShow(Transaction tx, Tree root, int showEntries = 25) { if (Debugger.IsAttached == false) return; var path = Path.Combine(Environment.CurrentDirectory, "test-tree.dot"); var rootPageNumber = tx.Environment.State.GetTree(tx,root.Name).State.RootPageNumber; TreeDumper.Dump(tx, path, tx.GetReadOnlyPage(rootPageNumber), showEntries); var output = Path.Combine(Environment.CurrentDirectory, "output.svg"); var p = Process.Start(DebugStuff.FindGraphviz() + @"\bin\dot.exe", "-Tsvg " + path + " -o " + output); p.WaitForExit(); Process.Start(output); }
private static unsafe void RenderPage(Transaction tx, Page page, TextWriter sw, string text, bool open) { sw.WriteLine( "<ul><li><input type='checkbox' id='page-{0}' {3} /><label for='page-{0}'>{4}: Page {0:#,#;;0} - {1} - {2:#,#;;0} entries</label><ul>", page.PageNumber, page.IsLeaf ? "Leaf" : "Branch", page.NumberOfEntries, open ? "checked" : "", text); for (int i = 0; i < page.NumberOfEntries; i++) { var nodeHeader = page.GetNode(i); if (page.IsLeaf) { var key = new Slice(nodeHeader).ToString(); sw.Write("<li>{0} {1} - size: {2:#,#}</li>", key, nodeHeader->Flags, NodeHeader.GetDataSize(tx, nodeHeader)); } else { var key = new Slice(nodeHeader).ToString(); var pageNum = nodeHeader->PageNumber; if (i == 0) key = "[smallest]"; RenderPage(tx, tx.GetReadOnlyPage(pageNum), sw, key, false); } } sw.WriteLine("</ul></li></ul>"); }
public static void RenderAndShow(Transaction tx, long startPageNumber, string headerData = null) { RenderHtmlTreeView(writer => { if (headerData != null) writer.WriteLine(headerData); writer.WriteLine("<div class='css-treeview'><ul>"); var page = tx.GetReadOnlyPage(startPageNumber); RenderPage(tx, page, writer, "Root", true); writer.WriteLine("</ul></div>"); }); }
public static void RenderAndShow(Transaction tx, long startPageNumber, int showNodesEvery = 25, string format = "svg") { if (Debugger.IsAttached == false) return; var dateTime = DateTime.UtcNow; if ((dateTime - _lastGenerated).TotalSeconds < 2.5) { return; } _lastGenerated = dateTime; var path = Path.Combine(Environment.CurrentDirectory, "output.dot"); TreeDumper.Dump(tx, path, tx.GetReadOnlyPage(startPageNumber), showNodesEvery); var output = Path.Combine(Environment.CurrentDirectory, "output." + format); var p = Process.Start( FindGraphviz() + @"\bin\dot.exe", "-T" + format + " " + path + " -o " + output); p.WaitForExit(); Process.Start(output); Thread.Sleep(500); }
public void RenderAndShow(Transaction tx, TableBase table, int showEntries = 25) { if (Debugger.IsAttached == false) return; var tree = tx.State.GetTree(tx, table.TableName); var path = Path.Combine(System.Environment.CurrentDirectory, "test-tree.dot"); var rootPageNumber = tree.State.RootPageNumber; TreeDumper.Dump(tx, path, tx.GetReadOnlyPage(rootPageNumber), showEntries); var output = Path.Combine(System.Environment.CurrentDirectory, "output.svg"); var p = Process.Start(@"c:\Program Files (x86)\Graphviz2.32\bin\dot.exe", "-Tsvg " + path + " -o " + output); p.WaitForExit(); Process.Start(output); }
private Page SearchForPage(Transaction tx, Slice key, ref Lazy<Cursor> cursor) { var p = tx.GetReadOnlyPage(State.RootPageNumber); var c = new Cursor(); c.Push(p); bool rightmostPage = true; bool leftmostPage = true; while (p.Flags == (PageFlags.Branch)) { int nodePos; if (key.Options == SliceOptions.BeforeAllKeys) { p.LastSearchPosition = nodePos = 0; rightmostPage = false; } else if (key.Options == SliceOptions.AfterAllKeys) { p.LastSearchPosition = nodePos = (ushort) (p.NumberOfEntries - 1); leftmostPage = false; } else { if (p.Search(key, _cmp) != null) { nodePos = p.LastSearchPosition; if (p.LastMatch != 0) { nodePos--; p.LastSearchPosition--; } if (nodePos != 0) leftmostPage = false; rightmostPage = false; } else { nodePos = (ushort) (p.LastSearchPosition - 1); leftmostPage = false; } } var node = p.GetNode(nodePos); p = tx.GetReadOnlyPage(node->PageNumber); Debug.Assert(node->PageNumber == p.PageNumber, string.Format("Requested Page: #{0}. Got Page: #{1}", node->PageNumber, p.PageNumber)); c.Push(p); } if (p.IsLeaf == false) throw new DataException("Index points to a non leaf page"); p.Search(key, _cmp); // will set the LastSearchPosition AddToRecentlyFoundPages(tx, c, p, leftmostPage, rightmostPage); cursor = new Lazy<Cursor>(() => c); return p; }