| public GetReadOnlyPage ( long pageNumber ) : Voron.Trees.Page | ||
| pageNumber | long | |
| return | Voron.Trees.Page | |
public static void DumpHumanReadable(Transaction tx, long startPageNumber,string filenamePrefix = null)
{
if (Debugger.IsAttached == false)
return;
var path = Path.Combine(Environment.CurrentDirectory, String.Format("{0}tree.hdump", filenamePrefix ?? String.Empty));
TreeDumper.DumpHumanReadable(tx, path, tx.GetReadOnlyPage(startPageNumber));
}
public static void DumpHumanReadable(Transaction tx, string path, Page start)
{
using (var writer = File.CreateText(path))
{
var stack = new Stack<Page>();
stack.Push(start);
writer.WriteLine("Root page #{0}",start.PageNumber);
while (stack.Count > 0)
{
var currentPage = stack.Pop();
if (currentPage.IsLeaf)
{
writer.WriteLine();
writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Leaf), Used: {3} : {4}", currentPage.PageNumber,currentPage.NumberOfEntries,currentPage.Flags, currentPage.SizeUsed, currentPage.CalcSizeUsed());
if(currentPage.NumberOfEntries <= 0)
writer.WriteLine("Empty page (tree corrupted?)");
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries;nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
var key = currentPage.GetNodeKey(node);
writer.WriteLine("Node #{0}, Flags = {1}, {4} = {2}, Key = {3}, Entry Size: {5}", nodeIndex, node->Flags, node->DataSize, MaxString(key.ToString(), 25), node->Flags == NodeFlags.Data ? "Size" : "Page",
SizeOf.NodeEntry(node));
}
writer.WriteLine();
}
else if(currentPage.IsBranch)
{
writer.WriteLine();
writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Branch), Used: {3} : {4}", currentPage.PageNumber, currentPage.NumberOfEntries, currentPage.Flags, currentPage.SizeUsed, currentPage.SizeUsed);
var key = new Slice(SliceOptions.Key);
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
writer.WriteLine("Node #{2}, {0} / to page #{1}, Entry Size: {3}", GetBranchNodeString(nodeIndex, key, currentPage, node), node->PageNumber, nodeIndex,
SizeOf.NodeEntry(node));
}
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber)
{
writer.Write("Found invalid reference to page #{0}", currentPage.PageNumber);
stack.Clear();
break;
}
var child = tx.GetReadOnlyPage(node->PageNumber);
stack.Push(child);
}
writer.WriteLine();
}
}
}
}
public static int GetDataSize(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return overFlowPage.OverflowSize;
}
return node->DataSize;
}
public unsafe static ValueReader Reader(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public static byte* DirectAccess(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return overFlowPage.Base + Constants.PageHeaderSize;
}
return (byte*) node + node->KeySize + Constants.NodeHeaderSize;
}
public static void CopyTo(Transaction tx, NodeHeader* node, byte* dest)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
Memory.Copy(dest, overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
Memory.Copy(dest, (byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public static unsafe bool HasDuplicateBranchReferences(Transaction tx, Page start,out long pageNumberWithDuplicates)
{
var stack = new Stack<Page>();
var existingTreeReferences = new ConcurrentDictionary<long, List<long>>();
stack.Push(start);
while (stack.Count > 0)
{
var currentPage = stack.Pop();
if (currentPage.IsBranch)
{
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
existingTreeReferences.AddOrUpdate(currentPage.PageNumber, new List<long> { node->PageNumber },
(branchPageNumber, pageNumberReferences) =>
{
pageNumberReferences.Add(node->PageNumber);
return pageNumberReferences;
});
}
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber)
{
throw new InvalidDataException("found invalid reference on branch - tree is corrupted");
}
var child = tx.GetReadOnlyPage(node->PageNumber);
stack.Push(child);
}
}
}
Func<long, HashSet<long>> relevantPageReferences =
branchPageNumber => new HashSet<long>(existingTreeReferences
.Where(kvp => kvp.Key != branchPageNumber)
.SelectMany(kvp => kvp.Value));
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var branchReferences in existingTreeReferences)
{
if (
branchReferences.Value.Any(
referencePageNumber => relevantPageReferences(branchReferences.Key).Contains(referencePageNumber)))
{
pageNumberWithDuplicates = branchReferences.Key;
return true;
}
}
pageNumberWithDuplicates = -1;
return false;
}
public static Slice GetData(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
if (overFlowPage.OverflowSize > ushort.MaxValue)
throw new InvalidOperationException("Cannot convert big data to a slice, too big");
return new Slice(overFlowPage.Base + Constants.PageHeaderSize, (ushort)overFlowPage.OverflowSize);
}
return new Slice((byte*)node + node->KeySize + Constants.NodeHeaderSize, (ushort) node->DataSize);
}
public static ValueReader Reader(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
Debug.Assert(overFlowPage.IsOverflow, "Requested oveflow page but got " + overFlowPage.Flags);
Debug.Assert(overFlowPage.OverflowSize > 0, "Overflow page cannot be size equal 0 bytes");
return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public void DebugValidateTree(Transaction tx, long rootPageNumber)
{
var pages = new HashSet<long>();
var stack = new Stack<Page>();
var root = tx.GetReadOnlyPage(rootPageNumber);
stack.Push(root);
pages.Add(rootPageNumber);
while (stack.Count > 0)
{
var p = stack.Pop();
if (p.NumberOfEntries == 0 && p != root)
{
DebugStuff.RenderAndShow(tx, rootPageNumber, 1);
throw new InvalidOperationException("The page " + p.PageNumber + " is empty");
}
p.DebugValidate(tx, _cmp, rootPageNumber);
if (p.IsBranch == false)
continue;
for (int i = 0; i < p.NumberOfEntries; i++)
{
var page = p.GetNode(i)->PageNumber;
if (pages.Add(page) == false)
{
DebugStuff.RenderAndShow(tx, rootPageNumber, 1);
throw new InvalidOperationException("The page " + page + " already appeared in the tree!");
}
stack.Push(tx.GetReadOnlyPage(page));
}
}
}
private bool TryOverwriteOverflowPages(Transaction tx, TreeMutableState treeState, NodeHeader* updatedNode,
Slice key, int len, ushort? version, out byte* pos)
{
if (updatedNode->Flags == NodeFlags.PageRef &&
tx.Id <= tx.Environment.OldestTransaction) // ensure MVCC - do not overwrite if there is some older active transaction that might read those overflows
{
var overflowPage = tx.GetReadOnlyPage(updatedNode->PageNumber);
if (len <= overflowPage.OverflowSize)
{
CheckConcurrency(key, version, updatedNode->Version, TreeActionType.Add);
if (updatedNode->Version == ushort.MaxValue)
updatedNode->Version = 0;
updatedNode->Version++;
var availableOverflows = tx.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize);
var requestedOverflows = tx.DataPager.GetNumberOfOverflowPages(len);
var overflowsToFree = availableOverflows - requestedOverflows;
for (int i = 0; i < overflowsToFree; i++)
{
tx.FreePage(overflowPage.PageNumber + requestedOverflows + i);
}
treeState.OverflowPages -= overflowsToFree;
treeState.PageCount -= overflowsToFree;
overflowPage.OverflowSize = len;
pos = overflowPage.Base + Constants.PageHeaderSize;
return true;
}
}
pos = null;
return false;
}
public List<long> AllPages(Transaction tx)
{
var results = new List<long>();
var stack = new Stack<Page>();
var root = tx.GetReadOnlyPage(State.RootPageNumber);
stack.Push(root);
while (stack.Count > 0)
{
var p = stack.Pop();
results.Add(p.PageNumber);
for (int i = 0; i < p.NumberOfEntries; i++)
{
var node = p.GetNode(i);
var pageNumber = node->PageNumber;
if (p.IsBranch)
{
stack.Push(tx.GetReadOnlyPage(pageNumber));
}
else if (node->Flags == NodeFlags.PageRef)
{
// This is an overflow page
var overflowPage = tx.GetReadOnlyPage(pageNumber);
var numberOfPages = tx.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize);
for (long j = 0; j < numberOfPages; ++j)
results.Add(overflowPage.PageNumber + j);
}
else if (node->Flags == NodeFlags.MultiValuePageRef)
{
var childTreeHeader = (TreeRootHeader*)((byte*)node + node->KeySize + Constants.NodeHeaderSize);
results.Add(childTreeHeader->RootPageNumber);
// this is a multi value
var tree = OpenOrCreateMultiValueTree(tx, new Slice(node), node);
results.AddRange(tree.AllPages(tx));
}
}
}
return results;
}
internal byte* DirectRead(Transaction tx, Slice key)
{
Lazy<Cursor> lazy;
var p = FindPageFor(tx, key, out lazy);
var node = p.Search(key, _cmp);
if (node == null)
return null;
var item1 = new Slice(node);
if (item1.Compare(key, _cmp) != 0)
return null;
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return overFlowPage.Base + Constants.PageHeaderSize;
}
return (byte*) node + node->KeySize + Constants.NodeHeaderSize;
}
private bool TryUseRecentTransactionPage(Transaction tx, Slice key, out Lazy<Cursor> cursor, out Page page)
{
page = null;
cursor = null;
var recentPages = tx.GetRecentlyFoundPages(this);
if (recentPages == null)
return false;
var foundPage = recentPages.Find(key);
if (foundPage == null)
return false;
var lastFoundPageNumber = foundPage.Number;
page = tx.GetReadOnlyPage(lastFoundPageNumber);
if (page.IsLeaf == false)
throw new DataException("Index points to a non leaf page");
page.NodePositionFor(key, _cmp); // will set the LastSearchPosition
var cursorPath = foundPage.CursorPath;
var pageCopy = page;
cursor = new Lazy<Cursor>(() =>
{
var c = new Cursor();
foreach (var p in cursorPath)
{
if (p == lastFoundPageNumber)
c.Push(pageCopy);
else
{
var cursorPage = tx.GetReadOnlyPage(p);
if (key.Options == SliceOptions.BeforeAllKeys)
{
cursorPage.LastSearchPosition = 0;
}
else if (key.Options == SliceOptions.AfterAllKeys)
{
cursorPage.LastSearchPosition = (ushort)(cursorPage.NumberOfEntries - 1);
}
else if (cursorPage.Search(key, _cmp) != null)
{
if (cursorPage.LastMatch != 0)
{
cursorPage.LastSearchPosition--;
}
}
c.Push(cursorPage);
}
}
return c;
});
return true;
}
public static void Dump(Transaction tx, string path, Page start, int showNodesEvery = 25)
{
using (var writer = File.CreateText(path))
{
writer.WriteLine(@"
digraph structs {
node [shape=Mrecord]
rankdir=LR;
bgcolor=transparent;
");
var stack = new Stack<Page>();
stack.Push(start);
var references = new StringBuilder();
while (stack.Count > 0)
{
var p = stack.Pop();
writer.WriteLine(@"
subgraph cluster_p_{0} {{
label=""Page #{0}"";
color={3};
p_{0} [label=""Page: {0}|{1}|Entries: {2:#,#} | {4:p} : {5:p} utilization""];
", p.PageNumber, p.Flags, p.NumberOfEntries, p.IsLeaf ? "black" : "blue",
Math.Round(((AbstractPager.PageSize - p.SizeLeft) / (double)AbstractPager.PageSize), 2),
Math.Round(((AbstractPager.PageSize - p.CalcSizeLeft()) / (double)AbstractPager.PageSize), 2));
MemorySlice key = new Slice(SliceOptions.Key);
if (p.IsLeaf && showNodesEvery > 0)
{
writer.WriteLine(" p_{0}_nodes [label=\" Entries:", p.PageNumber);
for (int i = 0; i < p.NumberOfEntries; i += showNodesEvery)
{
if (i != 0 && showNodesEvery >= 5)
{
writer.WriteLine(" ... {0:#,#} keys redacted ...", showNodesEvery - 1);
}
var node = p.GetNode(i);
key = p.GetNodeKey(node);
writer.WriteLine("{0} - {2} {1:#,#}", MaxString(key.ToString(), 25),
node->DataSize, node->Flags == NodeFlags.Data ? "Size" : "Page");
}
if (p.NumberOfEntries < showNodesEvery)
{
writer.WriteLine(" ... {0:#,#} keys redacted ...", p.NumberOfEntries - 1);
}
writer.WriteLine("\"];");
}
else if (p.IsBranch)
{
writer.Write(" p_{0}_refs [label=\"", p.PageNumber);
for (int i = 0; i < p.NumberOfEntries; i++)
{
var node = p.GetNode(i);
writer.Write("{3}<{2}> {0} / to page {1}", GetBranchNodeString(i, key, p, node), node->PageNumber,
i, i == 0 ? "" : "|");
}
writer.WriteLine("\"];");
var prev = -1L;
for (int i = 0; i < p.NumberOfEntries; i++)
{
var node = p.GetNode(i);
if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber)
{
writer.Write(" p_{0}_refs [label=\"CORRUPTED\"; Color=RED];", p.PageNumber);
stack.Clear();
break;
}
var child = tx.GetReadOnlyPage(node->PageNumber);
stack.Push(child);
references.AppendFormat(" p_{0}_refs:{3} -> p_{1} [label=\"{2}\"];", p.PageNumber, child.PageNumber, GetBranchNodeString(i, key, p, node), i).AppendLine();
if (prev > -1)
references.AppendFormat(" p_{0} -> p_{1} [style=\"invis\"];", child.PageNumber, prev);
prev = child.PageNumber;
}
}
writer.WriteLine(" }");
}
writer.WriteLine(references.ToString());
writer.WriteLine("}");
}
}
private void RemoveLeafNode(Transaction tx, Page page, out ushort nodeVersion)
{
var node = page.GetNode(page.LastSearchPosition);
nodeVersion = node->Version;
if (node->Flags == (NodeFlags.PageRef)) // this is an overflow pointer
{
var overflowPage = tx.GetReadOnlyPage(node->PageNumber);
var numberOfPages = tx.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize);
for (int i = 0; i < numberOfPages; i++)
{
tx.FreePage(overflowPage.PageNumber + i);
}
State.OverflowPages -= numberOfPages;
State.PageCount -= numberOfPages;
}
page.RemoveNode(page.LastSearchPosition);
}
protected void RenderAndShow(Transaction tx, Tree root, int showEntries = 25)
{
if (Debugger.IsAttached == false)
return;
var path = Path.Combine(Environment.CurrentDirectory, "test-tree.dot");
var rootPageNumber = tx.Environment.State.GetTree(tx,root.Name).State.RootPageNumber;
TreeDumper.Dump(tx, path, tx.GetReadOnlyPage(rootPageNumber), showEntries);
var output = Path.Combine(Environment.CurrentDirectory, "output.svg");
var p = Process.Start(DebugStuff.FindGraphviz() + @"\bin\dot.exe", "-Tsvg " + path + " -o " + output);
p.WaitForExit();
Process.Start(output);
}
private static unsafe void RenderPage(Transaction tx, Page page, TextWriter sw, string text, bool open)
{
sw.WriteLine(
"<ul><li><input type='checkbox' id='page-{0}' {3} /><label for='page-{0}'>{4}: Page {0:#,#;;0} - {1} - {2:#,#;;0} entries</label><ul>",
page.PageNumber, page.IsLeaf ? "Leaf" : "Branch", page.NumberOfEntries, open ? "checked" : "", text);
for (int i = 0; i < page.NumberOfEntries; i++)
{
var nodeHeader = page.GetNode(i);
if (page.IsLeaf)
{
var key = new Slice(nodeHeader).ToString();
sw.Write("<li>{0} {1} - size: {2:#,#}</li>", key, nodeHeader->Flags, NodeHeader.GetDataSize(tx, nodeHeader));
}
else
{
var key = new Slice(nodeHeader).ToString();
var pageNum = nodeHeader->PageNumber;
if (i == 0)
key = "[smallest]";
RenderPage(tx, tx.GetReadOnlyPage(pageNum), sw, key, false);
}
}
sw.WriteLine("</ul></li></ul>");
}
public static void RenderAndShow(Transaction tx, long startPageNumber, string headerData = null)
{
RenderHtmlTreeView(writer =>
{
if (headerData != null)
writer.WriteLine(headerData);
writer.WriteLine("<div class='css-treeview'><ul>");
var page = tx.GetReadOnlyPage(startPageNumber);
RenderPage(tx, page, writer, "Root", true);
writer.WriteLine("</ul></div>");
});
}
public static void RenderAndShow(Transaction tx, long startPageNumber, int showNodesEvery = 25, string format = "svg")
{
if (Debugger.IsAttached == false)
return;
var dateTime = DateTime.UtcNow;
if ((dateTime - _lastGenerated).TotalSeconds < 2.5)
{
return;
}
_lastGenerated = dateTime;
var path = Path.Combine(Environment.CurrentDirectory, "output.dot");
TreeDumper.Dump(tx, path, tx.GetReadOnlyPage(startPageNumber), showNodesEvery);
var output = Path.Combine(Environment.CurrentDirectory, "output." + format);
var p = Process.Start( FindGraphviz() + @"\bin\dot.exe", "-T" + format + " " + path + " -o " + output);
p.WaitForExit();
Process.Start(output);
Thread.Sleep(500);
}
public void RenderAndShow(Transaction tx, TableBase table, int showEntries = 25)
{
if (Debugger.IsAttached == false)
return;
var tree = tx.State.GetTree(tx, table.TableName);
var path = Path.Combine(System.Environment.CurrentDirectory, "test-tree.dot");
var rootPageNumber = tree.State.RootPageNumber;
TreeDumper.Dump(tx, path, tx.GetReadOnlyPage(rootPageNumber), showEntries);
var output = Path.Combine(System.Environment.CurrentDirectory, "output.svg");
var p = Process.Start(@"c:\Program Files (x86)\Graphviz2.32\bin\dot.exe", "-Tsvg " + path + " -o " + output);
p.WaitForExit();
Process.Start(output);
}
private Page SearchForPage(Transaction tx, Slice key, ref Lazy<Cursor> cursor)
{
var p = tx.GetReadOnlyPage(State.RootPageNumber);
var c = new Cursor();
c.Push(p);
bool rightmostPage = true;
bool leftmostPage = true;
while (p.Flags == (PageFlags.Branch))
{
int nodePos;
if (key.Options == SliceOptions.BeforeAllKeys)
{
p.LastSearchPosition = nodePos = 0;
rightmostPage = false;
}
else if (key.Options == SliceOptions.AfterAllKeys)
{
p.LastSearchPosition = nodePos = (ushort) (p.NumberOfEntries - 1);
leftmostPage = false;
}
else
{
if (p.Search(key, _cmp) != null)
{
nodePos = p.LastSearchPosition;
if (p.LastMatch != 0)
{
nodePos--;
p.LastSearchPosition--;
}
if (nodePos != 0)
leftmostPage = false;
rightmostPage = false;
}
else
{
nodePos = (ushort) (p.LastSearchPosition - 1);
leftmostPage = false;
}
}
var node = p.GetNode(nodePos);
p = tx.GetReadOnlyPage(node->PageNumber);
Debug.Assert(node->PageNumber == p.PageNumber,
string.Format("Requested Page: #{0}. Got Page: #{1}", node->PageNumber, p.PageNumber));
c.Push(p);
}
if (p.IsLeaf == false)
throw new DataException("Index points to a non leaf page");
p.Search(key, _cmp); // will set the LastSearchPosition
AddToRecentlyFoundPages(tx, c, p, leftmostPage, rightmostPage);
cursor = new Lazy<Cursor>(() => c);
return p;
}