public Searcher(FullTextIndex index)
{
_index = index;
_tx = _index.StorageEnvironment.NewTransaction(TransactionFlags.Read);
_docs = _tx.ReadTree("Docs");
}
public PageSplitter(Transaction tx,
Tree tree,
SliceComparer cmp,
Slice newKey,
int len,
long pageNumber,
NodeFlags nodeType,
ushort nodeVersion,
Cursor cursor,
TreeMutableState treeState)
{
_tx = tx;
_tree = tree;
_cmp = cmp;
_newKey = newKey;
_len = len;
_pageNumber = pageNumber;
_nodeType = nodeType;
_nodeVersion = nodeVersion;
_cursor = cursor;
_treeState = treeState;
Page page = _cursor.Pages.First.Value;
_page = tx.ModifyPage(page.PageNumber, page);
_cursor.Pop();
}
public StorageEnvironmentState(Tree freeSpaceRoot, Tree root, long nextPageNumber)
{
if (freeSpaceRoot != null)
FreeSpaceRoot = freeSpaceRoot.State;
if (root != null)
Root = root.State;
NextPageNumber = nextPageNumber;
}
public ParentPageAction(Page parentPage, Page currentPage, Tree tree, Cursor cursor, Transaction tx)
{
_parentPage = parentPage;
_currentPage = currentPage;
_tree = tree;
_cursor = cursor;
_tx = tx;
}
public TreeIterator(Tree tree, Transaction tx)
{
_tree = tree;
_tx = tx;
if (tree.KeysPrefixing)
_currentInternalKey = new PrefixedSlice(SliceOptions.Key);
else
_currentInternalKey = new Slice(SliceOptions.Key);
}
protected override void Init()
{
_fieldTree = Transaction.ReadTree("@fld_" + _field);
if (_fieldTree == null)
return;
var termFreqInDocs = _fieldTree.State.EntriesCount;
var numberOfDocs = Transaction.ReadTree("$metadata").Read(Transaction, "docs").Reader.ReadInt64();
var idf = Index.Conventions.Idf(termFreqInDocs, numberOfDocs);
_weight = idf*idf;
}
protected override void Init()
{
base.Init();
_positionsTree = Transaction.ReadTree("TermPositions");
_docsTree = Transaction.ReadTree("Docs");
_fieldId = Index.GetFieldNumber(Field);
_prefix = new byte[FullTextIndex.FieldDocumentSize];
_prefixSlice = new Slice(_prefix);
_maxKey = new byte[FullTextIndex.FieldDocumentSize];
_maxKeySlice = new Slice(_maxKey);
}
public static Tree Create(Transaction tx, SliceComparer cmp, TreeFlags flags = TreeFlags.None)
{
var newRootPage = NewPage(tx, PageFlags.Leaf, 1);
var tree = new Tree(cmp, newRootPage.PageNumber)
{
_state =
{
Depth = 1,
Flags = flags,
InWriteTransaction = true
}
};
tree.State.RecordNewPage(newRootPage, 1);
return tree;
}
public static Tree Create(Transaction tx, bool keysPrefixing, TreeFlags flags = TreeFlags.None)
{
var newRootPage = NewPage(tx, keysPrefixing ? PageFlags.Leaf | PageFlags.KeysPrefixed : PageFlags.Leaf, 1);
var tree = new Tree(tx, newRootPage.PageNumber)
{
_state =
{
Depth = 1,
Flags = flags,
InWriteTransaction = true,
KeysPrefixing = keysPrefixing
}
};
tree.State.RecordNewPage(newRootPage, 1);
return tree;
}
private static long ReadIsTopologyChanged(Tree metadata)
{
var readResult = metadata.Read("current-topology-index");
if (readResult == null)
return -1;
return readResult.Reader.ReadLittleEndianInt64();
}
protected void RenderAndShow(Transaction tx, Tree root, int showEntries = 25)
{
if (Debugger.IsAttached == false)
return;
var path = Path.Combine(Environment.CurrentDirectory, "test-tree.dot");
var rootPageNumber = tx.Environment.State.GetTree(tx,root.Name).State.RootPageNumber;
TreeDumper.Dump(tx, path, tx.GetReadOnlyPage(rootPageNumber), showEntries);
var output = Path.Combine(Environment.CurrentDirectory, "output.svg");
var p = Process.Start(DebugStuff.FindGraphviz() + @"\bin\dot.exe", "-Tsvg " + path + " -o " + output);
p.WaitForExit();
Process.Start(output);
}
public StorageEnvironmentState(Tree freeSpaceRoot, Tree root, long nextPageNumber)
{
FreeSpaceRoot = freeSpaceRoot;
Root = root;
NextPageNumber = nextPageNumber;
}
internal RecentlyFoundPages GetRecentlyFoundPages(Tree tree)
{
RecentlyFoundPages pages;
if (_recentlyFoundPages.TryGetValue(tree, out pages))
return pages;
return null;
}
public byte *Execute()
{
Page rightPage = Tree.NewPage(_tx, _page.Flags, 1);
_treeState.RecordNewPage(_page, 1);
if (_cursor.PageCount == 0) // we need to do a root split
{
Page newRootPage = Tree.NewPage(_tx, _tree.KeysPrefixing ? PageFlags.Branch | PageFlags.KeysPrefixed : PageFlags.Branch, 1);
_cursor.Push(newRootPage);
_treeState.RootPageNumber = newRootPage.PageNumber;
_treeState.Depth++;
_treeState.RecordNewPage(newRootPage, 1);
// now add implicit left page
newRootPage.AddPageRefNode(0, _tree.KeysPrefixing ? (MemorySlice)PrefixedSlice.BeforeAllKeys : Slice.BeforeAllKeys, _page.PageNumber);
_parentPage = newRootPage;
_parentPage.LastSearchPosition++;
}
else
{
// we already popped the page, so the current one on the stack is the parent of the page
if (_tree.Name == Constants.FreeSpaceTreeName)
{
// a special case for FreeSpaceTree because the allocation of a new page called above
// can cause a delete of a free space section resulting in a run of the tree rebalancer
// and here the parent page that exists in cursor can be outdated
_parentPage = _tx.ModifyPage(_cursor.CurrentPage.PageNumber, null); // pass _null_ to make sure we'll get the most updated parent page
_parentPage.LastSearchPosition = _cursor.CurrentPage.LastSearchPosition;
_parentPage.LastMatch = _cursor.CurrentPage.LastMatch;
}
else
{
_parentPage = _tx.ModifyPage(_cursor.CurrentPage.PageNumber, _cursor.CurrentPage);
}
_cursor.Update(_cursor.Pages.First, _parentPage);
}
if (_page.IsLeaf)
{
_tree.ClearRecentFoundPages();
}
if (_tree.Name == Constants.FreeSpaceTreeName)
{
// we need to refresh the LastSearchPosition of the split page which is used by the free space handling
// because the allocation of a new page called above could remove some sections
// from the page that is being split
_page.NodePositionFor(_newKey);
}
if (_page.LastSearchPosition >= _page.NumberOfEntries)
{
// when we get a split at the end of the page, we take that as a hint that the user is doing
// sequential inserts, at that point, we are going to keep the current page as is and create a new
// page, this will allow us to do minimal amount of work to get the best density
byte *pos;
if (_page.IsBranch)
{
// here we steal the last entry from the current page so we maintain the implicit null left entry
NodeHeader *node = _page.GetNode(_page.NumberOfEntries - 1);
Debug.Assert(node->Flags == NodeFlags.PageRef);
rightPage.AddPageRefNode(0, _tree.KeysPrefixing ? (MemorySlice)PrefixedSlice.BeforeAllKeys : Slice.BeforeAllKeys, node->PageNumber);
pos = AddNodeToPage(rightPage, 1);
var separatorKey = _page.GetNodeKey(node);
AddSeparatorToParentPage(rightPage.PageNumber, separatorKey);
_page.RemoveNode(_page.NumberOfEntries - 1);
}
else
{
AddSeparatorToParentPage(rightPage.PageNumber, _newKey);
pos = AddNodeToPage(rightPage, 0);
}
_cursor.Push(rightPage);
return(pos);
}
return(SplitPageInHalf(rightPage));
}
internal bool TryRemoveMultiValueTree(Tree parentTree, MemorySlice key)
{
var keyToRemove = Tuple.Create(parentTree, key);
if (_multiValueTrees == null || !_multiValueTrees.ContainsKey(keyToRemove))
return false;
return _multiValueTrees.Remove(keyToRemove);
}
internal void AddMultiValueTree(Tree tree, MemorySlice key, Tree mvTree)
{
if (_multiValueTrees == null)
_multiValueTrees = new Dictionary<Tuple<Tree, MemorySlice>, Tree>(new TreeAndSliceComparer());
mvTree.IsMultiValueTree = true;
_multiValueTrees.Add(Tuple.Create(tree, key), mvTree);
}
internal Page ModifyPage(long num, Tree tree, Page page)
{
_env.AssertFlushingNotFailed();
page = page ?? GetReadOnlyPage(num);
if (page.Dirty)
return page;
if (_dirtyPages.Contains(num))
{
page.Dirty = true;
return page;
}
var newPage = AllocatePage(1, PageFlags.None, num); // allocate new page in a log file but with the same number
Memory.Copy(newPage.Base, page.Base, AbstractPager.PageSize);
newPage.LastSearchPosition = page.LastSearchPosition;
newPage.LastMatch = page.LastMatch;
tree.RecentlyFoundPages.Reset(num);
return newPage;
}
internal void UpdateRootsIfNeeded(Tree root, Tree freeSpace)
{
//can only happen during initial transaction that creates Root and FreeSpaceRoot trees
if (State.Root == null && State.FreeSpaceRoot == null)
{
State.Root = root;
State.FreeSpaceRoot = freeSpace;
}
}
public TreeRebalancer(Transaction tx, Tree tree, Cursor cursor)
{
_tx = tx;
_tree = tree;
_cursor = cursor;
}
private static long GetLastEntryNumber(Tree logs, Transaction tx)
{
long lastEntry;
var lastKey = logs.LastKeyOrDefault();
if (lastKey != null)
{
lastEntry = lastKey.CreateReader().ReadBigEndianInt64();
}
else
{
var metadata = tx.ReadTree(MetadataTreeName);
// maybe there is a snapshot?
var snapshotIndex = metadata.Read("snapshot-index");
if (snapshotIndex != null)
lastEntry = snapshotIndex.Reader.ReadLittleEndianInt64();
else
lastEntry = 0;
}
return lastEntry;
}
private OverflowsAddResult AddOverflows(Transaction tx, Tree tree, int treeNumber, Random r)
{
var minOverflowSize = AbstractPager.NodeMaxSize - Constants.PageHeaderSize + 1;
var entriesAdded = new List<string>();
var overflowsAdded = 0;
for (int j = 0; j < treeNumber; j++)
{
var overflowSize = r.Next(minOverflowSize, 10000);
string key = "overflow_" + j;
tree.Add(key, new MemoryStream(new byte[overflowSize]));
entriesAdded.Add(key);
overflowsAdded += tx.DataPager.GetNumberOfOverflowPages(overflowSize);
}
return new OverflowsAddResult
{
AddedEntries = entriesAdded,
NumberOfOverflowPages = overflowsAdded
};
}
public TreeRebalancer(Transaction tx, Tree tree)
{
_tx = tx;
_tree = tree;
}
public TreeIterator(Tree tree, Transaction tx)
{
_tree = tree;
_tx = tx;
}
public unsafe List<Slice> Keys(Tree t, Transaction tx)
{
var results = new List<Slice>();
using (var it = t.Iterate())
{
if (it.Seek(Slice.BeforeAllKeys) == false)
return results;
do
{
results.Add(it.CurrentKey);
} while (it.MoveNext());
}
return results;
}
private void InitializeRoots()
{
if (_state.Root != null)
{
_state.Root.InWriteTransaction = Flags == TransactionFlags.ReadWrite;
_root = new Tree(this, _state.Root) {Name = Constants.RootTreeName};
}
if (_state.FreeSpaceRoot != null)
{
_state.FreeSpaceRoot.InWriteTransaction = Flags == TransactionFlags.ReadWrite;
_freeSpace = new Tree(this, _state.FreeSpaceRoot) {Name = Constants.FreeSpaceTreeName};
}
}
public Reader(CounterStorage parent, Transaction t)
{
this.parent = parent;
transaction = t;
serverNamesToIds = transaction.State.GetTree(transaction, "serverNames->Ids");
serverIdsToNames = transaction.State.GetTree(transaction, "Ids->serverNames");
serversLastEtag = transaction.State.GetTree(transaction, "servers->lastEtag");
counters = transaction.State.GetTree(transaction, "counters");
countersGroups = transaction.State.GetTree(transaction, "countersGroups");
countersEtags = transaction.State.GetTree(transaction, "counters->etags");
etagsCounters = transaction.State.GetTree(transaction, "etags->counters");
metadata = transaction.State.GetTree(transaction, "$metadata");
}
internal void UpdateRootsIfNeeded(Tree root, Tree freeSpace)
{
//can only happen during initial transaction that creates Root and FreeSpaceRoot trees
if (State.Root != null || State.FreeSpaceRoot != null)
return;
State.Root = root.State;
State.FreeSpaceRoot = freeSpace.State;
_root = root;
_freeSpace = freeSpace;
}
public Writer(CounterStorage parent, StorageEnvironment storageEnvironment)
{
this.parent = parent;
transaction = storageEnvironment.NewTransaction(TransactionFlags.ReadWrite);
reader = new Reader(parent, transaction);
serverNamesToIds = transaction.State.GetTree(transaction, "serverNames->Ids");
serverIdsToNames = transaction.State.GetTree(transaction, "Ids->serverNames");
serversLastEtag = transaction.State.GetTree(transaction, "servers->lastEtag");
counters = transaction.State.GetTree(transaction, "counters");
countersGroups = transaction.State.GetTree(transaction, "countersGroups");
etagsCountersIx = transaction.State.GetTree(transaction, "etags->counters");
countersEtagIx = transaction.State.GetTree(transaction, "counters->etags");
metadata = transaction.State.GetTree(transaction, "$metadata");
storeBuffer = new byte[sizeof(long) + //positive
sizeof(long)]; // negative
storeBufferLength = storeBuffer.Length;
}
internal bool TryGetMultiValueTree(Tree tree, MemorySlice key, out Tree mvTree)
{
mvTree = null;
if (_multiValueTrees == null)
return false;
return _multiValueTrees.TryGetValue(Tuple.Create(tree, key), out mvTree);
}
internal void AddRecentlyFoundPage(Tree tree, RecentlyFoundPages.FoundPage foundPage)
{
RecentlyFoundPages pages;
if (_recentlyFoundPages.TryGetValue(tree, out pages) == false)
_recentlyFoundPages[tree] = pages = new RecentlyFoundPages(Flags == TransactionFlags.Read ? 8 : 2);
pages.Add(foundPage);
}
internal void AddTree(string name, Tree tree)
{
Tree value;
if (_trees.TryGetValue(name, out value) && value != null)
{
throw new InvalidOperationException("Tree already exists: " + name);
}
_trees[name] = tree;
}
internal void ClearRecentFoundPages(Tree tree)
{
_recentlyFoundPages.Remove(tree);
}
