public Searcher(FullTextIndex index)
{
_index = index;
_tx = _index.StorageEnvironment.NewTransaction(TransactionFlags.Read);
_docs = _tx.ReadTree("Docs");
}
public TransactionActivityEntry(Transaction tx, DebugActionType actionType)
{
TransactionId = tx.Id;
Flags = tx.Flags;
ActionType = actionType;
CreatedByJournalApplicator = tx.CreatedByJournalApplicator;
}
public bool TryGetValue(Transaction tx, long page, out PagePosition value)
{
ImmutableAppendOnlyList<PagePosition> list;
if (_values.TryGetValue(page, out list) == false)
{
value = null;
return false;
}
for (int i = list.Count - 1; i >= 0; i--)
{
var it = list[i];
if (it.TransactionId > tx.Id)
continue;
if (it.IsFreedPageMarker)
break;
value = it;
Debug.Assert(value != null);
return true;
}
// all the current values are _after_ this transaction started, so it sees nothing
value = null;
return false;
}
public PageSplitter(Transaction tx,
Tree tree,
SliceComparer cmp,
Slice newKey,
int len,
long pageNumber,
NodeFlags nodeType,
ushort nodeVersion,
Cursor cursor,
TreeMutableState treeState)
{
_tx = tx;
_tree = tree;
_cmp = cmp;
_newKey = newKey;
_len = len;
_pageNumber = pageNumber;
_nodeType = nodeType;
_nodeVersion = nodeVersion;
_cursor = cursor;
_treeState = treeState;
Page page = _cursor.Pages.First.Value;
_page = tx.ModifyPage(page.PageNumber, page);
_cursor.Pop();
}
public static void DumpHumanReadable(Transaction tx, long startPageNumber,string filenamePrefix = null)
{
if (Debugger.IsAttached == false)
return;
var path = Path.Combine(Environment.CurrentDirectory, String.Format("{0}tree.hdump", filenamePrefix ?? String.Empty));
TreeDumper.DumpHumanReadable(tx, path, tx.GetReadOnlyPage(startPageNumber));
}
public override void AllocateMorePages(Transaction tx, long newLength)
{
ThrowObjectDisposedIfNeeded();
var newLengthAfterAdjustment = NearestSizeToPageSize(newLength);
if (newLengthAfterAdjustment <= _totalAllocationSize)
return;
var allocationSize = newLengthAfterAdjustment - _totalAllocationSize;
PosixHelper.AllocateFileSpace(_fd, (ulong) (_totalAllocationSize + allocationSize));
PagerState newPagerState = CreatePagerState();
if (newPagerState == null)
{
var errorMessage = string.Format(
"Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes",
(_totalAllocationSize + allocationSize));
throw new OutOfMemoryException(errorMessage);
}
newPagerState.DebugVerify(newLengthAfterAdjustment);
if (tx != null)
{
tx.EnsurePagerStateReference(newPagerState);
}
var tmp = PagerState;
PagerState = newPagerState;
tmp.Release(); //replacing the pager state --> so one less reference for it
_totalAllocationSize += allocationSize;
NumberOfAllocatedPages = _totalAllocationSize/PageSize;
}
public void Initialize(FullTextIndex index, Transaction tx, IndexingConventions.ScorerCalc score)
{
Index = index;
Transaction = tx;
Score = score;
Init();
}
public static void DumpHumanReadable(Transaction tx, string path, Page start)
{
using (var writer = File.CreateText(path))
{
var stack = new Stack<Page>();
stack.Push(start);
writer.WriteLine("Root page #{0}",start.PageNumber);
while (stack.Count > 0)
{
var currentPage = stack.Pop();
if (currentPage.IsLeaf)
{
writer.WriteLine();
writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Leaf), Used: {3} : {4}", currentPage.PageNumber,currentPage.NumberOfEntries,currentPage.Flags, currentPage.SizeUsed, currentPage.CalcSizeUsed());
if(currentPage.NumberOfEntries <= 0)
writer.WriteLine("Empty page (tree corrupted?)");
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries;nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
var key = currentPage.GetNodeKey(node);
writer.WriteLine("Node #{0}, Flags = {1}, {4} = {2}, Key = {3}, Entry Size: {5}", nodeIndex, node->Flags, node->DataSize, MaxString(key.ToString(), 25), node->Flags == NodeFlags.Data ? "Size" : "Page",
SizeOf.NodeEntry(node));
}
writer.WriteLine();
}
else if(currentPage.IsBranch)
{
writer.WriteLine();
writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Branch), Used: {3} : {4}", currentPage.PageNumber, currentPage.NumberOfEntries, currentPage.Flags, currentPage.SizeUsed, currentPage.SizeUsed);
var key = new Slice(SliceOptions.Key);
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
writer.WriteLine("Node #{2}, {0} / to page #{1}, Entry Size: {3}", GetBranchNodeString(nodeIndex, key, currentPage, node), node->PageNumber, nodeIndex,
SizeOf.NodeEntry(node));
}
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber)
{
writer.Write("Found invalid reference to page #{0}", currentPage.PageNumber);
stack.Clear();
break;
}
var child = tx.GetReadOnlyPage(node->PageNumber);
stack.Push(child);
}
writer.WriteLine();
}
}
}
}
public ParentPageAction(Page parentPage, Page currentPage, Tree tree, Cursor cursor, Transaction tx)
{
_parentPage = parentPage;
_currentPage = currentPage;
_tree = tree;
_cursor = cursor;
_tx = tx;
}
public static void CopyTo(Transaction tx, NodeHeader* node, byte* dest)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
Memory.Copy(dest, overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
Memory.Copy(dest, (byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public static byte* DirectAccess(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return overFlowPage.Base + Constants.PageHeaderSize;
}
return (byte*) node + node->KeySize + Constants.NodeHeaderSize;
}
public static int GetDataSize(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return overFlowPage.OverflowSize;
}
return node->DataSize;
}
public unsafe static ValueReader Reader(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
private void UpdateMaxSeenTxId(Transaction tx)
{
if (_maxSeenTransaction > tx.Id)
{
throw new InvalidOperationException("Transaction ids has to always increment, but got " + tx.Id +
" when already seen tx " + _maxSeenTransaction);
}
_maxSeenTransaction = tx.Id;
}
public StorageEnvironmentState Clone(Transaction tx)
{
return new StorageEnvironmentState
{
Root = Root != null ? Root.Clone(tx) : null,
FreeSpaceRoot = FreeSpaceRoot != null ? FreeSpaceRoot.Clone(tx) : null,
NextPageNumber = NextPageNumber
};
}
public static unsafe bool HasDuplicateBranchReferences(Transaction tx, Page start,out long pageNumberWithDuplicates)
{
var stack = new Stack<Page>();
var existingTreeReferences = new ConcurrentDictionary<long, List<long>>();
stack.Push(start);
while (stack.Count > 0)
{
var currentPage = stack.Pop();
if (currentPage.IsBranch)
{
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
existingTreeReferences.AddOrUpdate(currentPage.PageNumber, new List<long> { node->PageNumber },
(branchPageNumber, pageNumberReferences) =>
{
pageNumberReferences.Add(node->PageNumber);
return pageNumberReferences;
});
}
for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++)
{
var node = currentPage.GetNode(nodeIndex);
if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber)
{
throw new InvalidDataException("found invalid reference on branch - tree is corrupted");
}
var child = tx.GetReadOnlyPage(node->PageNumber);
stack.Push(child);
}
}
}
Func<long, HashSet<long>> relevantPageReferences =
branchPageNumber => new HashSet<long>(existingTreeReferences
.Where(kvp => kvp.Key != branchPageNumber)
.SelectMany(kvp => kvp.Value));
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var branchReferences in existingTreeReferences)
{
if (
branchReferences.Value.Any(
referencePageNumber => relevantPageReferences(branchReferences.Key).Contains(referencePageNumber)))
{
pageNumberWithDuplicates = branchReferences.Key;
return true;
}
}
pageNumberWithDuplicates = -1;
return false;
}
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages)
{
if (tx == null) throw new ArgumentNullException("tx");
var size = Utils.NearestPowerOfTwo(numberOfPages);
PageFromScratchBuffer result;
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
return result;
if ((_lastUsedPage + size)*AbstractPager.PageSize > _sizeLimit)
{
var sp = Stopwatch.StartNew();
// Our problem is that we don't have any available free pages, probably because
// there are read transactions that are holding things open. We are going to see if
// there are any free pages that _might_ be freed for us if we wait for a bit. The idea
// is that we let the read transactions time to complete and do their work, at which point
// we can continue running.
// We start this by forcing a flush, then we are waiting up to the timeout for we are waiting
// for the read transactions to complete. It is possible that a long running read transaction
// would in fact generate enough work for us to timeout, but hopefully we can avoid that.
tx.Environment.ForceLogFlushToDataFile(tx);
while (sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout)
{
if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result))
return result;
Thread.Sleep(32);
}
string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" +
"Current size is:\t{0:#,#;;0} kb.\r\n" +
"Limit:\t\t\t{1:#,#;;0} kb.\r\n" +
"Requested Size:\t{2:#,#;;0} kb.\r\n" +
"Already flushed and waited for {3:#,#;;0} ms for read transactions to complete.\r\n" +
"Do you have a long running read transaction executing?",
(_scratchPager.NumberOfAllocatedPages*AbstractPager.PageSize)/1024,
_sizeLimit/1024,
((_lastUsedPage + size)*AbstractPager.PageSize)/1024,
sp.ElapsedMilliseconds);
throw new ScratchBufferSizeLimitException(message);
}
// we don't have free pages to give out, need to allocate some
_scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size);
result = new PageFromScratchBuffer
{
PositionInScratchBuffer = _lastUsedPage,
Size = size,
NumberOfPages = numberOfPages
};
_allocatedPages.Add(_lastUsedPage, result);
_lastUsedPage += size;
return result;
}
public TreeIterator(Tree tree, Transaction tx)
{
_tree = tree;
_tx = tx;
if (tree.KeysPrefixing)
_currentInternalKey = new PrefixedSlice(SliceOptions.Key);
else
_currentInternalKey = new Slice(SliceOptions.Key);
}
public static Slice GetData(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
if (overFlowPage.OverflowSize > ushort.MaxValue)
throw new InvalidOperationException("Cannot convert big data to a slice, too big");
return new Slice(overFlowPage.Base + Constants.PageHeaderSize, (ushort)overFlowPage.OverflowSize);
}
return new Slice((byte*)node + node->KeySize + Constants.NodeHeaderSize, (ushort) node->DataSize);
}
public void SetItems(Transaction tx, Dictionary<long, PagePosition> items)
{
UpdateMaxSeenTxId(tx);
foreach (var item in items)
{
var copy = item;
_values.AddOrUpdate(copy.Key, l => ImmutableAppendOnlyList<PagePosition>.Empty.Append(copy.Value),
(l, list) => list.Append(copy.Value));
}
}
public void Add(Transaction tx, Slice key, Stream value, ushort? version = null)
{
if (value == null) throw new ArgumentNullException("value");
if (value.Length > int.MaxValue)
throw new ArgumentException("Cannot add a value that is over 2GB in size", "value");
State.IsModified = true;
var pos = DirectAdd(tx, key, (int)value.Length, version: version);
CopyStreamToPointer(tx, value, pos);
}
private static void SetCurrentTopologyInternal(Topology currentTopology, long index, Transaction tx)
{
if (currentTopology.TopologyId == Guid.Empty)
throw new InvalidOperationException("Cannot set topology with an empty TopologyId");
var metadata = tx.ReadTree(MetadataTreeName);
var current = metadata.Read("current-topology");
metadata.Add("previous-topology", current == null ? "{}" : current.Reader.ToStringValue());
metadata.Add("current-topology", JsonConvert.SerializeObject(currentTopology));
metadata.Add("current-topology-index", EndianBitConverter.Little.GetBytes(index));
}
public static ValueReader Reader(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
Debug.Assert(overFlowPage.IsOverflow, "Requested oveflow page but got " + overFlowPage.Flags);
Debug.Assert(overFlowPage.OverflowSize > 0, "Overflow page cannot be size equal 0 bytes");
return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public long Increment(Transaction tx, Slice key, long delta, ushort? version = null)
{
long currentValue = 0;
var read = Read(tx, key);
if (read != null)
currentValue = read.Reader.ReadLittleEndianInt64();
var value = currentValue + delta;
Add(tx, key, BitConverter.GetBytes(value), version);
return value;
}
public static Tree Create(Transaction tx, SliceComparer cmp, TreeFlags flags = TreeFlags.None)
{
var newRootPage = NewPage(tx, PageFlags.Leaf, 1);
var tree = new Tree(cmp, newRootPage.PageNumber)
{
_state =
{
Depth = 1,
Flags = flags,
InWriteTransaction = true
}
};
tree.State.RecordNewPage(newRootPage, 1);
return tree;
}
public void SetItems(Transaction tx, Dictionary<long, JournalFile.PagePosition> items)
{
UpdateMaxSeenTxId(tx);
foreach (var item in items)
{
var copy = item;
_values.AddOrUpdate(copy.Key, l => ImmutableAppendOnlyList<PageValue>.Empty.Append(new PageValue
{
Transaction = tx.Id,
Value = copy.Value
}), (l, list) => list.Append(new PageValue
{
Transaction = tx.Id,
Value = copy.Value
}));
}
}
public static Tree Create(Transaction tx, bool keysPrefixing, TreeFlags flags = TreeFlags.None)
{
var newRootPage = NewPage(tx, keysPrefixing ? PageFlags.Leaf | PageFlags.KeysPrefixed : PageFlags.Leaf, 1);
var tree = new Tree(tx, newRootPage.PageNumber)
{
_state =
{
Depth = 1,
Flags = flags,
InWriteTransaction = true,
KeysPrefixing = keysPrefixing
}
};
tree.State.RecordNewPage(newRootPage, 1);
return tree;
}
public static Tree Open(Transaction tx, SliceComparer cmp, TreeRootHeader* header)
{
return new Tree(cmp, header->RootPageNumber)
{
_state =
{
PageCount = header->PageCount,
BranchPages = header->BranchPages,
Depth = header->Depth,
OverflowPages = header->OverflowPages,
LeafPages = header->LeafPages,
EntriesCount = header->EntriesCount,
Flags = header->Flags,
InWriteTransaction = tx.Flags.HasFlag(TransactionFlags.ReadWrite)
}
};
}
public static Tree GetTree(this StorageEnvironmentState state, Transaction tx, string treeName)
{
if (String.IsNullOrEmpty(treeName))
throw new InvalidOperationException("Cannot fetch tree with empty name");
if (treeName.Equals(Constants.RootTreeName, StringComparison.InvariantCultureIgnoreCase))
return state.Root;
if (treeName.Equals(Constants.FreeSpaceTreeName, StringComparison.InvariantCultureIgnoreCase))
return state.FreeSpaceRoot;
Tree tree = tx.ReadTree(treeName);
if (tree != null)
return tree;
throw new InvalidOperationException("No such tree: " + treeName);
}
public Page TryAllocateFromFreeSpace(Transaction tx, int num)
{
if (_env.State.FreeSpaceRoot == null)
return null;// this can happen the first time FreeSpaceRoot tree is created
long page;
if (_current == null ||
_current.Sequences.TryAllocate(num, out page) == false)
{
if (_current != null)
{
// now we need to decide whatever we discard the current section (in favor of a better one?)
// or just let this write go to the end of the file
if (_current.Sequences.Count > _lastTransactionPageUsage)
{
// we still have a lot of free pages here we can use, let us continue using this section
return null;
}
}
// need to find a new one
var old = _current;
try
{
if (SetupNextSection(tx, num, _currentKey) == false)
{
_currentKey = null;
_current = null;
_currentChanged = false;
return null;
}
}
finally
{
DiscardSection(old);
}
return TryAllocateFromFreeSpace(tx, num);
}
_currentChanged = true;
var newPage = tx.Pager.Get(tx, page);
newPage.PageNumber = page;
return newPage;
}
