public Searcher(FullTextIndex index) { _index = index; _tx = _index.StorageEnvironment.NewTransaction(TransactionFlags.Read); _docs = _tx.ReadTree("Docs"); }
public TransactionActivityEntry(Transaction tx, DebugActionType actionType) { TransactionId = tx.Id; Flags = tx.Flags; ActionType = actionType; CreatedByJournalApplicator = tx.CreatedByJournalApplicator; }
public bool TryGetValue(Transaction tx, long page, out PagePosition value) { ImmutableAppendOnlyList<PagePosition> list; if (_values.TryGetValue(page, out list) == false) { value = null; return false; } for (int i = list.Count - 1; i >= 0; i--) { var it = list[i]; if (it.TransactionId > tx.Id) continue; if (it.IsFreedPageMarker) break; value = it; Debug.Assert(value != null); return true; } // all the current values are _after_ this transaction started, so it sees nothing value = null; return false; }
public PageSplitter(Transaction tx, Tree tree, SliceComparer cmp, Slice newKey, int len, long pageNumber, NodeFlags nodeType, ushort nodeVersion, Cursor cursor, TreeMutableState treeState) { _tx = tx; _tree = tree; _cmp = cmp; _newKey = newKey; _len = len; _pageNumber = pageNumber; _nodeType = nodeType; _nodeVersion = nodeVersion; _cursor = cursor; _treeState = treeState; Page page = _cursor.Pages.First.Value; _page = tx.ModifyPage(page.PageNumber, page); _cursor.Pop(); }
public static void DumpHumanReadable(Transaction tx, long startPageNumber,string filenamePrefix = null) { if (Debugger.IsAttached == false) return; var path = Path.Combine(Environment.CurrentDirectory, String.Format("{0}tree.hdump", filenamePrefix ?? String.Empty)); TreeDumper.DumpHumanReadable(tx, path, tx.GetReadOnlyPage(startPageNumber)); }
public override void AllocateMorePages(Transaction tx, long newLength) { ThrowObjectDisposedIfNeeded(); var newLengthAfterAdjustment = NearestSizeToPageSize(newLength); if (newLengthAfterAdjustment <= _totalAllocationSize) return; var allocationSize = newLengthAfterAdjustment - _totalAllocationSize; PosixHelper.AllocateFileSpace(_fd, (ulong) (_totalAllocationSize + allocationSize)); PagerState newPagerState = CreatePagerState(); if (newPagerState == null) { var errorMessage = string.Format( "Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes", (_totalAllocationSize + allocationSize)); throw new OutOfMemoryException(errorMessage); } newPagerState.DebugVerify(newLengthAfterAdjustment); if (tx != null) { tx.EnsurePagerStateReference(newPagerState); } var tmp = PagerState; PagerState = newPagerState; tmp.Release(); //replacing the pager state --> so one less reference for it _totalAllocationSize += allocationSize; NumberOfAllocatedPages = _totalAllocationSize/PageSize; }
public void Initialize(FullTextIndex index, Transaction tx, IndexingConventions.ScorerCalc score) { Index = index; Transaction = tx; Score = score; Init(); }
public static void DumpHumanReadable(Transaction tx, string path, Page start) { using (var writer = File.CreateText(path)) { var stack = new Stack<Page>(); stack.Push(start); writer.WriteLine("Root page #{0}",start.PageNumber); while (stack.Count > 0) { var currentPage = stack.Pop(); if (currentPage.IsLeaf) { writer.WriteLine(); writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Leaf), Used: {3} : {4}", currentPage.PageNumber,currentPage.NumberOfEntries,currentPage.Flags, currentPage.SizeUsed, currentPage.CalcSizeUsed()); if(currentPage.NumberOfEntries <= 0) writer.WriteLine("Empty page (tree corrupted?)"); for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries;nodeIndex++) { var node = currentPage.GetNode(nodeIndex); var key = currentPage.GetNodeKey(node); writer.WriteLine("Node #{0}, Flags = {1}, {4} = {2}, Key = {3}, Entry Size: {5}", nodeIndex, node->Flags, node->DataSize, MaxString(key.ToString(), 25), node->Flags == NodeFlags.Data ? "Size" : "Page", SizeOf.NodeEntry(node)); } writer.WriteLine(); } else if(currentPage.IsBranch) { writer.WriteLine(); writer.WriteLine("Page #{0}, NumberOfEntries = {1}, Flags = {2} (Branch), Used: {3} : {4}", currentPage.PageNumber, currentPage.NumberOfEntries, currentPage.Flags, currentPage.SizeUsed, currentPage.SizeUsed); var key = new Slice(SliceOptions.Key); for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); writer.WriteLine("Node #{2}, {0} / to page #{1}, Entry Size: {3}", GetBranchNodeString(nodeIndex, key, currentPage, node), node->PageNumber, nodeIndex, SizeOf.NodeEntry(node)); } for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber) { writer.Write("Found invalid reference to page #{0}", currentPage.PageNumber); stack.Clear(); break; } var child = tx.GetReadOnlyPage(node->PageNumber); stack.Push(child); } writer.WriteLine(); } } } }
public ParentPageAction(Page parentPage, Page currentPage, Tree tree, Cursor cursor, Transaction tx) { _parentPage = parentPage; _currentPage = currentPage; _tree = tree; _cursor = cursor; _tx = tx; }
public static void CopyTo(Transaction tx, NodeHeader* node, byte* dest) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); Memory.Copy(dest, overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize); } Memory.Copy(dest, (byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize); }
public static byte* DirectAccess(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); return overFlowPage.Base + Constants.PageHeaderSize; } return (byte*) node + node->KeySize + Constants.NodeHeaderSize; }
public static int GetDataSize(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); return overFlowPage.OverflowSize; } return node->DataSize; }
public unsafe static ValueReader Reader(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize); } return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize); }
private void UpdateMaxSeenTxId(Transaction tx) { if (_maxSeenTransaction > tx.Id) { throw new InvalidOperationException("Transaction ids has to always increment, but got " + tx.Id + " when already seen tx " + _maxSeenTransaction); } _maxSeenTransaction = tx.Id; }
public StorageEnvironmentState Clone(Transaction tx) { return new StorageEnvironmentState { Root = Root != null ? Root.Clone(tx) : null, FreeSpaceRoot = FreeSpaceRoot != null ? FreeSpaceRoot.Clone(tx) : null, NextPageNumber = NextPageNumber }; }
public static unsafe bool HasDuplicateBranchReferences(Transaction tx, Page start,out long pageNumberWithDuplicates) { var stack = new Stack<Page>(); var existingTreeReferences = new ConcurrentDictionary<long, List<long>>(); stack.Push(start); while (stack.Count > 0) { var currentPage = stack.Pop(); if (currentPage.IsBranch) { for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); existingTreeReferences.AddOrUpdate(currentPage.PageNumber, new List<long> { node->PageNumber }, (branchPageNumber, pageNumberReferences) => { pageNumberReferences.Add(node->PageNumber); return pageNumberReferences; }); } for (int nodeIndex = 0; nodeIndex < currentPage.NumberOfEntries; nodeIndex++) { var node = currentPage.GetNode(nodeIndex); if (node->PageNumber < 0 || node->PageNumber > tx.State.NextPageNumber) { throw new InvalidDataException("found invalid reference on branch - tree is corrupted"); } var child = tx.GetReadOnlyPage(node->PageNumber); stack.Push(child); } } } Func<long, HashSet<long>> relevantPageReferences = branchPageNumber => new HashSet<long>(existingTreeReferences .Where(kvp => kvp.Key != branchPageNumber) .SelectMany(kvp => kvp.Value)); // ReSharper disable once LoopCanBeConvertedToQuery foreach (var branchReferences in existingTreeReferences) { if ( branchReferences.Value.Any( referencePageNumber => relevantPageReferences(branchReferences.Key).Contains(referencePageNumber))) { pageNumberWithDuplicates = branchReferences.Key; return true; } } pageNumberWithDuplicates = -1; return false; }
public PageFromScratchBuffer Allocate(Transaction tx, int numberOfPages) { if (tx == null) throw new ArgumentNullException("tx"); var size = Utils.NearestPowerOfTwo(numberOfPages); PageFromScratchBuffer result; if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) return result; if ((_lastUsedPage + size)*AbstractPager.PageSize > _sizeLimit) { var sp = Stopwatch.StartNew(); // Our problem is that we don't have any available free pages, probably because // there are read transactions that are holding things open. We are going to see if // there are any free pages that _might_ be freed for us if we wait for a bit. The idea // is that we let the read transactions time to complete and do their work, at which point // we can continue running. // We start this by forcing a flush, then we are waiting up to the timeout for we are waiting // for the read transactions to complete. It is possible that a long running read transaction // would in fact generate enough work for us to timeout, but hopefully we can avoid that. tx.Environment.ForceLogFlushToDataFile(tx); while (sp.ElapsedMilliseconds < tx.Environment.Options.ScratchBufferOverflowTimeout) { if (TryGettingFromAllocatedBuffer(tx, numberOfPages, size, out result)) return result; Thread.Sleep(32); } string message = string.Format("Cannot allocate more space for the scratch buffer.\r\n" + "Current size is:\t{0:#,#;;0} kb.\r\n" + "Limit:\t\t\t{1:#,#;;0} kb.\r\n" + "Requested Size:\t{2:#,#;;0} kb.\r\n" + "Already flushed and waited for {3:#,#;;0} ms for read transactions to complete.\r\n" + "Do you have a long running read transaction executing?", (_scratchPager.NumberOfAllocatedPages*AbstractPager.PageSize)/1024, _sizeLimit/1024, ((_lastUsedPage + size)*AbstractPager.PageSize)/1024, sp.ElapsedMilliseconds); throw new ScratchBufferSizeLimitException(message); } // we don't have free pages to give out, need to allocate some _scratchPager.EnsureContinuous(tx, _lastUsedPage, (int)size); result = new PageFromScratchBuffer { PositionInScratchBuffer = _lastUsedPage, Size = size, NumberOfPages = numberOfPages }; _allocatedPages.Add(_lastUsedPage, result); _lastUsedPage += size; return result; }
public TreeIterator(Tree tree, Transaction tx) { _tree = tree; _tx = tx; if (tree.KeysPrefixing) _currentInternalKey = new PrefixedSlice(SliceOptions.Key); else _currentInternalKey = new Slice(SliceOptions.Key); }
public static Slice GetData(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); if (overFlowPage.OverflowSize > ushort.MaxValue) throw new InvalidOperationException("Cannot convert big data to a slice, too big"); return new Slice(overFlowPage.Base + Constants.PageHeaderSize, (ushort)overFlowPage.OverflowSize); } return new Slice((byte*)node + node->KeySize + Constants.NodeHeaderSize, (ushort) node->DataSize); }
public void SetItems(Transaction tx, Dictionary<long, PagePosition> items) { UpdateMaxSeenTxId(tx); foreach (var item in items) { var copy = item; _values.AddOrUpdate(copy.Key, l => ImmutableAppendOnlyList<PagePosition>.Empty.Append(copy.Value), (l, list) => list.Append(copy.Value)); } }
public void Add(Transaction tx, Slice key, Stream value, ushort? version = null) { if (value == null) throw new ArgumentNullException("value"); if (value.Length > int.MaxValue) throw new ArgumentException("Cannot add a value that is over 2GB in size", "value"); State.IsModified = true; var pos = DirectAdd(tx, key, (int)value.Length, version: version); CopyStreamToPointer(tx, value, pos); }
private static void SetCurrentTopologyInternal(Topology currentTopology, long index, Transaction tx) { if (currentTopology.TopologyId == Guid.Empty) throw new InvalidOperationException("Cannot set topology with an empty TopologyId"); var metadata = tx.ReadTree(MetadataTreeName); var current = metadata.Read("current-topology"); metadata.Add("previous-topology", current == null ? "{}" : current.Reader.ToStringValue()); metadata.Add("current-topology", JsonConvert.SerializeObject(currentTopology)); metadata.Add("current-topology-index", EndianBitConverter.Little.GetBytes(index)); }
public static ValueReader Reader(Transaction tx, NodeHeader* node) { if (node->Flags == (NodeFlags.PageRef)) { var overFlowPage = tx.GetReadOnlyPage(node->PageNumber); Debug.Assert(overFlowPage.IsOverflow, "Requested oveflow page but got " + overFlowPage.Flags); Debug.Assert(overFlowPage.OverflowSize > 0, "Overflow page cannot be size equal 0 bytes"); return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize); } return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize); }
public long Increment(Transaction tx, Slice key, long delta, ushort? version = null) { long currentValue = 0; var read = Read(tx, key); if (read != null) currentValue = read.Reader.ReadLittleEndianInt64(); var value = currentValue + delta; Add(tx, key, BitConverter.GetBytes(value), version); return value; }
public static Tree Create(Transaction tx, SliceComparer cmp, TreeFlags flags = TreeFlags.None) { var newRootPage = NewPage(tx, PageFlags.Leaf, 1); var tree = new Tree(cmp, newRootPage.PageNumber) { _state = { Depth = 1, Flags = flags, InWriteTransaction = true } }; tree.State.RecordNewPage(newRootPage, 1); return tree; }
public void SetItems(Transaction tx, Dictionary<long, JournalFile.PagePosition> items) { UpdateMaxSeenTxId(tx); foreach (var item in items) { var copy = item; _values.AddOrUpdate(copy.Key, l => ImmutableAppendOnlyList<PageValue>.Empty.Append(new PageValue { Transaction = tx.Id, Value = copy.Value }), (l, list) => list.Append(new PageValue { Transaction = tx.Id, Value = copy.Value })); } }
public static Tree Create(Transaction tx, bool keysPrefixing, TreeFlags flags = TreeFlags.None) { var newRootPage = NewPage(tx, keysPrefixing ? PageFlags.Leaf | PageFlags.KeysPrefixed : PageFlags.Leaf, 1); var tree = new Tree(tx, newRootPage.PageNumber) { _state = { Depth = 1, Flags = flags, InWriteTransaction = true, KeysPrefixing = keysPrefixing } }; tree.State.RecordNewPage(newRootPage, 1); return tree; }
public static Tree Open(Transaction tx, SliceComparer cmp, TreeRootHeader* header) { return new Tree(cmp, header->RootPageNumber) { _state = { PageCount = header->PageCount, BranchPages = header->BranchPages, Depth = header->Depth, OverflowPages = header->OverflowPages, LeafPages = header->LeafPages, EntriesCount = header->EntriesCount, Flags = header->Flags, InWriteTransaction = tx.Flags.HasFlag(TransactionFlags.ReadWrite) } }; }
public static Tree GetTree(this StorageEnvironmentState state, Transaction tx, string treeName) { if (String.IsNullOrEmpty(treeName)) throw new InvalidOperationException("Cannot fetch tree with empty name"); if (treeName.Equals(Constants.RootTreeName, StringComparison.InvariantCultureIgnoreCase)) return state.Root; if (treeName.Equals(Constants.FreeSpaceTreeName, StringComparison.InvariantCultureIgnoreCase)) return state.FreeSpaceRoot; Tree tree = tx.ReadTree(treeName); if (tree != null) return tree; throw new InvalidOperationException("No such tree: " + treeName); }
public Page TryAllocateFromFreeSpace(Transaction tx, int num) { if (_env.State.FreeSpaceRoot == null) return null;// this can happen the first time FreeSpaceRoot tree is created long page; if (_current == null || _current.Sequences.TryAllocate(num, out page) == false) { if (_current != null) { // now we need to decide whatever we discard the current section (in favor of a better one?) // or just let this write go to the end of the file if (_current.Sequences.Count > _lastTransactionPageUsage) { // we still have a lot of free pages here we can use, let us continue using this section return null; } } // need to find a new one var old = _current; try { if (SetupNextSection(tx, num, _currentKey) == false) { _currentKey = null; _current = null; _currentChanged = false; return null; } } finally { DiscardSection(old); } return TryAllocateFromFreeSpace(tx, num); } _currentChanged = true; var newPage = tx.Pager.Get(tx, page); newPage.PageNumber = page; return newPage; }