public override void Setup() { base.Setup(); using (var tx = Env.WriteTransaction()) { tx.CreateTree(TreeNameSlice); tx.Commit(); } var totalPairs = Utils.GenerateUniqueRandomSlicePairs( NumberOfTransactions * NumberOfRecordsPerTransaction, KeyLength, RandomSeed == -1 ? null as int? : RandomSeed); // This will sort just the KEYS totalPairs.Sort((x, y) => SliceComparer.Compare(x.Item1, y.Item1)); // Distribute keys in such a way that _pairs[i][k] < _pairs[j][m] // iff i < j, for all k and m. _pairs = new List <Tuple <Slice, Slice> > [NumberOfTransactions]; for (var i = 0; i < NumberOfTransactions; i++) { _pairs[i] = totalPairs.Take(NumberOfRecordsPerTransaction).ToList(); totalPairs.RemoveRange(0, NumberOfRecordsPerTransaction); } }
public FoundTreePage Find(Slice key) { int position = _current; int itemsLeft = _cacheSize; while (itemsLeft > 0) { var page = _cache[position % _cacheSize]; if (page == null) { itemsLeft--; position++; continue; } var first = page.FirstKey; var last = page.LastKey; switch (key.Options) { case SliceOptions.Key: if ((first.Options != SliceOptions.BeforeAllKeys && SliceComparer.Compare(key, first) < 0)) { break; } if (last.Options != SliceOptions.AfterAllKeys && SliceComparer.Compare(key, last) > 0) { break; } return(page); case SliceOptions.BeforeAllKeys: if (first.Options == SliceOptions.BeforeAllKeys) { return(page); } break; case SliceOptions.AfterAllKeys: if (last.Options == SliceOptions.AfterAllKeys) { return(page); } break; default: throw new ArgumentException(key.Options.ToString()); } itemsLeft--; position++; } return(null); }
public override void Setup() { base.Setup(); using (var tx = Env.WriteTransaction()) { Schema.Create(tx, TableNameSlice, 16); tx.Commit(); } var totalPairs = Utils.GenerateUniqueRandomSlicePairs( NumberOfTransactions * NumberOfRecordsPerTransaction, KeyLength, RandomSeed == -1 ? null as int? : RandomSeed); // This will sort just the KEYS totalPairs.Sort((x, y) => SliceComparer.Compare(x.Item1, y.Item1)); // Distribute keys in such a way that _valueBuilders[i][k] < // _valueBuilders[j][m] iff i < j, for all k and m. _valueBuilders = new List <TableValueBuilder> [NumberOfTransactions]; for (int i = 0; i < NumberOfTransactions; i++) { var values = totalPairs.Take(NumberOfRecordsPerTransaction); totalPairs.RemoveRange(0, NumberOfRecordsPerTransaction); _valueBuilders[i] = new List <TableValueBuilder>(); foreach (var pair in values) { _valueBuilders[i].Add(new TableValueBuilder { pair.Item1, pair.Item2 }); } } }
public bool TryFindPageForReading(Slice key, LowLevelTransaction tx, out DecompressedLeafPage result) { Debug.Assert(key.Options == SliceOptions.Key); var position = _current; var itemsLeft = Size; while (itemsLeft > 0) { var page = _cache[position % Size]; if (page == null || page.Usage != DecompressionUsage.Read || page.NumberOfEntries == 0) // decompressed page can has 0 entries if each compressed entry had a tombstone marker { itemsLeft--; position++; continue; } Slice first; Slice last; using (page.GetNodeKey(tx, 0, out first)) using (page.GetNodeKey(tx, page.NumberOfEntries - 1, out last)) { if (SliceComparer.Compare(key, first) >= 0 && SliceComparer.Compare(key, last) <= 0) { result = page; return(true); } } itemsLeft--; position++; } result = null; return(false); }
private byte *SplitPageInHalf(TreePage rightPage) { bool toRight; var currentIndex = _page.LastSearchPosition; var splitIndex = _page.NumberOfEntries / 2; if (currentIndex <= splitIndex) { toRight = false; } else { toRight = true; var leftPageEntryCount = splitIndex; var rightPageEntryCount = _page.NumberOfEntries - leftPageEntryCount + 1; if (rightPageEntryCount > leftPageEntryCount) { splitIndex++; Debug.Assert(splitIndex < _page.NumberOfEntries); } } if (_page.IsLeaf) { splitIndex = AdjustSplitPosition(currentIndex, splitIndex, ref toRight); } Slice currentKey; using (_page.GetNodeKey(_tx, splitIndex, out currentKey)) { Slice seperatorKey; if (toRight && splitIndex == currentIndex) { seperatorKey = SliceComparer.Compare(currentKey, _newKey) < 0 ? currentKey : _newKey; } else { seperatorKey = currentKey; } var addedAsImplicitRef = false; var parentOfPage = _cursor.CurrentPage; TreePage parentOfRight; DecompressedLeafPage rightDecompressed = null; if (_pageDecompressed != null) { // splitting the decompressed page, let's allocate the page of the same size to ensure enough space rightDecompressed = _tx.Environment.DecompressionBuffers.GetPage(_tx, _pageDecompressed.PageSize, DecompressionUsage.Write, rightPage); rightPage = rightDecompressed; } using (rightDecompressed) { AddSeparatorToParentPage(rightPage.PageNumber, seperatorKey, out parentOfRight); if (_page.IsBranch && toRight && SliceComparer.EqualsInline(seperatorKey, _newKey)) { // _newKey needs to be inserted as first key (BeforeAllKeys) to the right page, so we need to add it before we move entries from the current page AddNodeToPage(rightPage, 0, Slices.BeforeAllKeys); addedAsImplicitRef = true; } // move the actual entries from page to right page ushort nKeys = _page.NumberOfEntries; for (int i = splitIndex; i < nKeys; i++) { TreeNodeHeader *node = _page.GetNode(i); if (_page.IsBranch && rightPage.NumberOfEntries == 0) { rightPage.CopyNodeDataToEndOfPage(node, Slices.BeforeAllKeys); } else { Slice instance; using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out instance)) { rightPage.CopyNodeDataToEndOfPage(node, instance); } } } if (rightDecompressed != null) { rightDecompressed.CopyToOriginal(_tx, defragRequired: false, wasModified: true); rightPage = rightDecompressed.Original; } } _page.Truncate(_tx, splitIndex); RecompressPageIfNeeded(wasModified: true); byte *pos; if (addedAsImplicitRef == false) { try { if (toRight && _cursor.CurrentPage.PageNumber != parentOfRight.PageNumber) { // modify the cursor if we are going to insert to the right page _cursor.Pop(); _cursor.Push(parentOfRight); } // actually insert the new key pos = InsertNewKey(toRight ? rightPage : _page); } catch (InvalidOperationException e) { if ( e.Message.StartsWith("The page is full and cannot add an entry", StringComparison.Ordinal) == false) { throw; } throw new InvalidOperationException( GatherDetailedDebugInfo(rightPage, currentKey, seperatorKey, currentIndex, splitIndex, toRight), e); } } else { pos = null; _cursor.Push(rightPage); } if (_page.IsBranch) // remove a branch that has only one entry, the page ref needs to be added to the parent of the current page { Debug.Assert(_page.NumberOfEntries > 0); Debug.Assert(rightPage.NumberOfEntries > 0); if (_page.NumberOfEntries == 1) { RemoveBranchWithOneEntry(_page, parentOfPage); } if (rightPage.NumberOfEntries == 1) { RemoveBranchWithOneEntry(rightPage, parentOfRight); } } return(pos); } }
private static bool SetupPrefix(IIterator it, string prefix, TransactionOperationContext context, out ByteStringContext.InternalScope?scope) { scope = Slice.From(context.Transaction.InnerTransaction.Allocator, prefix, out Slice prefixSlice); it.SetRequiredPrefix(prefixSlice); if (it.Seek(prefixSlice)) { return(true); } scope.Value.Dispose(); scope = null; it.SetRequiredPrefix(Slices.Empty); if (it.Seek(Slices.BeforeAllKeys) == false) { return(false); } if (SliceComparer.Compare(it.CurrentKey, MapReduceIndexingContext.LastMapResultIdKey) == 0) { if (it.MoveNext() == false) { return(false); } } var firstKey = it.CurrentKey.ToString(); if (it.Seek(Slices.AfterAllKeys) == false) { return(false); } var lastKey = it.CurrentKey.ToString(); int index = -1; for (int i = 0; i < Math.Min(firstKey.Length, lastKey.Length); i++) { if (firstKey[i] != lastKey[i]) { break; } index = i; } if (index == -1) { return(false); } prefix = firstKey.Substring(0, index + 1) + prefix; scope = Slice.From(context.Transaction.InnerTransaction.Allocator, prefix, out prefixSlice); it.SetRequiredPrefix(prefixSlice); if (it.Seek(prefixSlice) == false) { scope.Value.Dispose(); scope = null; return(false); } return(true); }
public int Compare(FdbKeySelector x, FdbKeySelector y) { return(s_comparer.Compare(x.Key, y.Key)); }
private byte *SplitPageInHalf(TreePage rightPage) { bool toRight; var currentIndex = _page.LastSearchPosition; var splitIndex = _page.NumberOfEntries / 2; if (currentIndex <= splitIndex) { toRight = false; } else { toRight = true; var leftPageEntryCount = splitIndex; var rightPageEntryCount = _page.NumberOfEntries - leftPageEntryCount + 1; if (rightPageEntryCount > leftPageEntryCount) { splitIndex++; Debug.Assert(splitIndex < _page.NumberOfEntries); } } if (_page.IsLeaf) { splitIndex = AdjustSplitPosition(currentIndex, splitIndex, ref toRight); } var currentKey = _page.GetNodeKey(_tx, splitIndex); Slice seperatorKey; if (toRight && splitIndex == currentIndex) { seperatorKey = SliceComparer.Compare(currentKey, _newKey) < 0 ? currentKey : _newKey; } else { seperatorKey = currentKey; } TreePage parentOfRight; AddSeparatorToParentPage(rightPage.PageNumber, seperatorKey, out parentOfRight); var parentOfPage = _cursor.CurrentPage; bool addedAsImplicitRef = false; if (_page.IsBranch && toRight && SliceComparer.EqualsInline(seperatorKey, _newKey)) { // _newKey needs to be inserted as first key (BeforeAllKeys) to the right page, so we need to add it before we move entries from the current page AddNodeToPage(rightPage, 0, Slices.BeforeAllKeys); addedAsImplicitRef = true; } // move the actual entries from page to right page var instance = new Slice(); ushort nKeys = _page.NumberOfEntries; for (int i = splitIndex; i < nKeys; i++) { TreeNodeHeader *node = _page.GetNode(i); if (_page.IsBranch && rightPage.NumberOfEntries == 0) { rightPage.CopyNodeDataToEndOfPage(node, Slices.BeforeAllKeys); } else { instance = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node); rightPage.CopyNodeDataToEndOfPage(node, instance); } } _page.Truncate(_tx, splitIndex); byte *pos; if (addedAsImplicitRef == false) { try { if (toRight && _cursor.CurrentPage.PageNumber != parentOfRight.PageNumber) { // modify the cursor if we are going to insert to the right page _cursor.Pop(); _cursor.Push(parentOfRight); } // actually insert the new key pos = toRight ? InsertNewKey(rightPage) : InsertNewKey(_page); } catch (InvalidOperationException e) { if (e.Message.StartsWith("The page is full and cannot add an entry", StringComparison.Ordinal) == false) { throw; } throw new InvalidOperationException(GatherDetailedDebugInfo(rightPage, currentKey, seperatorKey, currentIndex, splitIndex, toRight), e); } } else { pos = null; _cursor.Push(rightPage); } if (_page.IsBranch) // remove a branch that has only one entry, the page ref needs to be added to the parent of the current page { Debug.Assert(_page.NumberOfEntries > 0); Debug.Assert(rightPage.NumberOfEntries > 0); if (_page.NumberOfEntries == 1) { RemoveBranchWithOneEntry(_page, parentOfPage); } if (rightPage.NumberOfEntries == 1) { RemoveBranchWithOneEntry(rightPage, parentOfRight); } } return(pos); }
private static long CopyTableTree(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token) { // Load table var tableTree = txr.ReadTree(treeName, RootObjectType.Table); // Get the table schema var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice); var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice); var schema = TableSchema.ReadFrom(txr.Allocator, schemaPtr, schemaSize); // Load table into structure var inputTable = txr.OpenTable(schema, treeName); // The next three variables are used to know what our current // progress is var copiedEntries = 0; // It is very important that these slices be allocated in the // txr.Allocator, as the intermediate write transactions on // the compacted environment will be destroyed between each // loop. var lastSlice = Slices.BeforeAllKeys; long lastFixedIndex = 0L; Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries}/{inputTable.NumberOfEntries} entries.", treeName); using (var txw = compactedEnv.WriteTransaction(context)) { schema.Create(txw, treeName, Math.Max((ushort)inputTable.ActiveDataSmallSection.NumberOfPages, (ushort)((ushort.MaxValue + 1) / Constants.Storage.PageSize))); txw.Commit(); // always create a table, even if it is empty } while (copiedEntries < inputTable.NumberOfEntries) { token.ThrowIfCancellationRequested(); using (var txw = compactedEnv.WriteTransaction(context)) { long transactionSize = 0L; var outputTable = txw.OpenTable(schema, treeName); if (schema.Key == null || schema.Key.IsGlobal) { // There is no primary key, or there is one that is global to multiple tables // we require a table to have at least a single local index that we'll use var variableSizeIndex = schema.Indexes.Values.FirstOrDefault(x => x.IsGlobal == false); if (variableSizeIndex != null) { // We have a variable size index, use it // In case we continue an existing compaction, skip to the next slice var skip = 0; if (SliceComparer.Compare(lastSlice, Slices.BeforeAllKeys) != 0) { skip = 1; } foreach (var tvr in inputTable.SeekForwardFrom(variableSizeIndex, lastSlice, skip)) { // The table will take care of reconstructing indexes automatically outputTable.Insert(ref tvr.Result.Reader); copiedEntries++; transactionSize += tvr.Result.Reader.Size; var reportRate = inputTable.NumberOfEntries / 33 + 1; if (copiedEntries % reportRate == 0) { Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries}/{inputTable.NumberOfEntries} entries.", treeName); } // The transaction has surpassed the allowed // size before a flush if (lastSlice.Equals(tvr.Key) == false && transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2) { lastSlice = tvr.Key.Clone(txr.Allocator); break; } } } else { // Use a fixed size index var fixedSizeIndex = schema.FixedSizeIndexes.Values.FirstOrDefault(x => x.IsGlobal == false); if (fixedSizeIndex == null) { throw new InvalidOperationException("Cannot compact table " + inputTable.Name + " because is has no local indexes, only global ones"); } foreach (var entry in inputTable.SeekForwardFrom(fixedSizeIndex, lastFixedIndex, lastFixedIndex > 0 ? 1 : 0)) { token.ThrowIfCancellationRequested(); // The table will take care of reconstructing indexes automatically outputTable.Insert(ref entry.Reader); copiedEntries++; transactionSize += entry.Reader.Size; var reportRate = inputTable.NumberOfEntries / 33 + 1; if (copiedEntries % reportRate == 0) { Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries}/{inputTable.NumberOfEntries} entries.", treeName); } // The transaction has surpassed the allowed // size before a flush if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2) { lastFixedIndex = fixedSizeIndex.GetValue(ref entry.Reader); break; } } } } else { // The table has a primary key, inserts in that order are expected to be faster foreach (var entry in inputTable.SeekByPrimaryKey(lastSlice, 0)) { token.ThrowIfCancellationRequested(); // The table will take care of reconstructing indexes automatically outputTable.Insert(ref entry.Reader); copiedEntries++; transactionSize += entry.Reader.Size; // The transaction has surpassed the allowed // size before a flush if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2) { schema.Key.GetSlice(txr.Allocator, ref entry.Reader, out lastSlice); break; } } } txw.Commit(); } if (copiedEntries == inputTable.NumberOfEntries) { copiedTrees++; Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Finished copying table tree '{treeName}'. Progress: {copiedEntries}/{inputTable.NumberOfEntries} entries.", treeName); } compactedEnv.FlushLogToDataFile(); } return(copiedTrees); }