/// <summary>Create a new pair of key selectors using FIRST_GREATER_OR_EQUAL on both keys</summary> public static KeySelectorPair Create(KeyRange range) { return(new KeySelectorPair( KeySelector.FirstGreaterOrEqual(range.Begin), KeySelector.FirstGreaterOrEqual(range.End) )); }
/// <summary>Create a new pair of key selectors using FIRST_GREATER_OR_EQUAL on both keys</summary> public static KeySelectorPair Create(Slice begin, Slice end) { return(new KeySelectorPair( KeySelector.FirstGreaterOrEqual(begin), KeySelector.FirstGreaterOrEqual(end) )); }
protected override async ValueTask <bool> OnFirstAsync() { this.RemainingCount = this.Query.Limit; this.RemainingSize = this.Query.TargetBytes; this.Begin = this.Query.Begin; this.End = this.Query.End; if (this.RemainingCount == 0) { // we can safely optimize this case by not doing any query, because it should not have any impact on conflict resolutions. // => The result of 'query.Take(0)' will not change even if someone adds/remove to the range // => The result of 'query.Take(X)' where X would be computed from reads in the db, and be equal to 0, would conflict because of those reads anyway. return(false); } var bounds = this.Query.OriginalRange; // if the original range has been changed, we need to ensure that the current begin/end do not overflow: if (this.Begin != bounds.Begin || this.End != bounds.End) { //TODO: find a better way to do this! var keys = await this.Transaction.GetKeysAsync(new[] { bounds.Begin, this.Begin, bounds.End, this.End }).ConfigureAwait(false); var min = keys[0] >= keys[1] ? keys[0] : keys[1]; var max = keys[2] <= keys[3] ? keys[2] : keys[3]; if (min >= max) { return(false); // range is empty } // rewrite the initial selectors with the bounded keys this.Begin = KeySelector.FirstGreaterOrEqual(min); this.End = KeySelector.FirstGreaterOrEqual(max); } return(true); }
/// <summary>Create a new pair of key selectors that will select all the keys that start with the specified prefix</summary> public static KeySelectorPair StartsWith(Slice prefix) { var range = KeyRange.StartsWith(prefix); return(new KeySelectorPair( KeySelector.FirstGreaterOrEqual(range.Begin), KeySelector.FirstGreaterOrEqual(range.End) )); }
/// <summary>Asynchronously fetch a new page of results</summary> /// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns> private Task <bool> FetchNextPageAsync() { Contract.Requires(!this.AtEnd); Contract.Requires(this.Iteration >= 0); m_ct.ThrowIfCancellationRequested(); this.Transaction.EnsureCanRead(); this.Iteration++; #if DEBUG_RANGE_PAGING Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync(iter=" + this.Iteration + ") started"); #endif var mode = this.Query.Mode; // select the appropriate streaming mode if purpose is not default switch (m_mode) { case AsyncIterationHint.Iterator: { // the caller is responsible for calling MoveNext(..) and deciding if it wants to continue or not.. mode = FdbStreamingMode.Iterator; break; } case AsyncIterationHint.All: { // we are in a ToList or ForEach, we want to read everything in as few chunks as possible mode = FdbStreamingMode.WantAll; break; } case AsyncIterationHint.Head: { // the caller only expect one (or zero) values mode = FdbStreamingMode.Iterator; break; } } //BUGBUG: mix the custom cancellation token with the transaction, if it is different ! var task = (this.Query.Snapshot ? this.Transaction.Snapshot : this.Transaction) .GetRangeAsync(this.Begin, this.End, this.RemainingCount ?? 0, this.Query.Reversed, this.RemainingSize ?? 0, mode, this.Query.Read, this.Iteration) .Then((result) => { this.Chunk = result.Items; this.RowCount += result.Count; this.HasMore = result.HasMore; // subtract number of row from the remaining allowed if (this.RemainingCount.HasValue) { this.RemainingCount = this.RemainingCount.Value - result.Count; } // subtract size of rows from the remaining allowed if (this.RemainingSize.HasValue) { this.RemainingSize = this.RemainingSize.Value - result.GetSize(); } this.AtEnd = !result.HasMore || (this.RemainingCount.HasValue && this.RemainingCount.Value <= 0) || (this.RemainingSize.HasValue && this.RemainingSize.Value <= 0); if (!this.AtEnd) { // update begin..end so that next call will continue from where we left... if (this.Query.Reversed) { this.End = KeySelector.FirstGreaterOrEqual(result.Last); } else { this.Begin = KeySelector.FirstGreaterThan(result.Last); } } #if DEBUG_RANGE_PAGING Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync() returned " + this.Chunk.Length + " results (" + this.RowCount + " total) " + (hasMore ? " with more to come" : " and has no more data")); #endif if (!result.IsEmpty && this.Transaction != null) { return(Task.FromResult(Publish(result.Items))); } return(Completed().AsTask()); }); // keep track of this operation this.PendingReadTask = task; return(task); }
/// <summary>Estimate the number of keys in the specified range.</summary> /// <param name="db">Database used for the operation</param> /// <param name="beginInclusive">Key defining the beginning of the range</param> /// <param name="endExclusive">Key defining the end of the range</param> /// <param name="onProgress">Optional callback called everytime the count is updated. The first argument is the current count, and the second argument is the last key that was found.</param> /// <param name="cancellationToken">Token used to cancel the operation</param> /// <returns>Number of keys k such that <paramref name="beginInclusive"/> <= k > <paramref name="endExclusive"/></returns> /// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks> public static async Task <long> EstimateCountAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, IProgress <STuple <long, Slice> > onProgress, CancellationToken cancellationToken) { const int INIT_WINDOW_SIZE = 1 << 8; // start at 256 //1024 const int MAX_WINDOW_SIZE = 1 << 13; // never use more than 4096 const int MIN_WINDOW_SIZE = 64; // use range reads when the windows size is smaller than 64 if (db == null) { throw new ArgumentNullException("db"); } if (endExclusive < beginInclusive) { throw new ArgumentException("The end key cannot be less than the begin key", "endExclusive"); } cancellationToken.ThrowIfCancellationRequested(); // To count the number of items in the range, we will scan it using a key selector with an offset equal to our window size // > if the returned key is still inside the range, we add the window size to the counter, and start again from the current key // > if the returned key is outside the range, we reduce the size of the window, and start again from the previous key // > if the returned key is exactly equal to the end of range, OR if the window size was 1, then we stop // Since we don't know in advance if the range contains 1 key or 1 Billion keys, choosing a good value for the window size is critical: // > if it is too small and the range is very large, we will need too many sequential reads and the network latency will quickly add up // > if it is too large and the range is small, we will spend too many times halving the window size until we get the correct value // A few optimizations are possible: // > we could start with a small window size, and then double its size on every full segment (up to a maximum) // > for the last segment, we don't need to wait for a GetKey to complete before issuing the next, so we could split the segment into 4 (or more), do the GetKeyAsync() in parallel, detect the quarter that cross the boundary, and iterate again until the size is small // > once the window size is small enough, we can switch to using GetRange to read the last segment in one shot, instead of iterating with window size 16, 8, 4, 2 and 1 (the wost case being 2^N - 1 items remaning) // note: we make a copy of the keys because the operation could take a long time and the key's could prevent a potentially large underlying buffer from being GCed var cursor = beginInclusive.Memoize(); var end = endExclusive.Memoize(); using (var tr = db.BeginReadOnlyTransaction(cancellationToken)) { #if TRACE_COUNTING tr.Annotate("Estimating number of keys in range {0}", KeyRange.Create(beginInclusive, endExclusive)); #endif tr.SetOption(FdbTransactionOption.ReadYourWritesDisable); // start looking for the first key in the range cursor = await tr.Snapshot.GetKeyAsync(KeySelector.FirstGreaterOrEqual(cursor)).ConfigureAwait(false); if (cursor >= end) { // the range is empty ! return(0); } // we already have seen one key, so add it to the count #if TRACE_COUNTING int iter = 1; #endif long counter = 1; // start with a medium-sized window int windowSize = INIT_WINDOW_SIZE; bool last = false; while (cursor < end) { Contract.Assert(windowSize > 0); var selector = KeySelector.FirstGreaterOrEqual(cursor) + windowSize; Slice next = Slice.Nil; FdbException error = null; try { next = await tr.Snapshot.GetKeyAsync(selector).ConfigureAwait(false); #if TRACE_COUNTING ++iter; #endif } catch (FdbException e) { error = e; } if (error != null) { // => from this point, the count returned will not be transactionally accurate if (error.Code == FdbError.PastVersion) { // the transaction used up its time window tr.Reset(); } else { // check to see if we can continue... await tr.OnErrorAsync(error.Code).ConfigureAwait(false); } // retry tr.SetOption(FdbTransactionOption.ReadYourWritesDisable); continue; } //BUGBUG: GetKey(...) always truncate the result to \xFF if the selected key would be past the end, // so we need to fall back immediately to the binary search and/or get_range if next == \xFF if (next > end) { // we have reached past the end, switch to binary search last = true; // if window size is already 1, then we have counted everything (the range.End key does not exist in the db) if (windowSize == 1) { break; } if (windowSize <= MIN_WINDOW_SIZE) { // The window is small enough to switch to reading for counting (will be faster than binary search) #if TRACE_COUNTING tr.Annotate("Switch to reading all items (window size = {0})", windowSize); #endif // Count the keys by reading them. Also, we know that there can not be more than windowSize - 1 remaining int n = await tr.Snapshot .GetRange( KeySelector.FirstGreaterThan(cursor), // cursor has already been counted once KeySelector.FirstGreaterOrEqual(end), new FdbRangeOptions() { Limit = windowSize - 1 } ) .CountAsync() .ConfigureAwait(false); counter += n; if (onProgress != null) { onProgress.Report(STuple.Create(counter, end)); } #if TRACE_COUNTING ++iter; #endif break; } windowSize >>= 1; continue; } // the range is not finished, advance the cursor counter += windowSize; cursor = next; if (onProgress != null) { onProgress.Report(STuple.Create(counter, cursor)); } if (!last) { // double the size of the window if we are not in the last segment windowSize = Math.Min(windowSize << 1, MAX_WINDOW_SIZE); } } #if TRACE_COUNTING tr.Annotate("Found {0} keys in {1} iterations", counter, iter); #endif return(counter); } }