/// <summary>Asynchronously fetch a new page of results</summary>
            /// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
            private Task <bool> FetchNextPageAsync()
            {
                Contract.Requires(!this.AtEnd);
                Contract.Requires(this.Iteration >= 0);

                m_ct.ThrowIfCancellationRequested();
                this.Transaction.EnsureCanRead();

                this.Iteration++;

#if DEBUG_RANGE_PAGING
                Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync(iter=" + this.Iteration + ") started");
#endif

                var mode = this.Query.Mode;
                // select the appropriate streaming mode if purpose is not default
                switch (m_mode)
                {
                case AsyncIterationHint.Iterator:
                {
                    // the caller is responsible for calling MoveNext(..) and deciding if it wants to continue or not..
                    mode = FdbStreamingMode.Iterator;
                    break;
                }

                case AsyncIterationHint.All:
                {
                    // we are in a ToList or ForEach, we want to read everything in as few chunks as possible
                    mode = FdbStreamingMode.WantAll;
                    break;
                }

                case AsyncIterationHint.Head:
                {
                    // the caller only expect one (or zero) values
                    mode = FdbStreamingMode.Iterator;
                    break;
                }
                }

                //BUGBUG: mix the custom cancellation token with the transaction, if it is different !
                var task = (this.Query.Snapshot ? this.Transaction.Snapshot : this.Transaction)
                           .GetRangeAsync(this.Begin, this.End, this.RemainingCount ?? 0, this.Query.Reversed, this.RemainingSize ?? 0, mode, this.Query.Read, this.Iteration)
                           .Then((result) =>
                {
                    this.Chunk     = result.Items;
                    this.RowCount += result.Count;
                    this.HasMore   = result.HasMore;
                    // subtract number of row from the remaining allowed
                    if (this.RemainingCount.HasValue)
                    {
                        this.RemainingCount = this.RemainingCount.Value - result.Count;
                    }
                    // subtract size of rows from the remaining allowed
                    if (this.RemainingSize.HasValue)
                    {
                        this.RemainingSize = this.RemainingSize.Value - result.GetSize();
                    }

                    this.AtEnd = !result.HasMore || (this.RemainingCount.HasValue && this.RemainingCount.Value <= 0) || (this.RemainingSize.HasValue && this.RemainingSize.Value <= 0);

                    if (!this.AtEnd)
                    {                             // update begin..end so that next call will continue from where we left...
                        if (this.Query.Reversed)
                        {
                            this.End = KeySelector.FirstGreaterOrEqual(result.Last);
                        }
                        else
                        {
                            this.Begin = KeySelector.FirstGreaterThan(result.Last);
                        }
                    }
#if DEBUG_RANGE_PAGING
                    Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync() returned " + this.Chunk.Length + " results (" + this.RowCount + " total) " + (hasMore ? " with more to come" : " and has no more data"));
#endif
                    if (!result.IsEmpty && this.Transaction != null)
                    {
                        return(Task.FromResult(Publish(result.Items)));
                    }
                    return(Completed().AsTask());
                });

                // keep track of this operation
                this.PendingReadTask = task;
                return(task);
            }
            /// <summary>Estimate the number of keys in the specified range.</summary>
            /// <param name="db">Database used for the operation</param>
            /// <param name="beginInclusive">Key defining the beginning of the range</param>
            /// <param name="endExclusive">Key defining the end of the range</param>
            /// <param name="onProgress">Optional callback called everytime the count is updated. The first argument is the current count, and the second argument is the last key that was found.</param>
            /// <param name="cancellationToken">Token used to cancel the operation</param>
            /// <returns>Number of keys k such that <paramref name="beginInclusive"/> &lt;= k &gt; <paramref name="endExclusive"/></returns>
            /// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks>
            public static async Task <long> EstimateCountAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, IProgress <STuple <long, Slice> > onProgress, CancellationToken cancellationToken)
            {
                const int INIT_WINDOW_SIZE = 1 << 8;            // start at 256 //1024
                const int MAX_WINDOW_SIZE  = 1 << 13;           // never use more than 4096
                const int MIN_WINDOW_SIZE  = 64;                // use range reads when the windows size is smaller than 64

                if (db == null)
                {
                    throw new ArgumentNullException("db");
                }
                if (endExclusive < beginInclusive)
                {
                    throw new ArgumentException("The end key cannot be less than the begin key", "endExclusive");
                }

                cancellationToken.ThrowIfCancellationRequested();

                // To count the number of items in the range, we will scan it using a key selector with an offset equal to our window size
                // > if the returned key is still inside the range, we add the window size to the counter, and start again from the current key
                // > if the returned key is outside the range, we reduce the size of the window, and start again from the previous key
                // > if the returned key is exactly equal to the end of range, OR if the window size was 1, then we stop

                // Since we don't know in advance if the range contains 1 key or 1 Billion keys, choosing a good value for the window size is critical:
                // > if it is too small and the range is very large, we will need too many sequential reads and the network latency will quickly add up
                // > if it is too large and the range is small, we will spend too many times halving the window size until we get the correct value

                // A few optimizations are possible:
                // > we could start with a small window size, and then double its size on every full segment (up to a maximum)
                // > for the last segment, we don't need to wait for a GetKey to complete before issuing the next, so we could split the segment into 4 (or more), do the GetKeyAsync() in parallel, detect the quarter that cross the boundary, and iterate again until the size is small
                // > once the window size is small enough, we can switch to using GetRange to read the last segment in one shot, instead of iterating with window size 16, 8, 4, 2 and 1 (the wost case being 2^N - 1 items remaning)

                // note: we make a copy of the keys because the operation could take a long time and the key's could prevent a potentially large underlying buffer from being GCed
                var cursor = beginInclusive.Memoize();
                var end    = endExclusive.Memoize();

                using (var tr = db.BeginReadOnlyTransaction(cancellationToken))
                {
#if TRACE_COUNTING
                    tr.Annotate("Estimating number of keys in range {0}", KeyRange.Create(beginInclusive, endExclusive));
#endif

                    tr.SetOption(FdbTransactionOption.ReadYourWritesDisable);

                    // start looking for the first key in the range
                    cursor = await tr.Snapshot.GetKeyAsync(KeySelector.FirstGreaterOrEqual(cursor)).ConfigureAwait(false);

                    if (cursor >= end)
                    {                     // the range is empty !
                        return(0);
                    }

                    // we already have seen one key, so add it to the count
#if TRACE_COUNTING
                    int iter = 1;
#endif
                    long counter = 1;
                    // start with a medium-sized window
                    int  windowSize = INIT_WINDOW_SIZE;
                    bool last       = false;

                    while (cursor < end)
                    {
                        Contract.Assert(windowSize > 0);

                        var          selector = KeySelector.FirstGreaterOrEqual(cursor) + windowSize;
                        Slice        next     = Slice.Nil;
                        FdbException error    = null;
                        try
                        {
                            next = await tr.Snapshot.GetKeyAsync(selector).ConfigureAwait(false);

#if TRACE_COUNTING
                            ++iter;
#endif
                        }
                        catch (FdbException e)
                        {
                            error = e;
                        }

                        if (error != null)
                        {
                            // => from this point, the count returned will not be transactionally accurate
                            if (error.Code == FdbError.PastVersion)
                            {                             // the transaction used up its time window
                                tr.Reset();
                            }
                            else
                            {                             // check to see if we can continue...
                                await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
                            }
                            // retry
                            tr.SetOption(FdbTransactionOption.ReadYourWritesDisable);
                            continue;
                        }

                        //BUGBUG: GetKey(...) always truncate the result to \xFF if the selected key would be past the end,
                        // so we need to fall back immediately to the binary search and/or get_range if next == \xFF

                        if (next > end)
                        {                         // we have reached past the end, switch to binary search
                            last = true;

                            // if window size is already 1, then we have counted everything (the range.End key does not exist in the db)
                            if (windowSize == 1)
                            {
                                break;
                            }

                            if (windowSize <= MIN_WINDOW_SIZE)
                            {                             // The window is small enough to switch to reading for counting (will be faster than binary search)
#if TRACE_COUNTING
                                tr.Annotate("Switch to reading all items (window size = {0})", windowSize);
#endif

                                // Count the keys by reading them. Also, we know that there can not be more than windowSize - 1 remaining
                                int n = await tr.Snapshot
                                        .GetRange(
                                    KeySelector.FirstGreaterThan(cursor),                                             // cursor has already been counted once
                                    KeySelector.FirstGreaterOrEqual(end),
                                    new FdbRangeOptions()
                                {
                                    Limit = windowSize - 1
                                }
                                    )
                                        .CountAsync()
                                        .ConfigureAwait(false);

                                counter += n;
                                if (onProgress != null)
                                {
                                    onProgress.Report(STuple.Create(counter, end));
                                }
#if TRACE_COUNTING
                                ++iter;
#endif
                                break;
                            }

                            windowSize >>= 1;
                            continue;
                        }

                        // the range is not finished, advance the cursor
                        counter += windowSize;
                        cursor   = next;
                        if (onProgress != null)
                        {
                            onProgress.Report(STuple.Create(counter, cursor));
                        }

                        if (!last)
                        {                         // double the size of the window if we are not in the last segment
                            windowSize = Math.Min(windowSize << 1, MAX_WINDOW_SIZE);
                        }
                    }
#if TRACE_COUNTING
                    tr.Annotate("Found {0} keys in {1} iterations", counter, iter);
#endif
                    return(counter);
                }
            }