/// <summary>Factory method for a pair of key selectors</summary>
public static FdbKeySelectorPair Create(FdbKeySelector beginInclusive, FdbKeySelector endExclusive)
{
return(new FdbKeySelectorPair(
beginInclusive,
endExclusive
));
}
/// <summary>Create a new pair of key selectors using FIRST_GREATER_OR_EQUAL on both keys</summary>
public static FdbKeySelectorPair Create(FdbKeyRange range)
{
return(new FdbKeySelectorPair(
FdbKeySelector.FirstGreaterOrEqual(range.Begin),
FdbKeySelector.FirstGreaterOrEqual(range.End)
));
}
/// <summary>Create a new pair of key selectors using FIRST_GREATER_OR_EQUAL on both keys</summary>
public static FdbKeySelectorPair Create(Slice begin, Slice end)
{
return(new FdbKeySelectorPair(
FdbKeySelector.FirstGreaterOrEqual(begin),
FdbKeySelector.FirstGreaterOrEqual(end)
));
}
protected override async Task <bool> OnFirstAsync(CancellationToken cancellationToken)
{
this.Remaining = this.Query.Limit;
this.Begin = this.Query.Begin;
this.End = this.Query.End;
if (this.Remaining == 0)
{
// we can safely optimize this case by not doing any query, because it should not have any impact on conflict resolutions.
// => The result of 'query.Take(0)' will not change even if someone adds/remove to the range
// => The result of 'query.Take(X)' where X would be computed from reads in the db, and be equal to 0, would conflict because of those reads anyway.
return(false);
}
var bounds = this.Query.OriginalRange;
// if the original range has been changed, we need to ensure that the current begin/end to not overflow:
if (this.Begin != bounds.Begin || this.End != bounds.End)
{
//TODO: find a better way to do this!
var keys = await this.Transaction.GetKeysAsync(new[] { bounds.Begin, this.Begin, bounds.End, this.End }).ConfigureAwait(false);
var min = keys[0] >= keys[1] ? keys[0] : keys[1];
var max = keys[2] <= keys[3] ? keys[2] : keys[3];
if (min >= max)
{
return(false); // range is empty
}
// rewrite the initial selectors with the bounded keys
this.Begin = FdbKeySelector.FirstGreaterOrEqual(min);
this.End = FdbKeySelector.FirstGreaterOrEqual(max);
}
return(true);
}
/// <summary>Resolve a single key selector from the database, using a dedicated transaction.</summary>
/// <param name="db">Database instance</param>
/// <remarks>
/// Use this method only if you intend to perform a single operation inside your execution context (ex: HTTP request).
/// If you need to combine multiple read or write operations, consider using on of the multiple <see cref="IFdbReadOnlyRetryable.ReadAsync"/> or <see cref="IFdbRetryable.ReadWriteAsync"/> overrides.
/// </remarks>
public static Task <Slice> GetKeyAsync(this IFdbReadOnlyRetryable db, FdbKeySelector keySelector, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
return(db.ReadAsync((tr) => tr.GetKeyAsync(keySelector), cancellationToken));
}
/// <summary>Create a new pair of key selectors that will select all the keys that start with the specified prefix</summary>
public static FdbKeySelectorPair StartsWith(Slice prefix)
{
var range = FdbKeyRange.StartsWith(prefix);
return(new FdbKeySelectorPair(
FdbKeySelector.FirstGreaterOrEqual(range.Begin),
FdbKeySelector.FirstGreaterOrEqual(range.End)
));
}
/// <summary>Construct a query with a set of initial settings</summary>
internal FdbRangeQuery([NotNull] IFdbReadOnlyTransaction transaction, FdbKeySelector begin, FdbKeySelector end, [NotNull] Func <KeyValuePair <Slice, Slice>, T> transform, bool snapshot, FdbRangeOptions options)
{
Contract.Requires(transaction != null && transform != null);
this.Transaction = transaction;
this.Begin = begin;
this.End = end;
this.Transform = transform;
this.Snapshot = snapshot;
this.Options = options ?? new FdbRangeOptions();
this.OriginalRange = FdbKeySelectorPair.Create(begin, end);
}
/// <summary>Create a new pair of key selectors using FIRST_GREATER_OR_EQUAL on both keys</summary>
public static FdbKeySelectorPair Create <TKey>(TKey begin, TKey end)
where TKey : IFdbKey
{
if (begin == null)
{
throw new ArgumentNullException("begin");
}
if (end == null)
{
throw new ArgumentNullException("end");
}
return(new FdbKeySelectorPair(
FdbKeySelector.FirstGreaterOrEqual(begin.ToFoundationDbKey()),
FdbKeySelector.FirstGreaterOrEqual(end.ToFoundationDbKey())
));
}
internal FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRangeCore(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, bool snapshot)
{
this.Database.EnsureKeyIsValid(begin.Key);
this.Database.EnsureKeyIsValid(end.Key, endExclusive: true);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
#if DEBUG
if (Logging.On && Logging.IsVerbose)
{
Logging.Verbose(this, "GetRangeCore", String.Format("Getting range '{0} <= x < {1}'", begin.ToString(), end.ToString()));
}
#endif
return(new FdbRangeQuery <KeyValuePair <Slice, Slice> >(this, begin, end, TaskHelpers.Cache <KeyValuePair <Slice, Slice> > .Identity, snapshot, options));
}
public Task <FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options, int iteration)
{
EnsureCanRead();
m_parent.m_database.EnsureKeyIsValid(beginInclusive.Key);
m_parent.m_database.EnsureKeyIsValid(endExclusive.Key);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
// The iteration value is only needed when in iterator mode, but then it should start from 1
if (iteration == 0)
{
iteration = 1;
}
return(m_parent.m_handler.GetRangeAsync(beginInclusive, endExclusive, options, iteration, snapshot: true, cancellationToken: m_parent.m_cancellation));
}
public async Task <Slice> GetKeyAsync(FdbKeySelector selector)
{
EnsureCanRead();
m_parent.m_database.EnsureKeyIsValid(selector.Key);
#if DEBUG
if (Logging.On && Logging.IsVerbose)
{
Logging.Verbose(this, "GetKeyAsync", String.Format("Getting key '{0}'", selector.ToString()));
}
#endif
var key = await m_parent.m_handler.GetKeyAsync(selector, snapshot : true, cancellationToken : m_parent.m_cancellation).ConfigureAwait(false);
// don't forget to truncate keys that would fall outside of the database's globalspace !
return(m_parent.m_database.BoundCheck(key));
}
/// <summary>Read a single page of a range query from the database, using a dedicated transaction.</summary>
/// <param name="db">Database instance</param>
/// <remarks>
/// Use this method only if you intend to perform a single operation inside your execution context (ex: HTTP request).
/// If you need to combine multiple read or write operations, consider using on of the multiple <see cref="IFdbReadOnlyRetryable.ReadAsync"/> or <see cref="IFdbRetryable.ReadWriteAsync"/> overrides.
/// </remarks>
public static Task <FdbRangeChunk> GetRangeAsync(this IFdbReadOnlyRetryable db, FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options, int iteration, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
return(db.ReadAsync((tr) => tr.GetRangeAsync(beginInclusive, endExclusive, options, iteration), cancellationToken));
}
private FdbKeySelector m_end; //PERF: readonly struct
/// <summary>Create a new pair of key selectors</summary>
/// <param name="beginInclusive">Selector for key from which to start iterating</param>
/// <param name="endExclusive">Selector for key where to stop iterating</param>
public FdbKeySelectorPair(FdbKeySelector beginInclusive, FdbKeySelector endExclusive)
{
m_begin = beginInclusive;
m_end = endExclusive;
}
/// <summary>Estimate the number of keys in the specified range.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="beginInclusive">Key defining the beginning of the range</param>
/// <param name="endExclusive">Key defining the end of the range</param>
/// <param name="onProgress">Optional callback called everytime the count is updated. The first argument is the current count, and the second argument is the last key that was found.</param>
/// <param name="cancellationToken">Token used to cancel the operation</param>
/// <returns>Number of keys k such that <paramref name="beginInclusive"/> <= k > <paramref name="endExclusive"/></returns>
/// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks>
public static async Task <long> EstimateCountAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, IProgress <FdbTuple <long, Slice> > onProgress, CancellationToken cancellationToken)
{
const int INIT_WINDOW_SIZE = 1 << 8; // start at 256 //1024
const int MAX_WINDOW_SIZE = 1 << 13; // never use more than 4096
const int MIN_WINDOW_SIZE = 64; // use range reads when the windows size is smaller than 64
if (db == null)
{
throw new ArgumentNullException("db");
}
if (endExclusive < beginInclusive)
{
throw new ArgumentException("The end key cannot be less than the begin key", "endExclusive");
}
cancellationToken.ThrowIfCancellationRequested();
// To count the number of items in the range, we will scan it using a key selector with an offset equal to our window size
// > if the returned key is still inside the range, we add the window size to the counter, and start again from the current key
// > if the returned key is outside the range, we reduce the size of the window, and start again from the previous key
// > if the returned key is exactly equal to the end of range, OR if the window size was 1, then we stop
// Since we don't know in advance if the range contains 1 key or 1 Billion keys, choosing a good value for the window size is critical:
// > if it is too small and the range is very large, we will need too many sequential reads and the network latency will quickly add up
// > if it is too large and the range is small, we will spend too many times halving the window size until we get the correct value
// A few optimizations are possible:
// > we could start with a small window size, and then double its size on every full segment (up to a maximum)
// > for the last segment, we don't need to wait for a GetKey to complete before issuing the next, so we could split the segment into 4 (or more), do the GetKeyAsync() in parallel, detect the quarter that cross the boundary, and iterate again until the size is small
// > once the window size is small enough, we can switch to using GetRange to read the last segment in one shot, instead of iterating with window size 16, 8, 4, 2 and 1 (the wost case being 2^N - 1 items remaning)
// note: we make a copy of the keys because the operation could take a long time and the key's could prevent a potentially large underlying buffer from being GCed
var cursor = beginInclusive.Memoize();
var end = endExclusive.Memoize();
using (var tr = db.BeginReadOnlyTransaction(cancellationToken))
{
#if TRACE_COUNTING
tr.Annotate("Estimating number of keys in range {0}", FdbKeyRange.Create(beginInclusive, endExclusive));
#endif
tr.SetOption(FdbTransactionOption.ReadYourWritesDisable);
// start looking for the first key in the range
cursor = await tr.Snapshot.GetKeyAsync(FdbKeySelector.FirstGreaterOrEqual(cursor)).ConfigureAwait(false);
if (cursor >= end)
{ // the range is empty !
return(0);
}
// we already have seen one key, so add it to the count
#if TRACE_COUNTING
int iter = 1;
#endif
long counter = 1;
// start with a medium-sized window
int windowSize = INIT_WINDOW_SIZE;
bool last = false;
while (cursor < end)
{
Contract.Assert(windowSize > 0);
var selector = FdbKeySelector.FirstGreaterOrEqual(cursor) + windowSize;
Slice next = Slice.Nil;
FdbException error = null;
try
{
next = await tr.Snapshot.GetKeyAsync(selector).ConfigureAwait(false);
#if TRACE_COUNTING
++iter;
#endif
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
// => from this point, the count returned will not be transactionally accurate
if (error.Code == FdbError.PastVersion)
{ // the transaction used up its time window
tr.Reset();
}
else
{ // check to see if we can continue...
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
}
// retry
tr.SetOption(FdbTransactionOption.ReadYourWritesDisable);
continue;
}
//BUGBUG: GetKey(...) always truncate the result to \xFF if the selected key would be past the end,
// so we need to fall back immediately to the binary search and/or get_range if next == \xFF
if (next > end)
{ // we have reached past the end, switch to binary search
last = true;
// if window size is already 1, then we have counted everything (the range.End key does not exist in the db)
if (windowSize == 1)
{
break;
}
if (windowSize <= MIN_WINDOW_SIZE)
{ // The window is small enough to switch to reading for counting (will be faster than binary search)
#if TRACE_COUNTING
tr.Annotate("Switch to reading all items (window size = {0})", windowSize);
#endif
// Count the keys by reading them. Also, we know that there can not be more than windowSize - 1 remaining
int n = await tr.Snapshot
.GetRange(
FdbKeySelector.FirstGreaterThan(cursor), // cursor has already been counted once
FdbKeySelector.FirstGreaterOrEqual(end),
new FdbRangeOptions()
{
Limit = windowSize - 1
}
)
.CountAsync()
.ConfigureAwait(false);
counter += n;
if (onProgress != null)
{
onProgress.Report(FdbTuple.Create(counter, end));
}
#if TRACE_COUNTING
++iter;
#endif
break;
}
windowSize >>= 1;
continue;
}
// the range is not finished, advance the cursor
counter += windowSize;
cursor = next;
if (onProgress != null)
{
onProgress.Report(FdbTuple.Create(counter, cursor));
}
if (!last)
{ // double the size of the window if we are not in the last segment
windowSize = Math.Min(windowSize << 1, MAX_WINDOW_SIZE);
}
}
#if TRACE_COUNTING
tr.Annotate("Found {0} keys in {1} iterations", counter, iter);
#endif
return(counter);
}
}
/// <summary>Asynchronously fetch a new page of results</summary>
/// <param name="cancellationToken"></param>
/// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
private Task <bool> FetchNextPageAsync(CancellationToken cancellationToken)
{
Contract.Requires(!this.AtEnd);
Contract.Requires(this.Iteration >= 0);
cancellationToken.ThrowIfCancellationRequested();
this.Transaction.EnsureCanRead();
this.Iteration++;
#if DEBUG_RANGE_PAGING
Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync(iter=" + this.Iteration + ") started");
#endif
var options = new FdbRangeOptions
{
Limit = this.Remaining,
TargetBytes = this.Query.TargetBytes,
Mode = this.Query.Mode,
Reverse = this.Query.Reversed
};
// select the appropriate streaming mode if purpose is not default
switch (m_mode)
{
case FdbAsyncMode.Iterator:
{
// the caller is responsible for calling MoveNext(..) and deciding if it wants to continue or not..
options.Mode = FdbStreamingMode.Iterator;
break;
}
case FdbAsyncMode.All:
{
// we are in a ToList or ForEach, we want to read everything in as few chunks as possible
options.Mode = FdbStreamingMode.WantAll;
break;
}
case FdbAsyncMode.Head:
{
// the caller only expect one (or zero) values
options.Mode = FdbStreamingMode.Iterator;
break;
}
}
var tr = this.Transaction;
if (this.Query.Snapshot)
{ // make sure we have the snapshot version !
tr = tr.Snapshot;
}
//BUGBUG: mix the custom cancellation token with the transaction, if it is diffent !
var task = tr
.GetRangeAsync(this.Begin, this.End, options, this.Iteration)
.Then((result) =>
{
this.Chunk = result.Chunk;
this.RowCount += result.Count;
this.HasMore = result.HasMore;
// subtract number of row from the remaining allowed
if (this.Remaining.HasValue)
{
this.Remaining = this.Remaining.Value - result.Count;
}
this.AtEnd = !result.HasMore || (this.Remaining.HasValue && this.Remaining.Value <= 0);
if (!this.AtEnd)
{ // update begin..end so that next call will continue from where we left...
var lastKey = result.Last.Key;
if (this.Query.Reversed)
{
this.End = FdbKeySelector.FirstGreaterOrEqual(lastKey);
}
else
{
this.Begin = FdbKeySelector.FirstGreaterThan(lastKey);
}
}
#if DEBUG_RANGE_PAGING
Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync() returned " + this.Chunk.Length + " results (" + this.RowCount + " total) " + (hasMore ? " with more to come" : " and has no more data"));
#endif
if (!result.IsEmpty && this.Transaction != null)
{
return(Publish(result.Chunk));
}
return(Completed());
});
// keep track of this operation
this.PendingReadTask = task;
return(task);
}
public FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options)
{
EnsureCanRead();
return(m_parent.GetRangeCore(beginInclusive, endExclusive, options, snapshot: true));
}
