/// <summary>Execute a retry loop on this context</summary> internal static async Task ExecuteInternal([NotNull] IFdbDatabase db, [NotNull] FdbOperationContext context, [NotNull] Delegate handler, Delegate onDone) { Contract.Requires(db != null && context != null && handler != null); Contract.Requires(context.Shared); if (context.Abort) { throw new InvalidOperationException("Operation context has already been aborted or disposed"); } try { // make sure to reset everything (in case a context is reused multiple times) context.Committed = false; context.Retries = 0; context.BaseDuration = TimeSpan.Zero; context.StartedUtc = DateTime.UtcNow; context.Clock.Start(); //note: we start the clock immediately, but the transaction's 5 seconde max lifetime is actually measured from the first read operation (Get, GetRange, GetReadVersion, etc...) // => algorithms that monitor the elapsed duration to rate limit themselves may think that the trans is older than it really is... // => we would need to plug into the transaction handler itself to be notified when exactly a read op starts... using (var trans = db.BeginTransaction(context.Mode, CancellationToken.None, context)) { while (!context.Committed && !context.Cancellation.IsCancellationRequested) { FdbException e = null; try { // call the user provided lambda if (handler is Func <IFdbTransaction, Task> funcWritable) { await funcWritable(trans).ConfigureAwait(false); } else if (handler is Action <IFdbTransaction> action) { action(trans); } else if (handler is Func <IFdbReadOnlyTransaction, Task> funcReadOnly) { await funcReadOnly(trans).ConfigureAwait(false); } else { throw new NotSupportedException($"Cannot execute handlers of type {handler.GetType().Name}"); } if (context.Abort) { break; } if (!trans.IsReadOnly) { // commit the transaction await trans.CommitAsync().ConfigureAwait(false); } // we are done context.Committed = true; if (onDone != null) { if (onDone is Action <IFdbReadOnlyTransaction> action1) { action1(trans); } else if (onDone is Action <IFdbTransaction> action2) { action2(trans); } else if (onDone is Func <IFdbReadOnlyTransaction, Task> func1) { await func1(trans).ConfigureAwait(false); } else if (onDone is Func <IFdbTransaction, Task> func2) { await func2(trans).ConfigureAwait(false); } else { throw new NotSupportedException($"Cannot execute completion handler of type {handler.GetType().Name}"); } } } catch (FdbException x) { //TODO: will be able to await in catch block in C# 6 ! e = x; } if (e != null) { if (Logging.On && Logging.IsVerbose) { Logging.Verbose(String.Format(CultureInfo.InvariantCulture, "fdb: transaction {0} failed with error code {1}", trans.Id, e.Code)); } await trans.OnErrorAsync(e.Code).ConfigureAwait(false); if (Logging.On && Logging.IsVerbose) { Logging.Verbose(String.Format(CultureInfo.InvariantCulture, "fdb: transaction {0} can be safely retried", trans.Id)); } } // update the base time for the next attempt context.BaseDuration = context.ElapsedTotal; if (context.BaseDuration.TotalSeconds >= 10) { //REVIEW: this may not be a goot idea to spam the logs with long running transactions?? if (Logging.On) { Logging.Info(String.Format(CultureInfo.InvariantCulture, "fdb WARNING: long transaction ({0:N1} sec elapsed in transaction lambda function ({1} retries, {2})", context.BaseDuration.TotalSeconds, context.Retries, context.Committed ? "committed" : "not yet committed")); } } context.Retries++; } } context.Cancellation.ThrowIfCancellationRequested(); if (context.Abort) { throw new OperationCanceledException(context.Cancellation); } } finally { context.Clock.Stop(); context.Dispose(); } }
internal static async Task ExecuteInternal([NotNull] IFdbDatabase db, [NotNull] FdbOperationContext context, [NotNull] Delegate handler, Delegate onDone) { Contract.Requires(db != null && context != null && handler != null); Contract.Requires(context.Shared); if (context.Abort) { throw new InvalidOperationException("Operation context has already been aborted or disposed"); } try { context.Committed = false; context.Retries = 0; context.StartedUtc = DateTime.UtcNow; context.Duration.Start(); using (var trans = db.BeginTransaction(context.Mode, CancellationToken.None, context)) { while (!context.Committed && !context.Cancellation.IsCancellationRequested) { FdbException e = null; try { // call the user provided lambda if (handler is Func <IFdbTransaction, Task> ) { await((Func <IFdbTransaction, Task>)handler)(trans).ConfigureAwait(false); } else if (handler is Action <IFdbTransaction> ) { ((Action <IFdbTransaction>)handler)(trans); } else if (handler is Func <IFdbReadOnlyTransaction, Task> ) { await((Func <IFdbReadOnlyTransaction, Task>)handler)(trans).ConfigureAwait(false); } else { throw new NotSupportedException(String.Format("Cannot execute handlers of type {0}", handler.GetType().Name)); } if (context.Abort) { break; } if (!trans.IsReadOnly) { // commit the transaction await trans.CommitAsync().ConfigureAwait(false); } // we are done context.Committed = true; if (onDone != null) { if (onDone is Action <IFdbReadOnlyTransaction> ) { ((Action <IFdbReadOnlyTransaction>)onDone)(trans); } else if (onDone is Action <IFdbTransaction> ) { ((Action <IFdbTransaction>)onDone)(trans); } else if (onDone is Func <IFdbReadOnlyTransaction, Task> ) { await((Func <IFdbReadOnlyTransaction, Task>)onDone)(trans).ConfigureAwait(false); } else if (onDone is Func <IFdbTransaction, Task> ) { await((Func <IFdbTransaction, Task>)onDone)(trans).ConfigureAwait(false); } else { throw new NotSupportedException(String.Format("Cannot execute completion handler of type {0}", handler.GetType().Name)); } } } catch (FdbException x) { e = x; } if (e != null) { if (Logging.On && Logging.IsVerbose) { Logging.Verbose(String.Format(CultureInfo.InvariantCulture, "fdb: transaction {0} failed with error code {1}", trans.Id, e.Code)); } await trans.OnErrorAsync(e.Code).ConfigureAwait(false); if (Logging.On && Logging.IsVerbose) { Logging.Verbose(String.Format(CultureInfo.InvariantCulture, "fdb: transaction {0} can be safely retried", trans.Id)); } } if (context.Duration.Elapsed.TotalSeconds >= 1) { if (Logging.On) { Logging.Info(String.Format(CultureInfo.InvariantCulture, "fdb WARNING: long transaction ({0:N1} sec elapsed in transaction lambda function ({1} retries, {2})", context.Duration.Elapsed.TotalSeconds, context.Retries, context.Committed ? "committed" : "not yet committed")); } } context.Retries++; } } context.Cancellation.ThrowIfCancellationRequested(); if (context.Abort) { throw new OperationCanceledException(context.Cancellation); } } finally { context.Duration.Stop(); context.Dispose(); } }
private static async Task <List <Slice> > GetBoundaryKeysInternalAsync([NotNull] IFdbReadOnlyTransaction trans, Slice begin, Slice end) { Contract.Requires(trans != null && end >= begin); #if TRACE_COUNTING trans.Annotate("Get boundary keys in range {0}", KeyRange.Create(begin, end)); #endif trans.WithReadAccessToSystemKeys(); var results = new List <Slice>(); int iterations = 0; var options = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll }; while (begin < end) { FdbException error = null; Slice lastBegin = begin; try { var chunk = await trans.Snapshot.GetRangeAsync(KeyServers + begin, KeyServers + end, options, iterations).ConfigureAwait(false); ++iterations; if (chunk.Count > 0) { foreach (var kvp in chunk) { results.Add(kvp.Key.Substring(KeyServers.Count)); } begin = chunk.Last.Substring(KeyServers.Count) + 0; } if (!chunk.HasMore) { begin = end; } } catch (FdbException e) { error = e; } if (error != null) { if (error.Code == FdbError.PastVersion && begin != lastBegin) { // if we get a PastVersion and *something* has happened, then we are no longer transactional trans.Reset(); } else { await trans.OnErrorAsync(error.Code).ConfigureAwait(false); } iterations = 0; trans.WithReadAccessToSystemKeys(); } } #if TRACE_COUNTING if (results.Count == 0) { trans.Annotate("There is no chunk boundary in range {0}", KeyRange.Create(begin, end)); } else { trans.Annotate("Found {0} boundaries in {1} iteration(s)", results.Count, iterations); } #endif return(results); }
/// <summary>Estimate the number of keys in the specified range.</summary> /// <param name="db">Database used for the operation</param> /// <param name="beginInclusive">Key defining the beginning of the range</param> /// <param name="endExclusive">Key defining the end of the range</param> /// <param name="onProgress">Optional callback called everytime the count is updated. The first argument is the current count, and the second argument is the last key that was found.</param> /// <param name="cancellationToken">Token used to cancel the operation</param> /// <returns>Number of keys k such that <paramref name="beginInclusive"/> <= k > <paramref name="endExclusive"/></returns> /// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks> public static async Task <long> EstimateCountAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, IProgress <FdbTuple <long, Slice> > onProgress, CancellationToken cancellationToken) { const int INIT_WINDOW_SIZE = 1 << 8; // start at 256 //1024 const int MAX_WINDOW_SIZE = 1 << 13; // never use more than 4096 const int MIN_WINDOW_SIZE = 64; // use range reads when the windows size is smaller than 64 if (db == null) { throw new ArgumentNullException("db"); } if (endExclusive < beginInclusive) { throw new ArgumentException("The end key cannot be less than the begin key", "endExclusive"); } cancellationToken.ThrowIfCancellationRequested(); // To count the number of items in the range, we will scan it using a key selector with an offset equal to our window size // > if the returned key is still inside the range, we add the window size to the counter, and start again from the current key // > if the returned key is outside the range, we reduce the size of the window, and start again from the previous key // > if the returned key is exactly equal to the end of range, OR if the window size was 1, then we stop // Since we don't know in advance if the range contains 1 key or 1 Billion keys, choosing a good value for the window size is critical: // > if it is too small and the range is very large, we will need too many sequential reads and the network latency will quickly add up // > if it is too large and the range is small, we will spend too many times halving the window size until we get the correct value // A few optimizations are possible: // > we could start with a small window size, and then double its size on every full segment (up to a maximum) // > for the last segment, we don't need to wait for a GetKey to complete before issuing the next, so we could split the segment into 4 (or more), do the GetKeyAsync() in parallel, detect the quarter that cross the boundary, and iterate again until the size is small // > once the window size is small enough, we can switch to using GetRange to read the last segment in one shot, instead of iterating with window size 16, 8, 4, 2 and 1 (the wost case being 2^N - 1 items remaning) // note: we make a copy of the keys because the operation could take a long time and the key's could prevent a potentially large underlying buffer from being GCed var cursor = beginInclusive.Memoize(); var end = endExclusive.Memoize(); using (var tr = db.BeginReadOnlyTransaction(cancellationToken)) { #if TRACE_COUNTING tr.Annotate("Estimating number of keys in range {0}", FdbKeyRange.Create(beginInclusive, endExclusive)); #endif tr.SetOption(FdbTransactionOption.ReadYourWritesDisable); // start looking for the first key in the range cursor = await tr.Snapshot.GetKeyAsync(FdbKeySelector.FirstGreaterOrEqual(cursor)).ConfigureAwait(false); if (cursor >= end) { // the range is empty ! return(0); } // we already have seen one key, so add it to the count #if TRACE_COUNTING int iter = 1; #endif long counter = 1; // start with a medium-sized window int windowSize = INIT_WINDOW_SIZE; bool last = false; while (cursor < end) { Contract.Assert(windowSize > 0); var selector = FdbKeySelector.FirstGreaterOrEqual(cursor) + windowSize; Slice next = Slice.Nil; FdbException error = null; try { next = await tr.Snapshot.GetKeyAsync(selector).ConfigureAwait(false); #if TRACE_COUNTING ++iter; #endif } catch (FdbException e) { error = e; } if (error != null) { // => from this point, the count returned will not be transactionally accurate if (error.Code == FdbError.PastVersion) { // the transaction used up its time window tr.Reset(); } else { // check to see if we can continue... await tr.OnErrorAsync(error.Code).ConfigureAwait(false); } // retry tr.SetOption(FdbTransactionOption.ReadYourWritesDisable); continue; } //BUGBUG: GetKey(...) always truncate the result to \xFF if the selected key would be past the end, // so we need to fall back immediately to the binary search and/or get_range if next == \xFF if (next > end) { // we have reached past the end, switch to binary search last = true; // if window size is already 1, then we have counted everything (the range.End key does not exist in the db) if (windowSize == 1) { break; } if (windowSize <= MIN_WINDOW_SIZE) { // The window is small enough to switch to reading for counting (will be faster than binary search) #if TRACE_COUNTING tr.Annotate("Switch to reading all items (window size = {0})", windowSize); #endif // Count the keys by reading them. Also, we know that there can not be more than windowSize - 1 remaining int n = await tr.Snapshot .GetRange( FdbKeySelector.FirstGreaterThan(cursor), // cursor has already been counted once FdbKeySelector.FirstGreaterOrEqual(end), new FdbRangeOptions() { Limit = windowSize - 1 } ) .CountAsync() .ConfigureAwait(false); counter += n; if (onProgress != null) { onProgress.Report(FdbTuple.Create(counter, end)); } #if TRACE_COUNTING ++iter; #endif break; } windowSize >>= 1; continue; } // the range is not finished, advance the cursor counter += windowSize; cursor = next; if (onProgress != null) { onProgress.Report(FdbTuple.Create(counter, cursor)); } if (!last) { // double the size of the window if we are not in the last segment windowSize = Math.Min(windowSize << 1, MAX_WINDOW_SIZE); } } #if TRACE_COUNTING tr.Annotate("Found {0} keys in {1} iterations", counter, iter); #endif return(counter); } }