private DocumentDbTranslatingReliableQueryProvider(IQueryProvider underlyingProvider, QueryExecutionHandler queryExecutionHandler, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry, params ExpressionVisitor[] visitors)
: base(underlyingProvider, visitors)
{
this.queryExecutionHandler = queryExecutionHandler;
this.enumerationExceptionHandler = enumerationExceptionHandler;
this.feedResponseHandler = feedResponseHandler;
this.maxRetries = maxRetries;
this.maxTime = maxTime;
this.shouldRetry = shouldRetry;
this.mode = Mode.Intercept;
}
public static IQueryable <TElement> InterceptForPagingContinuationOnly <TElement>(IQueryable <TElement> underlyingQuery, QueryExecutionHandler queryExecutionHandler, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
var provider = new DocumentDbTranslatingReliableQueryProvider(underlyingQuery.Provider, queryExecutionHandler, enumerationExceptionHandler, feedResponseHandler, maxRetries, maxTime, shouldRetry, new DocumentDbTranslateExpressionVisitor(typeof(TElement)));
provider.mode = Mode.ResumePaging;
return(provider.CreateQuery <TElement>(underlyingQuery.Expression));
}
internal static async Task <DocumentsPage <R> > GetNextPageWithRetry <R>(IDocumentQuery <R> query, QueryExecutionHandler queryExecutionHandler, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
var context = GetContextForPaging(query);
FeedResponse <R> nextPageResponse = null;
while (nextPageResponse == null)
{
try
{
nextPageResponse = await DocumentDbReliableExecution.ExecuteResultWithRetry(() =>
query.ExecuteNextAsync <R>(),
null,
maxRetries,
maxTime,
shouldRetry);
}
catch (Exception ex)
{
bool handled = enumerationExceptionHandler(context, ex);
if (!handled)
{
feedResponseHandler(context, FeedResponseType.EnumerationAborted, null);
if (ex.Message.StartsWith("Invalid Continuation Token"))
{
throw new DocumentDbNonRetriableResponse("Unable to continue paging, this probably means that the queryable used for GetNextPage did not represent an identical query to that which was used to get the continuation token.", ex);
}
throw;
}
}
}
// lots of interesting info in intermediateResults such as RU usage, etc.
List <R> nextPageResults = nextPageResponse.ToList();
UpdateContext <R>(context, nextPageResponse, JsonConvert.SerializeObject(nextPageResults).Length, query.HasMoreResults);
feedResponseHandler(context, FeedResponseType.PageReceived, new FeedResponseWrapper <R>(nextPageResponse));
if (!query.HasMoreResults)
{
feedResponseHandler(context, FeedResponseType.AfterEnumeration, null);
}
if (!query.HasMoreResults)
{
DeletePagingContext(query);
}
return(new DocumentsPage <R>(nextPageResults, query.HasMoreResults ? nextPageResponse.ResponseContinuation : null));
}
/// <summary>
/// This will execute a DocumentDB FeedResponse method and return the results.
///
/// It handles paging, continuation tokens, and retriable errors such as "too many requests" for you,
/// while streaming query results out in chunks via IEnumerable / yield.
/// </summary>
/// <typeparam name="R"></typeparam>
/// <param name="feedTakingContinuation"></param>
/// <param name="enumerationExceptionHandler"></param>
/// <param name="feedResponseHandler"></param>
/// <param name="maxRetries"></param>
/// <param name="maxTime"></param>
/// <param name="shouldRetry"></param>
/// <returns></returns>
public static IEnumerable <R> StreamFeedWithContinuationAndRetry <R>(Func <string, Task <FeedResponse <R> > > feedTakingContinuation, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
var context = new FeedResponseContext();
feedResponseHandler(context, FeedResponseType.BeforeEnumeration, null);
FeedResponse <R> intermediateResponse = null;
do
{
Task <FeedResponse <R> > t;
try
{
t = Task.Run(async() => await ExecuteResultWithRetry(() =>
feedTakingContinuation(intermediateResponse?.ResponseContinuation),
null,
maxRetries,
maxTime,
shouldRetry));
t.Wait();
}
catch (Exception ex)
{
// note: if an IQueryable is returned to OData, throwing an exception here will cause it to take down w3wp.exe
bool handled = enumerationExceptionHandler(null, ex);
if (!handled)
{
feedResponseHandler(context, FeedResponseType.EnumerationAborted, null);
throw;
}
else
{
break;
}
}
intermediateResponse = t.Result;
// lots of interesting info in intermediateResults such as RU usage, etc.
List <R> intermediateResults = intermediateResponse.ToList();
UpdateContext <R>(context, intermediateResponse, JsonConvert.SerializeObject(intermediateResults).Length, !string.IsNullOrEmpty(intermediateResponse.ResponseContinuation));
feedResponseHandler(context, FeedResponseType.PageReceived, new FeedResponseWrapper <R>(intermediateResponse));
foreach (var result in intermediateResults)
{
yield return(result);
}
} while (!string.IsNullOrEmpty(intermediateResponse.ResponseContinuation));
feedResponseHandler(context, FeedResponseType.AfterEnumeration, null);
}
internal static async Task <DocumentsPage <R> > BeginPagingWithRetry <R>(IDocumentQuery <R> query, QueryExecutionHandler queryExecutionHandler, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
var context = MakeFirstContextForPaging(query);
queryExecutionHandler(context, query.ToString());
feedResponseHandler(context, FeedResponseType.BeforeEnumeration, null);
FeedResponse <R> firstPageResponse = null;
while (firstPageResponse == null)
{
try
{
firstPageResponse = await DocumentDbReliableExecution.ExecuteResultWithRetry(() =>
query.ExecuteNextAsync <R>(),
null,
maxRetries,
maxTime,
shouldRetry);
}
catch (Exception ex)
{
bool handled = enumerationExceptionHandler(context, ex);
if (!handled)
{
feedResponseHandler(context, FeedResponseType.EnumerationAborted, null);
throw;
}
}
}
// lots of interesting info in intermediateResults such as RU usage, etc.
List <R> firstPageResults = firstPageResponse.ToList();
UpdateContext(context, firstPageResponse, JsonConvert.SerializeObject(firstPageResults).Length, query.HasMoreResults);
feedResponseHandler(context, FeedResponseType.PageReceived, new FeedResponseWrapper <R>(firstPageResponse));
if (!query.HasMoreResults)
{
DeletePagingContext(query);
}
return(new DocumentsPage <R>(firstPageResults, query.HasMoreResults ? firstPageResponse.ResponseContinuation : null));
}
/// <summary>
/// This will execute a DocumentDB FeedResponse method and return the results.
///
/// It handles paging, continuation tokens, and retriable errors such as "too many requests" for you,
/// while streaming query results out in chunks via IEnumerable / yield.
/// </summary>
/// <typeparam name="R"></typeparam>
/// <param name="feedTakingContinuation"></param>
/// <param name="enumerationExceptionHandler"></param>
/// <param name="feedResponseHandler"></param>
/// <param name="maxRetries"></param>
/// <param name="maxTime"></param>
/// <param name="shouldRetry"></param>
/// <returns></returns>
public static IEnumerable <R> StreamFeedWithContinuationAndRetry <R>(Func <string, FeedResponse <R> > feedTakingContinuation, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
return(StreamFeedWithContinuationAndRetry <R>((continuation) => Task <R> .Run(() => feedTakingContinuation(continuation)), enumerationExceptionHandler, feedResponseHandler, maxRetries, maxTime, shouldRetry));
}
/// <summary>
/// This will execute a DocumentDB FeedResponse method return the results.
///
/// It handles paging, continuation tokens, and retriable errors such as "too many requests" for you,
/// while aggregating all query results in-memory before returning.
/// </summary>
/// <typeparam name="R"></typeparam>
/// <param name="feedTakingContinuation"></param>
/// <param name="enumerationExceptionHandler"></param>
/// <param name="feedResponseHandler"></param>
/// <param name="maxRetries"></param>
/// <param name="maxTime"></param>
/// <param name="shouldRetry"></param>
/// <returns></returns>
public static async Task <IList <R> > ExecuteFeedWithContinuationAndRetry <R>(Func <string, Task <FeedResponse <R> > > feedTakingContinuation, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
var context = new FeedResponseContext();
feedResponseHandler(context, FeedResponseType.BeforeEnumeration, null);
var allResults = new List <R>();
FeedResponse <R> intermediateResponse = null;
do
{
try
{
intermediateResponse = await DocumentDbReliableExecution.ExecuteResultWithRetry(() =>
feedTakingContinuation(intermediateResponse?.ResponseContinuation),
null,
maxRetries,
maxTime,
shouldRetry);
}
catch (Exception ex)
{
bool handled = enumerationExceptionHandler(null, ex);
if (!handled)
{
feedResponseHandler(context, FeedResponseType.EnumerationAborted, null);
throw;
}
else
{
break;
}
}
// lots of interesting info in intermediateResults such as RU usage, etc.
List <R> intermediateResults = intermediateResponse.ToList();
UpdateContext <R>(context, intermediateResponse, JsonConvert.SerializeObject(intermediateResults).Length, !string.IsNullOrEmpty(intermediateResponse.ResponseContinuation));
feedResponseHandler(context, FeedResponseType.PageReceived, new FeedResponseWrapper <R>(intermediateResponse));
allResults.AddRange(intermediateResults);
} while (!string.IsNullOrEmpty(intermediateResponse.ResponseContinuation));
feedResponseHandler(context, FeedResponseType.AfterEnumeration, null);
return(allResults);
}
/// <summary>
/// This will execute a DocumentDB FeedResponse method return the results.
///
/// It handles paging, continuation tokens, and retriable errors such as "too many requests" for you,
/// while aggregating all query results in-memory before returning.
/// </summary>
/// <typeparam name="R"></typeparam>
/// <param name="feedTakingContinuation"></param>
/// <param name="enumerationExceptionHandler"></param>
/// <param name="feedResponseHandler"></param>
/// <param name="maxRetries"></param>
/// <param name="maxTime"></param>
/// <param name="shouldRetry"></param>
/// <returns></returns>
public static async Task <IList <R> > ExecuteFeedWithContinuationAndRetry <R>(Func <string, FeedResponse <R> > feedTakingContinuation, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
return(await ExecuteFeedWithContinuationAndRetry <R>((continuation) => Task <R> .Run(() => feedTakingContinuation(continuation)), enumerationExceptionHandler, feedResponseHandler, maxRetries, maxTime, shouldRetry));
}
/// <summary>
/// This will execute a DocumentDB query in the form of an IQueryable (Linq form) and return the results.
///
/// It handles paging, continuation tokens, and retriable errors such as "too many requests" for you,
/// while streaming query results out in chunks via IEnumerable / yield.
/// </summary>
/// <typeparam name="R"></typeparam>
/// <param name="queryable"></param>
/// <param name="queryExecutionHandler"></param>
/// <param name="enumerationExceptionHandler"></param>
/// <param name="feedResponseHandler"></param>
/// <param name="maxRetries"></param>
/// <param name="maxTime"></param>
/// <param name="shouldRetry"></param>
/// <returns></returns>
public static IEnumerable <R> StreamQueryWithContinuationAndRetry <R>(IQueryable <R> queryable, QueryExecutionHandler queryExecutionHandler, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
IDocumentQuery <R> query = null;
try
{
query = queryable.AsDocumentQuery();
}
catch (Exception e)
{
throw new ArgumentException(BadQueryableMessage, e);
}
var context = new FeedResponseContext();
queryExecutionHandler(context, query.ToString());
feedResponseHandler(context, FeedResponseType.BeforeEnumeration, null);
while (query.HasMoreResults)
{
Task <FeedResponse <R> > t;
try
{
t = Task.Run(async() => await ExecuteResultWithRetry(() =>
query.ExecuteNextAsync <R>(),
null,
maxRetries,
maxTime,
shouldRetry));
t.Wait();
}
catch (Exception ex)
{
// note: if an IQueryable is returned to OData, throwing an exception here will cause it to take down w3wp.exe
bool handled = enumerationExceptionHandler(context, ex);
if (!handled)
{
feedResponseHandler(context, FeedResponseType.EnumerationAborted, null);
throw;
}
else
{
break;
}
}
var intermediateResponse = t.Result;
// lots of interesting info in intermediateResults such as RU usage, etc.
List <R> intermediateResults = intermediateResponse.ToList();
UpdateContext <R>(context, intermediateResponse, JsonConvert.SerializeObject(intermediateResults).Length, query.HasMoreResults);
feedResponseHandler(context, FeedResponseType.PageReceived, new FeedResponseWrapper <R>(intermediateResponse));
foreach (var result in intermediateResults)
{
yield return(result);
}
}
feedResponseHandler(context, FeedResponseType.AfterEnumeration, null);
}
/// <summary>
/// This will execute a DocumentDB query in the form of an IQueryable (Linq form) and return the results.
///
/// It handles paging, continuation tokens, and retriable errors such as "too many requests" for you,
/// while aggregating all query results in-memory before returning.
/// </summary>
/// <typeparam name="R"></typeparam>
/// <param name="queryable"></param>
/// <param name="queryExecutionHandler"></param>
/// <param name="enumerationExceptionHandler"></param>
/// <param name="feedResponseHandler"></param>
/// <param name="maxRetries"></param>
/// <param name="maxTime"></param>
/// <param name="shouldRetry"></param>
/// <returns></returns>
public static async Task <IList <R> > ExecuteQueryWithContinuationAndRetry <R>(IQueryable <R> queryable, QueryExecutionHandler queryExecutionHandler, EnumerationExceptionHandler enumerationExceptionHandler, FeedResponseHandler feedResponseHandler, int maxRetries, TimeSpan maxTime, ShouldRetry shouldRetry)
{
IDocumentQuery <R> query = null;
try
{
query = queryable.AsDocumentQuery();
}
catch (Exception e)
{
throw new ArgumentException(BadQueryableMessage, e);
}
var context = new FeedResponseContext();
queryExecutionHandler(context, query.ToString());
feedResponseHandler(context, FeedResponseType.BeforeEnumeration, null);
var allResults = new List <R>();
while (query.HasMoreResults)
{
FeedResponse <R> intermediateResponse = null;
try
{
intermediateResponse = await DocumentDbReliableExecution.ExecuteResultWithRetry(() =>
query.ExecuteNextAsync <R>(),
null,
maxRetries,
maxTime,
shouldRetry);
}
catch (Exception ex)
{
bool handled = enumerationExceptionHandler(context, ex);
if (!handled)
{
feedResponseHandler(context, FeedResponseType.EnumerationAborted, null);
throw;
}
else
{
break;
}
}
// lots of interesting info in intermediateResults such as RU usage, etc.
List <R> intermediateResults = intermediateResponse.ToList();
UpdateContext <R>(context, intermediateResponse, JsonConvert.SerializeObject(intermediateResults).Length, query.HasMoreResults);
feedResponseHandler(context, FeedResponseType.PageReceived, new FeedResponseWrapper <R>(intermediateResponse));
allResults.AddRange(intermediateResults);
}
feedResponseHandler(context, FeedResponseType.AfterEnumeration, null);
return(allResults);
}
