protected async Task <long> BatchProcessAsAsync <TQuery, TResult>(TQuery query, Func <FindResults <TResult>, Task <bool> > processAsync) where TQuery : IPagableQuery, ISelectedFieldsQuery, IRepositoryQuery where TResult : class, new()
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
if (processAsync == null)
{
throw new ArgumentNullException(nameof(processAsync));
}
var elasticPagingOptions = query.Options as ElasticPagingOptions;
if (query.Options == null || elasticPagingOptions == null)
{
elasticPagingOptions = ElasticPagingOptions.FromOptions(query.Options);
query.Options = elasticPagingOptions;
}
elasticPagingOptions.UseSnapshotPaging = true;
if (!elasticPagingOptions.SnapshotLifetime.HasValue)
{
elasticPagingOptions.SnapshotLifetime = TimeSpan.FromMinutes(5);
}
long recordsProcessed = 0;
var results = await FindAsAsync <TResult>(query).AnyContext();
do
{
if (results.Hits.Count == 0)
{
break;
}
// TODO: We need a generic way to do bulk operations and do exponential backoffs when we encounter on 429's (bulk queue is full). https://github.com/elastic/elasticsearch-net/pull/2162
if (await processAsync(results).AnyContext())
{
recordsProcessed += results.Documents.Count;
continue;
}
_logger.Trace("Aborted batch processing.");
break;
} while (await results.NextPageAsync().AnyContext());
_logger.Trace("{0} records processed", recordsProcessed);
return(recordsProcessed);
}
public static T WithSnapshotLifetime <T>(this T query, TimeSpan lifetime) where T : IPagableQuery
{
if (query == null)
{
return(query);
}
var elasticOptions = query.Options as ElasticPagingOptions;
if (elasticOptions == null)
{
elasticOptions = ElasticPagingOptions.FromOptions(query.Options);
query.Options = elasticOptions;
}
elasticOptions.SnapshotLifetime = lifetime;
return(query);
}
public static T WithSnapshotPaging <T>(this T query, bool useSnapshotPaging = true) where T : IPagableQuery
{
if (query == null)
{
return(query);
}
var elasticOptions = query.Options as ElasticPagingOptions;
if (elasticOptions == null)
{
elasticOptions = ElasticPagingOptions.FromOptions(query.Options);
query.Options = elasticOptions;
}
elasticOptions.UseSnapshotPaging = useSnapshotPaging;
return(query);
}