protected async Task <long> BatchProcessAsAsync <TQuery, TResult>(TQuery query, Func <FindResults <TResult>, Task <bool> > processAsync) where TQuery : IPagableQuery, ISelectedFieldsQuery, IRepositoryQuery where TResult : class, new() { if (query == null) { throw new ArgumentNullException(nameof(query)); } if (processAsync == null) { throw new ArgumentNullException(nameof(processAsync)); } var elasticPagingOptions = query.Options as ElasticPagingOptions; if (query.Options == null || elasticPagingOptions == null) { elasticPagingOptions = ElasticPagingOptions.FromOptions(query.Options); query.Options = elasticPagingOptions; } elasticPagingOptions.UseSnapshotPaging = true; if (!elasticPagingOptions.SnapshotLifetime.HasValue) { elasticPagingOptions.SnapshotLifetime = TimeSpan.FromMinutes(5); } long recordsProcessed = 0; var results = await FindAsAsync <TResult>(query).AnyContext(); do { if (results.Hits.Count == 0) { break; } // TODO: We need a generic way to do bulk operations and do exponential backoffs when we encounter on 429's (bulk queue is full). https://github.com/elastic/elasticsearch-net/pull/2162 if (await processAsync(results).AnyContext()) { recordsProcessed += results.Documents.Count; continue; } _logger.Trace("Aborted batch processing."); break; } while (await results.NextPageAsync().AnyContext()); _logger.Trace("{0} records processed", recordsProcessed); return(recordsProcessed); }
public static T WithSnapshotLifetime <T>(this T query, TimeSpan lifetime) where T : IPagableQuery { if (query == null) { return(query); } var elasticOptions = query.Options as ElasticPagingOptions; if (elasticOptions == null) { elasticOptions = ElasticPagingOptions.FromOptions(query.Options); query.Options = elasticOptions; } elasticOptions.SnapshotLifetime = lifetime; return(query); }
public static T WithSnapshotPaging <T>(this T query, bool useSnapshotPaging = true) where T : IPagableQuery { if (query == null) { return(query); } var elasticOptions = query.Options as ElasticPagingOptions; if (elasticOptions == null) { elasticOptions = ElasticPagingOptions.FromOptions(query.Options); query.Options = elasticOptions; } elasticOptions.UseSnapshotPaging = useSnapshotPaging; return(query); }