protected async Task <long> BatchProcessAsAsync <TQuery, TResult>(TQuery query, Func <FindResults <TResult>, Task <bool> > processAsync) where TQuery : IPagableQuery, ISelectedFieldsQuery, IRepositoryQuery where TResult : class, new()
        {
            if (query == null)
            {
                throw new ArgumentNullException(nameof(query));
            }

            if (processAsync == null)
            {
                throw new ArgumentNullException(nameof(processAsync));
            }

            var elasticPagingOptions = query.Options as ElasticPagingOptions;

            if (query.Options == null || elasticPagingOptions == null)
            {
                elasticPagingOptions = ElasticPagingOptions.FromOptions(query.Options);
                query.Options        = elasticPagingOptions;
            }

            elasticPagingOptions.UseSnapshotPaging = true;
            if (!elasticPagingOptions.SnapshotLifetime.HasValue)
            {
                elasticPagingOptions.SnapshotLifetime = TimeSpan.FromMinutes(5);
            }

            long recordsProcessed = 0;
            var  results          = await FindAsAsync <TResult>(query).AnyContext();

            do
            {
                if (results.Hits.Count == 0)
                {
                    break;
                }

                // TODO: We need a generic way to do bulk operations and do exponential backoffs when we encounter on 429's (bulk queue is full). https://github.com/elastic/elasticsearch-net/pull/2162
                if (await processAsync(results).AnyContext())
                {
                    recordsProcessed += results.Documents.Count;
                    continue;
                }

                _logger.Trace("Aborted batch processing.");
                break;
            } while (await results.NextPageAsync().AnyContext());

            _logger.Trace("{0} records processed", recordsProcessed);
            return(recordsProcessed);
        }
Example #2
0
        public static T WithSnapshotLifetime <T>(this T query, TimeSpan lifetime) where T : IPagableQuery
        {
            if (query == null)
            {
                return(query);
            }

            var elasticOptions = query.Options as ElasticPagingOptions;

            if (elasticOptions == null)
            {
                elasticOptions = ElasticPagingOptions.FromOptions(query.Options);
                query.Options  = elasticOptions;
            }

            elasticOptions.SnapshotLifetime = lifetime;
            return(query);
        }
Example #3
0
        public static T WithSnapshotPaging <T>(this T query, bool useSnapshotPaging = true) where T : IPagableQuery
        {
            if (query == null)
            {
                return(query);
            }

            var elasticOptions = query.Options as ElasticPagingOptions;

            if (elasticOptions == null)
            {
                elasticOptions = ElasticPagingOptions.FromOptions(query.Options);
                query.Options  = elasticOptions;
            }

            elasticOptions.UseSnapshotPaging = useSnapshotPaging;
            return(query);
        }