/// <summary> /// Gets all documents within elastic. /// Based upon http://telegraphrepaircompany.com/elasticsearch-nest-scroll-api-c/ /// </summary> /// <param name="client">The client to use.</param> /// <returns>Enumerable of all articles within the ElasticSearch server.</returns> private static IEnumerable <Article> GetAllDocumentsInIndex(IElasticClient client) { var initialResponse = client.Search <Article> (scr => scr.Index(AppConfiguration.ElasticPermanentIndexName) .From(0) .Take(1000) .MatchAll() .Scroll("2m")); var results = new List <Article>(); if (initialResponse.Documents.Any()) { results.AddRange(initialResponse.Documents); } var scrollId = initialResponse.ScrollId; var dataRemaining = true; while (dataRemaining) { var loopingResponse = client.Scroll <Article>("2m", scrollId); if (loopingResponse.IsValid) { results.AddRange(loopingResponse.Documents); scrollId = loopingResponse.ScrollId; } dataRemaining = loopingResponse.Documents.Any(); } client.ClearScroll(new ClearScrollRequest(scrollId)); return(results); }
private IReadOnlyCollection <IHit <TModel> > FetchAllHitsByQuery(QueryContainer container, string indexName) { var response = NestScrollSearchInit(container, indexName); var result = new List <IHit <TModel> >(); var anyHitsLeft = true; string scrollId = response.ScrollId; while (anyHitsLeft) { if (response.IsValid) { result.AddRange(response.Hits); scrollId = response.ScrollId; response = elasticClient.Scroll <TModel>("2m", scrollId); ElasticResponseValidator.Validate(response); } anyHitsLeft = response.Hits.Any(); } elasticClient.ClearScroll(new ClearScrollRequest(scrollId)); return(new ReadOnlyCollection <IHit <TModel> >(result)); }
private IEnumerable <TModel> FetchAllByQuery(QueryContainer queryContainer, string indexName) { var response = NestScrollSearchInit(queryContainer, indexName); var result = new List <TModel>(); var anyHitsLeft = true; string scrollId = response.ScrollId; while (anyHitsLeft) { if (response.IsValid) { result.AddRange(response.Documents); scrollId = response.ScrollId; response = elasticClient.Scroll <TModel>("2m", scrollId); ElasticResponseValidator.Validate(response); } anyHitsLeft = response.Documents.Any(); } elasticClient.ClearScroll(new ClearScrollRequest(scrollId)); return(result); }
public ICollection <IndexArticles> GetAll() { var result = _elasticClient.Search <IndexArticles>(s => s .Index(IndexArticles.ArticleIndex) .Sort(q => q.Descending(p => p.PublishDate)))?.Documents; #region var result2 = _elasticClient.Search <IndexArticles>(s => s .Index(IndexArticles.ArticleIndex) .MatchAll()).Documents.ToList(); var result3 = _elasticClient.Search <IndexArticles>(s => s .Index(IndexArticles.ArticleIndex) .From(0) .Size(5) .MatchAll()).Documents.ToList(); //scroll var result4 = _elasticClient.Search <IndexArticles>(s => s .Index(IndexArticles.ArticleIndex) .From(0) .Size(5) .Scroll("1m") .MatchAll()); List <IndexArticles> results = new List <IndexArticles>(); if (result4.Documents.Any()) { results.AddRange(result4.Documents); } string scrollid = result4.ScrollId; bool isScrollSetHasData = true; while (isScrollSetHasData) { ISearchResponse <IndexArticles> loopingResponse = _elasticClient.Scroll <IndexArticles>("1m", scrollid); if (loopingResponse.IsValid) { results.AddRange(loopingResponse.Documents); scrollid = loopingResponse.ScrollId; } isScrollSetHasData = loopingResponse.Documents.Any(); } _elasticClient.ClearScroll(new ClearScrollRequest(scrollid)); #endregion return(results); }
public ICollection <IndexActors> GetAll() { var results = new List <IndexActors>(); var isScrollSetHasData = true; var result = _elasticClient.Search <IndexActors>(s => s .Index(nameof(IndexActors).ToLower()) .Sort(q => q.Descending(p => p.BirthDate)))?.Documents; var result2 = _elasticClient.Search <IndexActors>(s => s .Index(nameof(IndexActors).ToLower()) .MatchAll()).Documents.ToList(); var result3 = _elasticClient.Search <IndexActors>(s => s .Index(nameof(IndexActors).ToLower()) .From(0) .Size(10) .MatchAll()).Documents.ToList(); //scroll var result4 = _elasticClient.Search <IndexActors>(s => s .Index(nameof(IndexActors).ToLower()) .From(0) .Size(10) .Scroll("1m") .MatchAll()); if (result4.Documents.Any()) { results.AddRange(result4.Documents); } var scrollid = result4.ScrollId; while (isScrollSetHasData) { var loopingResponse = _elasticClient.Scroll <IndexActors>("1m", scrollid); if (loopingResponse.IsValid) { results.AddRange(loopingResponse.Documents); scrollid = loopingResponse.ScrollId; } isScrollSetHasData = loopingResponse.Documents.Any(); } _elasticClient.ClearScroll(new ClearScrollRequest(scrollid)); return(result.ToList()); }
public Task <List <ShareUrl> > GetUrls() { //The thing to know about scrollTimeout is that it resets after each call to the scroll so it only needs to be big enough to stay alive between calls. //when it expires, elastic will delete the entire scroll. _logger.LogInformation("Starting to get all shares"); ISearchResponse <ShareUrl> initialResponse = _elasticClient.Search <ShareUrl>(s => s .Index(SHARES) .From(0) .Size(1000) .MatchAll() .Scroll("10m")); List <ShareUrl> results = new List <ShareUrl>(); if (!initialResponse.IsValid || string.IsNullOrEmpty(initialResponse.ScrollId)) { throw new Exception(initialResponse.ServerError?.Error?.Reason ?? "Unable to get urls"); } if (initialResponse.Documents.Any()) { results.AddRange(initialResponse.Documents); } string scrollid = initialResponse.ScrollId; bool isScrollSetHasData = true; int page = 0; while (isScrollSetHasData) { page++; _logger.LogInformation($"More data needs to be fetched, page: {page}"); ISearchResponse <ShareUrl> loopingResponse = _elasticClient.Scroll <ShareUrl>("10m", scrollid); if (loopingResponse.IsValid) { results.AddRange(loopingResponse.Documents); scrollid = loopingResponse.ScrollId; } isScrollSetHasData = loopingResponse.Documents.Any(); } //This would be garbage collected on it's own after scrollTimeout expired from it's last call but we'll clean up our room when we're done per best practice. _elasticClient.ClearScroll(new ClearScrollRequest(scrollid)); _logger.LogInformation("Finished getting all shares: " + results.Count); return(Task.FromResult(results)); }
protected override void OnAfterCall(IElasticClient client) => client.ClearScroll(cs => cs.ScrollId(_scrollId));
/// <summary> /// Deletes a registered scroll request on the cluster /// <para> </para>http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-scroll.html /// </summary> /// <param name="client"></param> /// <param name="scrollId">The scrollId to clear</param> public static IEmptyResponse ClearScroll(this IElasticClient client, string scrollId) { return(client.ClearScroll(s => s.ScrollId(scrollId))); }
protected override void OnAfterCall(IElasticClient client) { client.ClearScroll(_scrollId); }