/// <summary>
/// Gets all documents within elastic.
/// Based upon http://telegraphrepaircompany.com/elasticsearch-nest-scroll-api-c/
/// </summary>
/// <param name="client">The client to use.</param>
/// <returns>Enumerable of all articles within the ElasticSearch server.</returns>
private static IEnumerable <Article> GetAllDocumentsInIndex(IElasticClient client)
{
var initialResponse = client.Search <Article>
(scr => scr.Index(AppConfiguration.ElasticPermanentIndexName)
.From(0)
.Take(1000)
.MatchAll()
.Scroll("2m"));
var results = new List <Article>();
if (initialResponse.Documents.Any())
{
results.AddRange(initialResponse.Documents);
}
var scrollId = initialResponse.ScrollId;
var dataRemaining = true;
while (dataRemaining)
{
var loopingResponse = client.Scroll <Article>("2m", scrollId);
if (loopingResponse.IsValid)
{
results.AddRange(loopingResponse.Documents);
scrollId = loopingResponse.ScrollId;
}
dataRemaining = loopingResponse.Documents.Any();
}
client.ClearScroll(new ClearScrollRequest(scrollId));
return(results);
}
private IReadOnlyCollection <IHit <TModel> > FetchAllHitsByQuery(QueryContainer container, string indexName)
{
var response = NestScrollSearchInit(container, indexName);
var result = new List <IHit <TModel> >();
var anyHitsLeft = true;
string scrollId = response.ScrollId;
while (anyHitsLeft)
{
if (response.IsValid)
{
result.AddRange(response.Hits);
scrollId = response.ScrollId;
response = elasticClient.Scroll <TModel>("2m", scrollId);
ElasticResponseValidator.Validate(response);
}
anyHitsLeft = response.Hits.Any();
}
elasticClient.ClearScroll(new ClearScrollRequest(scrollId));
return(new ReadOnlyCollection <IHit <TModel> >(result));
}
private IEnumerable <TModel> FetchAllByQuery(QueryContainer queryContainer, string indexName)
{
var response = NestScrollSearchInit(queryContainer, indexName);
var result = new List <TModel>();
var anyHitsLeft = true;
string scrollId = response.ScrollId;
while (anyHitsLeft)
{
if (response.IsValid)
{
result.AddRange(response.Documents);
scrollId = response.ScrollId;
response = elasticClient.Scroll <TModel>("2m", scrollId);
ElasticResponseValidator.Validate(response);
}
anyHitsLeft = response.Documents.Any();
}
elasticClient.ClearScroll(new ClearScrollRequest(scrollId));
return(result);
}
public ICollection <IndexArticles> GetAll()
{
var result = _elasticClient.Search <IndexArticles>(s => s
.Index(IndexArticles.ArticleIndex)
.Sort(q => q.Descending(p => p.PublishDate)))?.Documents;
#region
var result2 = _elasticClient.Search <IndexArticles>(s => s
.Index(IndexArticles.ArticleIndex)
.MatchAll()).Documents.ToList();
var result3 = _elasticClient.Search <IndexArticles>(s => s
.Index(IndexArticles.ArticleIndex)
.From(0)
.Size(5)
.MatchAll()).Documents.ToList();
//scroll
var result4 = _elasticClient.Search <IndexArticles>(s => s
.Index(IndexArticles.ArticleIndex)
.From(0)
.Size(5)
.Scroll("1m")
.MatchAll());
List <IndexArticles> results = new List <IndexArticles>();
if (result4.Documents.Any())
{
results.AddRange(result4.Documents);
}
string scrollid = result4.ScrollId;
bool isScrollSetHasData = true;
while (isScrollSetHasData)
{
ISearchResponse <IndexArticles> loopingResponse = _elasticClient.Scroll <IndexArticles>("1m", scrollid);
if (loopingResponse.IsValid)
{
results.AddRange(loopingResponse.Documents);
scrollid = loopingResponse.ScrollId;
}
isScrollSetHasData = loopingResponse.Documents.Any();
}
_elasticClient.ClearScroll(new ClearScrollRequest(scrollid));
#endregion
return(results);
}
public ICollection <IndexActors> GetAll()
{
var results = new List <IndexActors>();
var isScrollSetHasData = true;
var result = _elasticClient.Search <IndexActors>(s => s
.Index(nameof(IndexActors).ToLower())
.Sort(q => q.Descending(p => p.BirthDate)))?.Documents;
var result2 = _elasticClient.Search <IndexActors>(s => s
.Index(nameof(IndexActors).ToLower())
.MatchAll()).Documents.ToList();
var result3 = _elasticClient.Search <IndexActors>(s => s
.Index(nameof(IndexActors).ToLower())
.From(0)
.Size(10)
.MatchAll()).Documents.ToList();
//scroll
var result4 = _elasticClient.Search <IndexActors>(s => s
.Index(nameof(IndexActors).ToLower())
.From(0)
.Size(10)
.Scroll("1m")
.MatchAll());
if (result4.Documents.Any())
{
results.AddRange(result4.Documents);
}
var scrollid = result4.ScrollId;
while (isScrollSetHasData)
{
var loopingResponse = _elasticClient.Scroll <IndexActors>("1m", scrollid);
if (loopingResponse.IsValid)
{
results.AddRange(loopingResponse.Documents);
scrollid = loopingResponse.ScrollId;
}
isScrollSetHasData = loopingResponse.Documents.Any();
}
_elasticClient.ClearScroll(new ClearScrollRequest(scrollid));
return(result.ToList());
}
public Task <List <ShareUrl> > GetUrls()
{
//The thing to know about scrollTimeout is that it resets after each call to the scroll so it only needs to be big enough to stay alive between calls.
//when it expires, elastic will delete the entire scroll.
_logger.LogInformation("Starting to get all shares");
ISearchResponse <ShareUrl> initialResponse = _elasticClient.Search <ShareUrl>(s => s
.Index(SHARES)
.From(0)
.Size(1000)
.MatchAll()
.Scroll("10m"));
List <ShareUrl> results = new List <ShareUrl>();
if (!initialResponse.IsValid || string.IsNullOrEmpty(initialResponse.ScrollId))
{
throw new Exception(initialResponse.ServerError?.Error?.Reason ?? "Unable to get urls");
}
if (initialResponse.Documents.Any())
{
results.AddRange(initialResponse.Documents);
}
string scrollid = initialResponse.ScrollId;
bool isScrollSetHasData = true;
int page = 0;
while (isScrollSetHasData)
{
page++;
_logger.LogInformation($"More data needs to be fetched, page: {page}");
ISearchResponse <ShareUrl> loopingResponse = _elasticClient.Scroll <ShareUrl>("10m", scrollid);
if (loopingResponse.IsValid)
{
results.AddRange(loopingResponse.Documents);
scrollid = loopingResponse.ScrollId;
}
isScrollSetHasData = loopingResponse.Documents.Any();
}
//This would be garbage collected on it's own after scrollTimeout expired from it's last call but we'll clean up our room when we're done per best practice.
_elasticClient.ClearScroll(new ClearScrollRequest(scrollid));
_logger.LogInformation("Finished getting all shares: " + results.Count);
return(Task.FromResult(results));
}
protected override void OnAfterCall(IElasticClient client) => client.ClearScroll(cs => cs.ScrollId(_scrollId));
/// <summary>
/// Deletes a registered scroll request on the cluster
/// <para> </para>http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-scroll.html
/// </summary>
/// <param name="client"></param>
/// <param name="scrollId">The scrollId to clear</param>
public static IEmptyResponse ClearScroll(this IElasticClient client, string scrollId)
{
return(client.ClearScroll(s => s.ScrollId(scrollId)));
}
protected override void OnAfterCall(IElasticClient client)
{
client.ClearScroll(_scrollId);
}