public virtual IEnumerable<TElastic> GetElastic(string fieldName, List<object> listValue)
{
if (string.IsNullOrWhiteSpace(fieldName))
{
throw new ArgumentException($"{nameof(fieldName)} is null or white space.");
}
listValue = listValue?.Where(x => x != null).ToList();
if (listValue == null || !listValue.Any())
{
throw new ArgumentException($"{nameof(listValue)} is null or empty.");
}
fieldName = fieldName.ToLower();
var search = new Search
{
Query = new Query(
new Filtered(
new Filter(
// filter by value
new TermsFilter(fieldName, listValue.ToList())))
{
Query = new Query(new MatchAllQuery())
})
};
var listElastic = new List<TElastic>();
using (var context = new ElasticContext(ElasticConnectionString, Config))
{
var scanScrollConfig =
new ScanAndScrollConfiguration(new TimeUnitMinute(1), Constants.ElasticSearch.MaxTakeRecord);
var searchResult = context.SearchCreateScanAndScroll<TElastic>(search, scanScrollConfig);
var scrollId = searchResult.PayloadResult?.ScrollId;
var total = searchResult.PayloadResult?.Hits?.Total ?? 0;
var processedResults = 0;
while (total > processedResults)
{
var resultCollection = context.SearchScanAndScroll<TElastic>(scrollId, scanScrollConfig);
scrollId = resultCollection.PayloadResult?.ScrollId;
listElastic.AddRange(resultCollection.PayloadResult?.Hits?.HitsResult?.Select(x => x.Source));
processedResults = listElastic.Count;
}
return listElastic;
}
}
public IHttpActionResult GetPersonsCsvExport()
{
_hubContext.Clients.All.addDiagnosisMessage(string.Format("Csv export starting"));
// force that this method always returns an excel document.
Request.Headers.Accept.Clear();
Request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.ms-excel"));
_hubContext.Clients.All.addDiagnosisMessage(string.Format("ScanAndScrollConfiguration: 1s, 300 items pro shard"));
_hubContext.Clients.All.addDiagnosisMessage(string.Format("sending scan and scroll _search"));
_hubContext.Clients.All.addDiagnosisMessage(BuildSearchMatchAll());
var result = new List <Person>();
using (var context = new ElasticsearchContext("http://localhost:9200/", new ElasticsearchMappingResolver()))
{
context.TraceProvider = new SignalRTraceProvider(_hubContext, TraceEventType.Information);
var scanScrollConfig = new ScanAndScrollConfiguration(new TimeUnitSecond(1), 300);
var scrollIdResult = context.SearchCreateScanAndScroll <Person>(BuildSearchMatchAll(), scanScrollConfig);
var scrollId = scrollIdResult.PayloadResult.ScrollId;
_hubContext.Clients.All.addDiagnosisMessage(string.Format("Total Hits: {0}", scrollIdResult.PayloadResult.Hits.Total));
int processedResults = 0;
while (scrollIdResult.PayloadResult.Hits.Total > processedResults)
{
var resultCollection = context.SearchScanAndScroll <Person>(scrollId, scanScrollConfig);
scrollId = resultCollection.PayloadResult.ScrollId;
result.AddRange(resultCollection.PayloadResult.Hits.HitsResult.Select(t => t.Source));
processedResults = result.Count;
_hubContext.Clients.All.addDiagnosisMessage(string.Format("Total Hits: {0}, Processed: {1}", scrollIdResult.PayloadResult.Hits.Total, processedResults));
}
}
_hubContext.Clients.All.addDiagnosisMessage(string.Format("Elasticsearch proccessing finished, starting to serialize csv"));
return(Ok(result));
}
public void TestScanAndScollReindexFor1000Entities()
{
using (var context = new ElasticsearchContext(ConnectionString, _elasticsearchMappingResolver))
{
//context.TraceProvider = new ConsoleTraceProvider();
for (int i = 0; i < 1000; i++)
{
context.AddUpdateDocument(_entitiesForTests[i], i);
}
// Save to Elasticsearch
var ret = context.SaveChanges();
Assert.AreEqual(ret.Status, HttpStatusCode.OK);
context.IndexClearCache <ScanScrollTypeV1>();
}
Task.Run(() =>
{
using (var context = new ElasticsearchContext(ConnectionString, _elasticsearchMappingResolver))
{
while (true)
{
Thread.Sleep(1300);
var itemOk = context.SearchById <ScanScrollTypeV1>(2);
if (itemOk != null)
{
_resetEvent.Set();
}
}
}
// ReSharper disable once FunctionNeverReturns
});
WaitForDataOrFail();
using (var context = new ElasticsearchContext(ConnectionString, _elasticsearchMappingResolver))
{
//context.TraceProvider = new ConsoleTraceProvider();
var scanScrollConfig = new ScanAndScrollConfiguration(new TimeUnitSecond(1), 100);
var result = context.SearchCreateScanAndScroll <ScanScrollTypeV1>(BuildSearchMatchAll(), scanScrollConfig);
var scrollId = result.PayloadResult.ScrollId;
int processedResults = 0;
while (result.PayloadResult.Hits.Total > processedResults)
{
var resultCollection = context.SearchScanAndScroll <ScanScrollTypeV1>(scrollId, scanScrollConfig);
scrollId = resultCollection.PayloadResult.ScrollId;
foreach (var item in resultCollection.PayloadResult.Hits.HitsResult)
{
processedResults++;
context.AddUpdateDocument(ConvertScanScrollTypeV1(item.Source), item.Id);
}
context.SaveChanges();
}
}
Task.Run(() =>
{
using (var context = new ElasticsearchContext(ConnectionString, _elasticsearchMappingResolver))
{
while (true)
{
Thread.Sleep(300);
if (10000 == context.Count <ScanScrollTypeV2>())
{
_resetEvent.Set();
}
}
}
// ReSharper disable once FunctionNeverReturns
});
WaitForDataOrFail();
}