private void fetchEntireDataSet(object contextObj)
{
var context = (FetchFullDataSetContext)contextObj;
var conf = new ConnectionConfiguration(new Uri(ClusterUrl));
var client = new ElasticLowLevelClient(conf);
// First establish initial search request to get a scroll token
var searchResponse = client.SearchGet <dynamic>(IndexPattern, x => x
.IgnoreUnavailable(true)
.AddQueryString("scroll", "1m")
.AddQueryString("q", context.Query)
.AddQueryString("size", "2000"));
if (searchResponse.Success)
{
context.Token.ThrowIfCancellationRequested();
context.ScrollToken = searchResponse.Body._scroll_id;
context.TotalDocuments = Convert.ToInt32(searchResponse.Body.hits.total);
reportProgress(context.ProcessedDocuments, context.TotalDocuments);
var table = new DataTable();
bool keepRequesting = searchResponse.Body.hits.hits.Count > 0;
using (table)
{
foreach (var hit in searchResponse.Body.hits.hits)
{
context.Token.ThrowIfCancellationRequested();
Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString());
var row = table.NewRow();
foreach (var item in rowSource)
{
if (!table.Columns.Contains(item.Key))
{
table.Columns.Add(item.Key);
}
if (!context.Columns.Contains(item.Key))
{
context.Columns.Add(item.Key);
}
row[item.Key] = item.Value;
context.Token.ThrowIfCancellationRequested();
}
table.Rows.Add(row);
}
string tempFilePath = Path.GetTempFileName();
context.TempFiles.Add(tempFilePath);
using (var fs = File.OpenWrite(tempFilePath))
{
BinaryFormatter bf = new BinaryFormatter();
bf.AssemblyFormat = FormatterAssemblyStyle.Simple;
bf.TypeFormat = FormatterTypeStyle.TypesWhenNeeded;
bf.Serialize(fs, table);
}
context.ProcessedDocuments += table.Rows.Count;
reportProgress(context.ProcessedDocuments, context.TotalDocuments);
}
// now subsequent request for another search
do
{
context.Token.ThrowIfCancellationRequested();
var scrollRequest = client.Scroll <dynamic>(new { scroll = "1m", scroll_id = context.ScrollToken });
if (scrollRequest.Success)
{
// delete last scroll token
//var clearRequest = client.ClearScroll<dynamic>(new { scroll_id = new string[] { scrollIdToken } });
// if (!clearRequest.Success)
// throw clearRequest.OriginalException;
keepRequesting = scrollRequest.Body.hits.hits.Count > 0;
using (table = new DataTable())
{
foreach (var hit in scrollRequest.Body.hits.hits)
{
context.Token.ThrowIfCancellationRequested();
Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString());
var row = table.NewRow();
foreach (var item in rowSource)
{
if (!table.Columns.Contains(item.Key))
{
table.Columns.Add(item.Key);
}
if (!context.Columns.Contains(item.Key))
{
context.Columns.Add(item.Key);
}
row[item.Key] = item.Value;
context.Token.ThrowIfCancellationRequested();
}
table.Rows.Add(row);
}
string tempFilePath = Path.GetTempFileName();
context.TempFiles.Add(tempFilePath);
using (var fs = File.OpenWrite(tempFilePath))
{
BinaryFormatter bf = new BinaryFormatter();
bf.AssemblyFormat = FormatterAssemblyStyle.Simple;
bf.TypeFormat = FormatterTypeStyle.TypesWhenNeeded;
bf.Serialize(fs, table);
}
context.ProcessedDocuments += table.Rows.Count;
reportProgress(context.ProcessedDocuments, context.TotalDocuments);
}
}
else
{
throw scrollRequest.OriginalException;
}
} while (keepRequesting);
}
else
{
throw searchResponse.OriginalException;
}
}
/// <summary>
/// Get documents from a started scroll.
/// </summary>
/// <param name="scroll">Information about the scroll.</param>
public DocumentFilterResponse GetScroll(ElasticsearchScroll scroll)
{
DocumentFilterResponse documentFilterResponse;
ElasticsearchResponse <DynamicResponse> response;
StringBuilder scrollInformation;
Int32 startIndex, stopIndex;
String maxScore, shardInformation, documentCount;
String[] splitShardInformation;
scrollInformation = new StringBuilder();
scrollInformation.Append("{");
scrollInformation.Append("\"scroll\" : \"" + scroll.KeepAlive + "m\"");
scrollInformation.Append(", \"scroll_id\" : \"" + scroll.ScrollId + "\"");
scrollInformation.Append("}");
response = _client.Scroll <DynamicResponse>(scrollInformation.ToString());
CheckResponse(response);
documentFilterResponse = new DocumentFilterResponse();
scroll.ScrollId = (String)(response.Body.Values.ElementAt(0));
if (response.Body.IsNotNull())
{
documentFilterResponse.ElapsedTime = (Int32)(response.Body.Values.ElementAt(1));
// Get shard information.
shardInformation = (String)(response.Body.Values.ElementAt(4));
splitShardInformation = shardInformation.Split(':');
documentFilterResponse.ShardTotalCount = splitShardInformation[1].Substring(0, splitShardInformation[1].IndexOf(',')).WebParseInt32();
documentFilterResponse.ShardSuccessfulCount = splitShardInformation[2].Substring(0, splitShardInformation[2].IndexOf(',')).WebParseInt32();
if (documentFilterResponse.ShardSuccessfulCount == documentFilterResponse.ShardTotalCount)
{
documentFilterResponse.ShardFailedCount = splitShardInformation[3].Substring(0, splitShardInformation[3].IndexOf('}')).WebParseInt32();
}
else
{
documentFilterResponse.ShardFailedCount = splitShardInformation[3].Substring(0, splitShardInformation[3].IndexOf(',')).WebParseInt32();
}
documentFilterResponse.DocumentCount = 0;
documentFilterResponse.DocumentsJson = (String)(response.Body.Values.ElementAt(5));
documentFilterResponse.TimedOut = (Boolean)(response.Body.Values.ElementAt(2));
if (!documentFilterResponse.TimedOut)
{
// Get species observation count.
startIndex = documentFilterResponse.DocumentsJson.IndexOf(':') + 1;
stopIndex = documentFilterResponse.DocumentsJson.IndexOf(',');
documentCount = documentFilterResponse.DocumentsJson.Substring(startIndex, stopIndex - startIndex);
documentFilterResponse.DocumentCount = documentCount.WebParseInt64();
startIndex = stopIndex + 1;
if (documentFilterResponse.DocumentCount > 0)
{
// Get max score
startIndex = documentFilterResponse.DocumentsJson.IndexOf(':', startIndex) + 1;
stopIndex = documentFilterResponse.DocumentsJson.IndexOf(',', startIndex);
maxScore = documentFilterResponse.DocumentsJson.Substring(startIndex, stopIndex - startIndex);
if (maxScore != "null")
{
documentFilterResponse.MaxScore = maxScore.WebParseDouble();
}
}
}
}
return(documentFilterResponse);
}
