private DataFetchResult fetchSampleDataSet(object tokenObj)
{
var token = (CancellationToken)tokenObj;
var conf = new ConnectionConfiguration(new Uri(ClusterUrl));
var client = new ElasticLowLevelClient(conf);
JsonSerializer serializer = new JsonSerializer();
serializer.DateParseHandling = DateParseHandling.DateTimeOffset;
var response = client.SearchGet <dynamic>(IndexPattern, x => x.IgnoreUnavailable(true).AddQueryString("q", txtLuceneQuery.Text).AddQueryString("size", "500"));
if (response.Success)
{
token.ThrowIfCancellationRequested();
var table = new DataTable();
var result = new DataFetchResult(table, new DataFetchContext()
{
TimeTaken = TimeSpan.FromMilliseconds(Convert.ToInt32(response.Body.took)), TotalHits = Convert.ToInt64(response.Body.hits.total)
});
foreach (var hit in response.Body.hits.hits)
{
token.ThrowIfCancellationRequested();
Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString());
var row = table.NewRow();
foreach (var item in rowSource)
{
if (!table.Columns.Contains(item.Key))
{
table.Columns.Add(item.Key);
}
row[item.Key] = item.Value;
token.ThrowIfCancellationRequested();
}
table.Rows.Add(row);
}
return(result);
}
else
{
throw response.OriginalException;
}
}
private static List<Tuple<string, string, string, DateTime>> HierarchySearch()
{
var node = new Uri("http://boom-box-1.boomerang.com:9200");
var config = new ConnectionConfiguration(node);
var client = new ElasticLowLevelClient(config);
var result = client.SearchGet<object>("hsbc_conform", "osf_hierarchies", (arg) => arg.AddQueryString("size", "100"));
var root = JObject.FromObject(result.Body);
var hierarchies = new List<Tuple<string, string, string, DateTime>>();
foreach (var item in root["hits"]["hits"])
{
var date = item["_source"]["ctl"]["effective_date"].Value<string>();
var country = item["_source"]["conform"]["geography"].Value<string>();
var channel = item["_source"]["conform"]["channel"].Value<string>();
var parsedDate = DateTime.ParseExact(date, "yyyyMMdd", CultureInfo.InvariantCulture);
var tuple = Tuple.Create(channel, country, date, parsedDate);
hierarchies.Add(tuple);
}
return hierarchies;
}
private void fetchEntireDataSet(object contextObj)
{
var context = (FetchFullDataSetContext)contextObj;
var conf = new ConnectionConfiguration(new Uri(ClusterUrl));
var client = new ElasticLowLevelClient(conf);
// First establish initial search request to get a scroll token
var searchResponse = client.SearchGet <dynamic>(IndexPattern, x => x
.IgnoreUnavailable(true)
.AddQueryString("scroll", "1m")
.AddQueryString("q", context.Query)
.AddQueryString("size", "2000"));
if (searchResponse.Success)
{
context.Token.ThrowIfCancellationRequested();
context.ScrollToken = searchResponse.Body._scroll_id;
context.TotalDocuments = Convert.ToInt32(searchResponse.Body.hits.total);
reportProgress(context.ProcessedDocuments, context.TotalDocuments);
var table = new DataTable();
bool keepRequesting = searchResponse.Body.hits.hits.Count > 0;
using (table)
{
foreach (var hit in searchResponse.Body.hits.hits)
{
context.Token.ThrowIfCancellationRequested();
Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString());
var row = table.NewRow();
foreach (var item in rowSource)
{
if (!table.Columns.Contains(item.Key))
{
table.Columns.Add(item.Key);
}
if (!context.Columns.Contains(item.Key))
{
context.Columns.Add(item.Key);
}
row[item.Key] = item.Value;
context.Token.ThrowIfCancellationRequested();
}
table.Rows.Add(row);
}
string tempFilePath = Path.GetTempFileName();
context.TempFiles.Add(tempFilePath);
using (var fs = File.OpenWrite(tempFilePath))
{
BinaryFormatter bf = new BinaryFormatter();
bf.AssemblyFormat = FormatterAssemblyStyle.Simple;
bf.TypeFormat = FormatterTypeStyle.TypesWhenNeeded;
bf.Serialize(fs, table);
}
context.ProcessedDocuments += table.Rows.Count;
reportProgress(context.ProcessedDocuments, context.TotalDocuments);
}
// now subsequent request for another search
do
{
context.Token.ThrowIfCancellationRequested();
var scrollRequest = client.Scroll <dynamic>(new { scroll = "1m", scroll_id = context.ScrollToken });
if (scrollRequest.Success)
{
// delete last scroll token
//var clearRequest = client.ClearScroll<dynamic>(new { scroll_id = new string[] { scrollIdToken } });
// if (!clearRequest.Success)
// throw clearRequest.OriginalException;
keepRequesting = scrollRequest.Body.hits.hits.Count > 0;
using (table = new DataTable())
{
foreach (var hit in scrollRequest.Body.hits.hits)
{
context.Token.ThrowIfCancellationRequested();
Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString());
var row = table.NewRow();
foreach (var item in rowSource)
{
if (!table.Columns.Contains(item.Key))
{
table.Columns.Add(item.Key);
}
if (!context.Columns.Contains(item.Key))
{
context.Columns.Add(item.Key);
}
row[item.Key] = item.Value;
context.Token.ThrowIfCancellationRequested();
}
table.Rows.Add(row);
}
string tempFilePath = Path.GetTempFileName();
context.TempFiles.Add(tempFilePath);
using (var fs = File.OpenWrite(tempFilePath))
{
BinaryFormatter bf = new BinaryFormatter();
bf.AssemblyFormat = FormatterAssemblyStyle.Simple;
bf.TypeFormat = FormatterTypeStyle.TypesWhenNeeded;
bf.Serialize(fs, table);
}
context.ProcessedDocuments += table.Rows.Count;
reportProgress(context.ProcessedDocuments, context.TotalDocuments);
}
}
else
{
throw scrollRequest.OriginalException;
}
} while (keepRequesting);
}
else
{
throw searchResponse.OriginalException;
}
}
