private DataFetchResult fetchSampleDataSet(object tokenObj) { var token = (CancellationToken)tokenObj; var conf = new ConnectionConfiguration(new Uri(ClusterUrl)); var client = new ElasticLowLevelClient(conf); JsonSerializer serializer = new JsonSerializer(); serializer.DateParseHandling = DateParseHandling.DateTimeOffset; var response = client.SearchGet <dynamic>(IndexPattern, x => x.IgnoreUnavailable(true).AddQueryString("q", txtLuceneQuery.Text).AddQueryString("size", "500")); if (response.Success) { token.ThrowIfCancellationRequested(); var table = new DataTable(); var result = new DataFetchResult(table, new DataFetchContext() { TimeTaken = TimeSpan.FromMilliseconds(Convert.ToInt32(response.Body.took)), TotalHits = Convert.ToInt64(response.Body.hits.total) }); foreach (var hit in response.Body.hits.hits) { token.ThrowIfCancellationRequested(); Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString()); var row = table.NewRow(); foreach (var item in rowSource) { if (!table.Columns.Contains(item.Key)) { table.Columns.Add(item.Key); } row[item.Key] = item.Value; token.ThrowIfCancellationRequested(); } table.Rows.Add(row); } return(result); } else { throw response.OriginalException; } }
private static List<Tuple<string, string, string, DateTime>> HierarchySearch() { var node = new Uri("http://boom-box-1.boomerang.com:9200"); var config = new ConnectionConfiguration(node); var client = new ElasticLowLevelClient(config); var result = client.SearchGet<object>("hsbc_conform", "osf_hierarchies", (arg) => arg.AddQueryString("size", "100")); var root = JObject.FromObject(result.Body); var hierarchies = new List<Tuple<string, string, string, DateTime>>(); foreach (var item in root["hits"]["hits"]) { var date = item["_source"]["ctl"]["effective_date"].Value<string>(); var country = item["_source"]["conform"]["geography"].Value<string>(); var channel = item["_source"]["conform"]["channel"].Value<string>(); var parsedDate = DateTime.ParseExact(date, "yyyyMMdd", CultureInfo.InvariantCulture); var tuple = Tuple.Create(channel, country, date, parsedDate); hierarchies.Add(tuple); } return hierarchies; }
private void fetchEntireDataSet(object contextObj) { var context = (FetchFullDataSetContext)contextObj; var conf = new ConnectionConfiguration(new Uri(ClusterUrl)); var client = new ElasticLowLevelClient(conf); // First establish initial search request to get a scroll token var searchResponse = client.SearchGet <dynamic>(IndexPattern, x => x .IgnoreUnavailable(true) .AddQueryString("scroll", "1m") .AddQueryString("q", context.Query) .AddQueryString("size", "2000")); if (searchResponse.Success) { context.Token.ThrowIfCancellationRequested(); context.ScrollToken = searchResponse.Body._scroll_id; context.TotalDocuments = Convert.ToInt32(searchResponse.Body.hits.total); reportProgress(context.ProcessedDocuments, context.TotalDocuments); var table = new DataTable(); bool keepRequesting = searchResponse.Body.hits.hits.Count > 0; using (table) { foreach (var hit in searchResponse.Body.hits.hits) { context.Token.ThrowIfCancellationRequested(); Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString()); var row = table.NewRow(); foreach (var item in rowSource) { if (!table.Columns.Contains(item.Key)) { table.Columns.Add(item.Key); } if (!context.Columns.Contains(item.Key)) { context.Columns.Add(item.Key); } row[item.Key] = item.Value; context.Token.ThrowIfCancellationRequested(); } table.Rows.Add(row); } string tempFilePath = Path.GetTempFileName(); context.TempFiles.Add(tempFilePath); using (var fs = File.OpenWrite(tempFilePath)) { BinaryFormatter bf = new BinaryFormatter(); bf.AssemblyFormat = FormatterAssemblyStyle.Simple; bf.TypeFormat = FormatterTypeStyle.TypesWhenNeeded; bf.Serialize(fs, table); } context.ProcessedDocuments += table.Rows.Count; reportProgress(context.ProcessedDocuments, context.TotalDocuments); } // now subsequent request for another search do { context.Token.ThrowIfCancellationRequested(); var scrollRequest = client.Scroll <dynamic>(new { scroll = "1m", scroll_id = context.ScrollToken }); if (scrollRequest.Success) { // delete last scroll token //var clearRequest = client.ClearScroll<dynamic>(new { scroll_id = new string[] { scrollIdToken } }); // if (!clearRequest.Success) // throw clearRequest.OriginalException; keepRequesting = scrollRequest.Body.hits.hits.Count > 0; using (table = new DataTable()) { foreach (var hit in scrollRequest.Body.hits.hits) { context.Token.ThrowIfCancellationRequested(); Dictionary <string, string> rowSource = flattenDocumentStructure(hit._source.ToString()); var row = table.NewRow(); foreach (var item in rowSource) { if (!table.Columns.Contains(item.Key)) { table.Columns.Add(item.Key); } if (!context.Columns.Contains(item.Key)) { context.Columns.Add(item.Key); } row[item.Key] = item.Value; context.Token.ThrowIfCancellationRequested(); } table.Rows.Add(row); } string tempFilePath = Path.GetTempFileName(); context.TempFiles.Add(tempFilePath); using (var fs = File.OpenWrite(tempFilePath)) { BinaryFormatter bf = new BinaryFormatter(); bf.AssemblyFormat = FormatterAssemblyStyle.Simple; bf.TypeFormat = FormatterTypeStyle.TypesWhenNeeded; bf.Serialize(fs, table); } context.ProcessedDocuments += table.Rows.Count; reportProgress(context.ProcessedDocuments, context.TotalDocuments); } } else { throw scrollRequest.OriginalException; } } while (keepRequesting); } else { throw searchResponse.OriginalException; } }