public IEnumerable <IEnumerable <Adjacent> > GetOutputAdjacent(Guid id) { NumberOfRequests++; var search = _client.Search <OutputAdjacent>(s => s .Source(src => src .Includes(i => i.Fields( f => f.TargetId, f => f.Weight )) ) .Query(q => q .Match(m => m .Field(t => t.SourceId) .Query(id.ToString()) ) ) .Size(Size) .Scroll(Scroll) ).Validate(); var remaining = search.Total - search.Hits.Count; yield return(search.Documents); while (remaining > 0) { search = _client.Scroll <OutputAdjacent>(Scroll, search.ScrollId).Validate(); remaining -= search.Hits.Count; yield return(search.Documents); } _client.ClearScroll(c => c.ScrollId(search.ScrollId)).Validate(); }
public void WriteFile <TDoc>(string index, int PageSize = 10000) where TDoc : class { var docCount = _client.Count <TDoc>(c => c.Index(index)); //int Start = 0; int CurrentCount = 0; var searchResponse = _client.Search <TDoc>(s => s .Size(1000) .Scroll("1m") .Index(index)); while (searchResponse.Documents.Any()) { switch (_outputSettings.Value.Type) { case "Txt": foreach (var document in searchResponse.Documents) { //this will have to enabled after elastic read implementation // WriteFile<TDoc>(document, CurrentCount); } break; } CurrentCount++; searchResponse = _client.Scroll <TDoc>("1m", searchResponse.ScrollId); } _client.ClearScroll(c => c.ScrollId(searchResponse.ScrollId)); }
protected static List <Indberetning> GetAllDocumentsInIndex(DateTime from, DateTime to, bool index, string scrollTimeout = "2m", int scrollSize = 2000) { //The thing to know about scrollTimeout is that it resets after each call to the scroll so it only needs to be big enough to stay alive between calls. //when it expires, elastic will delete the entire scroll. var initialResponse = (index) ? UseIndex(from, to, scrollTimeout, scrollSize): UsePublic(from, to, scrollTimeout, scrollSize); List <Indberetning> results = new List <Indberetning>(); if (!initialResponse.IsValid || string.IsNullOrEmpty(initialResponse.ScrollId)) { throw new Exception(initialResponse.ServerError.Error.Reason); } if (initialResponse.Documents.Any()) { results.AddRange(initialResponse.Documents); } string scrollid = initialResponse.ScrollId; bool isScrollSetHasData = true; while (isScrollSetHasData) { ISearchResponse <Indberetning> loopingResponse = es_client.Scroll <Indberetning>(scrollTimeout, scrollid); if (loopingResponse.IsValid) { results.AddRange(loopingResponse.Documents); scrollid = loopingResponse.ScrollId; } isScrollSetHasData = loopingResponse.Documents.Any(); } //This would be garbage collected on it's own after scrollTimeout expired from it's last call but we'll clean up our room when we're done per best practice. es_client.ClearScroll(new ClearScrollRequest(scrollid)); return(results); }
/// <summary> /// 获取数据 /// </summary> /// <param name="search"></param> /// <param name="max"></param> /// <param name="isThrow"></param> /// <returns></returns> public List <T> GetList <T>( ISearchResponse <T> search, int?max = null, bool isThrow = false) where T : class { List <T> data = new List <T>(); bool Transfinite = Total <T>() > 10000; if (Transfinite) { while (true) { if (!search.IsValid || search.ScrollId == null) { if (isThrow) { throw new Exception(search.ServerError.Error.Reason); } break; } if (!search.Documents.Any_Ex()) { break; } data.AddRange(search.Documents); if (max != null && data.Count >= max) { break; } search = ElasticClient.Scroll <T>("2m", search.ScrollId); } } else { if (!search.IsValid) { if (isThrow) { throw new Exception(search.ServerError.Error.Reason); } } else { data.AddRange(search.Documents); } } ElasticClient.ClearScroll(new ClearScrollRequest(search.ScrollId)); return(data); }
public ConcurrentBag <IHit <T> > GetObjectScroll <T>(string _default_index, QueryContainer query, SourceFilter so, string scroll_timeout = "5m", int scroll_pageize = 2000) where T : class { if (query == null) { query = new MatchAllQuery(); } if (so == null) { so = new SourceFilter() { } } ; ConcurrentBag <IHit <T> > bag = new ConcurrentBag <IHit <T> >(); try { var searchResponse = client.Search <T>(sd => sd.Source(s => so).Index(_default_index).From(0).Take(scroll_pageize).Query(q => query).Scroll(scroll_timeout)); while (true) { if (!searchResponse.IsValid || string.IsNullOrEmpty(searchResponse.ScrollId)) { break; } if (!searchResponse.Documents.Any()) { break; } var tmp = searchResponse.Hits; foreach (var item in tmp) { bag.Add(item); } searchResponse = client.Scroll <T>(scroll_timeout, searchResponse.ScrollId); } client.ClearScroll(new ClearScrollRequest(searchResponse.ScrollId)); } catch (Exception) { } finally { } return(bag); }
/// <summary> /// /// </summary> /// <typeparam name="T"></typeparam> /// <param name="_esClient"></param> /// <param name="index"></param> /// <param name="type"></param> /// <param name="query"></param> /// <param name="scrollTimeout"></param> /// <param name="scrollSize"></param> /// <returns></returns> public static async Task <List <T> > GetAllDocuments <T>(this ElasticClient _esClient, string index, string type, QueryContainer query, string scrollTimeout = "2m", int scrollSize = 10000) where T : class { //The thing to know about scrollTimeout is that it resets after each call to the scroll so it only needs to be big enough to stay alive between calls. //when it expires, elastic will delete the entire scroll. ISearchResponse <T> initialResponse = await _esClient.SearchAsync <T> (scr => scr .Index(index) .Type(type) .From(0) .Take(scrollSize) .Query(q => { return(query); }) .Scroll(scrollTimeout)); List <T> results = new List <T>(); if (!initialResponse.IsValid || string.IsNullOrEmpty(initialResponse.ScrollId)) { throw new Exception(initialResponse.ServerError.Error.Reason); } if (initialResponse.Documents.Any()) { results.AddRange(initialResponse.Documents); } string scrollid = initialResponse.ScrollId; bool isScrollSetHasData = true; while (isScrollSetHasData) { ISearchResponse <T> loopingResponse = await _esClient.ScrollAsync <T>(scrollTimeout, scrollid); if (loopingResponse.IsValid) { results.AddRange(loopingResponse.Documents); scrollid = loopingResponse.ScrollId; } isScrollSetHasData = loopingResponse.Documents.Any(); } //This would be garbage collected on it's own after scrollTimeout expired from it's last call but we'll clean up our room when we're done per best practice. _esClient.ClearScroll(new ClearScrollRequest(scrollid)); return(results); }
public IList <ElasticRawLogRecord> GetLogData(LogDataFilter filter) { var takeNumber = 10000; var scrollTimeout = new Time("1m"); var retVal = new List <ElasticRawLogRecord>(); var query = BuildQuery(filter); var initialResponse = client.Search <ElasticRawLogRecord>(s => s .AllTypes() .Query(q => query) .Sort(sort => sort.Ascending($"@{nameof(ElasticRawLogRecord.Timestamp).ToLowerCamelCase()}")) .From(0) .Take(takeNumber) .Scroll(scrollTimeout) ); // Add the first batch to the result retVal.AddRange(GetLogRecords(initialResponse)); // Get the rest of the data var scrollid = initialResponse.ScrollId; var isScrollSetHasData = true; while (isScrollSetHasData) { var loopingResponse = client.Scroll <ElasticRawLogRecord>(scrollTimeout, scrollid); if (loopingResponse.IsValid) { retVal.AddRange(GetLogRecords(loopingResponse)); scrollid = loopingResponse.ScrollId; } isScrollSetHasData = loopingResponse.Documents.Any(); } client.ClearScroll(new ClearScrollRequest(scrollid)); return(retVal); }
public async Task <IEnumerable <MessageModel> > GetMessages <T>(Expression <Func <MessageModel, T> > valuePath, IEnumerable <T> values, string environmentTldCsv, int count = -1) { List <MessageModel> toReturn = new List <MessageModel>(); if (count <= 0) { count = values.Count(); } if (this.Nodes != null && this.Nodes.Any()) { IElasticClient client = new ElasticClient(new ConnectionSettings(new StaticConnectionPool(this.Nodes.Select(n => new Uri(n.Trim())).ToArray())) .EnableDebugMode() .DisableDirectStreaming() .PrettyJson()); for (int idx = 0; idx *this.MaxValuesPerQuery < count; idx++) { try { ISearchResponse <MessageModel> resp = await client.SearchAsync <MessageModel>(s => s.Index("*gwar*") .From(0) .Size(Math.Min(this.MaxValuesPerQuery, count)) .Query(q => this.GenerateQuery(valuePath, values.Skip(idx * this.MaxValuesPerQuery).Take(this.MaxValuesPerQuery), environmentTldCsv)) .Sort(ss => ss.Descending("@timestamp"))); while (resp.Documents.Any()) { toReturn.AddRange(resp.Documents); resp = client.Scroll <MessageModel>("10m", resp.ScrollId); } client.ClearScroll(new ClearScrollRequest(resp.ScrollId)); } catch (UnexpectedElasticsearchClientException) { } } } return(toReturn); }
public async Task <ApiResult> WriteFileAsync <TDoc>() where TDoc : class { try { //Log message if directory not found and return if (!Directory.Exists(Path.GetDirectoryName(_outputSettings.Value.OutputFilePath))) { _logger.LogError("WriteFileAsync - No Directory found with given path- POFineLine: {Reason}", _outputSettings.Value.OutputFilePath); return(new ApiResult <string>()); } string path = GetOutputFilePath(); if (string.IsNullOrEmpty(path)) { _logger.LogError("WriteFileAsync - Output path not specified."); return(new ApiResult <string>()); } bool atLeastOneRecord = false; //reads for that day from elastic //reads in chunks of 100, can be configured var docCount = _client.Count <TDoc>(c => c.Index(_currentIndex)); _logger.LogInformation($"WriteFileAsync - Inspecting {docCount.Count} documents for changes..."); var searchResponse = _client.Search <TDoc>(s => s .Size(100) .Scroll("1m") .Index(_currentIndex)); var outputType = _outputSettings.Value.Type.ToLower(); switch (outputType) { case "text": case "txt": while (searchResponse.Documents.Any()) { foreach (var document in searchResponse.Documents) { var pofloutputcurrent = document as POFineLineOutput; //If we're supposed to include this no matter what, just do so if (pofloutputcurrent.ForceInclude) { if (pofloutputcurrent.POSkus != null && pofloutputcurrent.POSkus.Count > 0) { pofloutputcurrent.POSkus.ForEach(posku => { posku.ActivityCode = ""; }); WriteFile <POFineLineOutput>(pofloutputcurrent, path); atLeastOneRecord = true; pofloutputcurrent.ForceInclude = false; var response = await _client.UpdateAsync <POFineLineOutput, object>(pofloutputcurrent, u => u.Doc(pofloutputcurrent).Index(_currentIndex).DocAsUpsert()); } else { _logger.LogInformation($"WriteFileAsync - Skipping output for PO - {pofloutputcurrent.PurchaseOrder}; no SKUs were found."); } } else { //Read from previous index for the document and compare with the current. var doc = await _client.GetAsync <TDoc>(new DocumentPath <TDoc>(pofloutputcurrent.PurchaseOrder), g => g.Index(_previouIndex)); if (doc.Source == null) { //PO data does not exist in previous, so send var podatanew = document as POFineLineOutput; if (podatanew.POSkus != null && podatanew.POSkus.Count > 0) { _logger.LogInformation($"WriteFileAsync - Outputting {pofloutputcurrent.PurchaseOrder} to file as new purchase order."); podatanew.POSkus.ForEach(x => { x.ActivityCode = ""; }); WriteFile <POFineLineOutput>(podatanew, path); atLeastOneRecord = true; } else { _logger.LogInformation($"WriteFileAsync - Skipping output for PO - {podatanew.PurchaseOrder}; no SKUs were found."); } } else { //See if there is any difference between PO and or the POskus; if so, send the current var pofloutputprev = doc.Source as POFineLineOutput; var compareddata = _pofinelineutilities.CompareFineLineData(pofloutputcurrent, pofloutputprev, _outputSettings.Value.MembersToInclude); if (compareddata != null && compareddata.POSkus != null && compareddata.POSkus.Count > 0) { _logger.LogInformation($"WriteFileAsync - Outputting changes to {pofloutputcurrent.PurchaseOrder} to file."); WriteFile <POFineLineOutput>(compareddata, path); atLeastOneRecord = true; } else { _logger.LogInformation($"WriteFileAsync - Skipping output for {pofloutputcurrent.PurchaseOrder}; no changes were found."); } } } if (atLeastOneRecord) { //save it to archive index before delete from current var archiveresponse = await _client.UpdateAsync <POFineLineOutput, object>(pofloutputcurrent, u => u.Doc(pofloutputcurrent).Index(_archiveIndex).DocAsUpsert()); //Save to Previous Index from Current var previousresponse = await _client.UpdateAsync <POFineLineOutput, object>(pofloutputcurrent, u => u.Doc(pofloutputcurrent).Index(_previouIndex).DocAsUpsert()); //delete from current var deletedcurrent = await _client.DeleteAsync <POFineLineOutput>(pofloutputcurrent, u => u.Index(_currentIndex)); _logger.LogInformation($"WriteFileAsync - Archived Fineline Index Successfully -- {pofloutputcurrent.PurchaseOrder}."); } else { _logger.LogInformation($"WriteFileAsync - No file updated. So, did not Archived Fineline Index -- {pofloutputcurrent.PurchaseOrder}."); } _logger.LogInformation($"WriteFileAsync - Successfully Processed PONumber -- {pofloutputcurrent.PurchaseOrder} ."); } //CurrentCount++; searchResponse = _client.Scroll <TDoc>("1m", searchResponse.ScrollId); } break; default: _logger.LogError($"WriteFileAsync - Invalid output type {outputType} specified."); return(new ApiResult <string>()); } _client.ClearScroll(c => c.ScrollId(searchResponse.ScrollId)); //Make sure the file was actually written to disk before finalizing if (atLeastOneRecord && !File.Exists(path)) { _logger.LogError($"WriteFileAsync - Output failed to write successfully to {path}"); } //else { // ReIndex<TDoc>(); //} return(new ApiResult <string>()); } catch (Exception ex) { _logger.LogError("Method Name -- WriteFileAsync: {Reason}", ex.Message); return(new ApiResult <string>(new[] { ex.Message })); } }
public static void DoActionForQuery <T>(ElasticClient client, Func <int, int, ISearchResponse <T> > searchFunc, System.Func <IHit <T>, object, Devmasters.Core.Batch.ActionOutputData> action, object actionParameters, System.Action <string> logOutputFunc, System.Action <Devmasters.Core.Batch.ActionProgressData> progressOutputFunc, bool parallel, int blockSize = 500, int?maxDegreeOfParallelism = null, string prefix = "" ) where T : class { DateTime started = DateTime.Now; long total = 0; int currIteration = 0; int processedCount = 0; ISearchResponse <T> result = default(ISearchResponse <T>); //var total = NoveInzeraty.Lib.Data.Smlouva.Search.NumberOfDocs(null, null); string scrollId = null; //create scroll search context bool firstResult = true; if (maxDegreeOfParallelism <= 1) { parallel = false; } try { result = searchFunc(blockSize, currIteration); if (result.IsValid == false) { Lib.ES.Manager.LogQueryError <T>(result); } } catch (Exception e1) { System.Threading.Thread.Sleep(10000); try { result = searchFunc(blockSize, currIteration); } catch (Exception e2) { System.Threading.Thread.Sleep(20000); try { result = searchFunc(blockSize, currIteration); } catch (Exception ex) { HlidacStatu.Util.Consts.Logger.Error("Cannot read data from Elastic, skipping iteration" + currIteration, ex); return; } } } scrollId = result.ScrollId; do { DateTime iterationStart = DateTime.Now; if (firstResult) { firstResult = false; } else { result = client.Scroll <T>(ScrollLifeTime, scrollId); scrollId = result.ScrollId; } currIteration++; if (result.Hits.Count() == 0) { break; } total = result.Total; bool canceled = false; if (parallel) { CancellationTokenSource cts = new CancellationTokenSource(); try { ParallelOptions pOptions = new ParallelOptions(); if (maxDegreeOfParallelism.HasValue) { pOptions.MaxDegreeOfParallelism = maxDegreeOfParallelism.Value; } pOptions.CancellationToken = cts.Token; Parallel.ForEach(result.Hits, (hit) => { if (action != null) { ActionOutputData cancel = null; try { cancel = action(hit, actionParameters); System.Threading.Interlocked.Increment(ref processedCount); if (logOutputFunc != null && !string.IsNullOrEmpty(cancel.Log)) { logOutputFunc(cancel.Log); } if (cancel.CancelRunning) { cts.Cancel(); } } catch (Exception e) { HlidacStatu.Util.Consts.Logger.Error("DoActionForAll action error", e); cts.Cancel(); } } if (progressOutputFunc != null) { ActionProgressData apd = new ActionProgressData(total, processedCount, started, prefix); progressOutputFunc(apd); } }); } catch (OperationCanceledException e) { //Catestrophic Failure canceled = true; } } else { foreach (var hit in result.Hits) { if (action != null) { ActionOutputData cancel = action(hit, actionParameters); System.Threading.Interlocked.Increment(ref processedCount); if (logOutputFunc != null && !string.IsNullOrEmpty(cancel.Log)) { logOutputFunc(cancel.Log); } if (cancel.CancelRunning) { canceled = true; break; } } if (progressOutputFunc != null) { ActionProgressData apd = new ActionProgressData(total, processedCount, started, prefix); progressOutputFunc(apd); } } } if (canceled) { break; } } while (result.Hits.Count() > 0); client.ClearScroll(c => c.ScrollId(scrollId)); if (logOutputFunc != null) { logOutputFunc("Done"); } }
//public ConcurrentBag<T> GetObjectScroll<T>(QueryContainer qc, SourceFilter so, List<ISort> lSort = null) where T : class //{ // return GetObjectScroll<T>(_DefaultIndex, qc, so, lSort); //} protected ConcurrentBag <T> GetObjectScroll <T>(string _DefaultIndex, QueryContainer query, SourceFilter so, List <ISort> lSort = null, string routing = "", int scroll_pageize = 2000) where T : class { try { string scroll_timeout = "5m"; //int scroll_pageize = 2000; var seenSlices = new ConcurrentBag <int>(); ConcurrentBag <T> bag = new ConcurrentBag <T>(); if (query == null) { query = new MatchAllQuery(); } if (so == null) { so = new SourceFilter() { } } ; try { var searchResponse = client.Search <T>(sd => sd.Source(s => so).Index(_DefaultIndex).From(0).Take(scroll_pageize).Query(q => query).Scroll(scroll_timeout)); while (true) { if (!searchResponse.IsValid || string.IsNullOrEmpty(searchResponse.ScrollId)) { break; } if (!searchResponse.Documents.Any()) { break; } var tmp = searchResponse.Hits; Parallel.ForEach(tmp, new ParallelOptions { MaxDegreeOfParallelism = 4 }, item => { var doc = HitToDocument(item); bag.Add(doc); }); /* * foreach (var item in tmp) * { * var doc = HitToDocument(item); * bag.Add(doc); * } */ searchResponse = client.Scroll <T>(scroll_timeout, searchResponse.ScrollId); } client.ClearScroll(new ClearScrollRequest(searchResponse.ScrollId)); } catch (Exception) { } finally { } return(bag); } catch { } return(null); }
/// <summary> /// 查询(sql) /// </summary> /// <param name="recordCount">总数据量(不分页)</param> /// <param name="indices">指定索引(null:默认,[]:全部,["i_0","i_1","i_2"]:指定)</param> /// <param name="query">sql查询语句</param> /// <param name="size">数据量</param> /// <param name="time">快照保存时间(秒,默认60)</param> /// <param name="isThrow">抛出异常</param> /// <returns></returns> public List <T> SearchWithSql <T>( out long recordCount, string[] indices = null, string query = null, int?size = null, int time = 60, bool isThrow = false) where T : class { //if (pageIndex != null && pageIndex * pageRows > 10000) // throw new Exception("数据量过大"); var indices_ = GetIndices(indices); var _time = TimeSpan.FromSeconds(time); List <T> data = new List <T>(); var sql = $"select * from {GetIndex(indices)} {query}"; if (size != null) { sql += $" limit {size}"; } bool Transfinite = Total(indices_) > 10000; var request = new TranslateSqlRequest() { Query = sql, FetchSize = size }; var respone_sql = ElasticClient.LowLevel.Sql.Translate <StringResponse>(PostData.Serializable(request)); var query_Dsl = respone_sql.Body; var requestParameters = Transfinite ? new SearchRequestParameters() { Scroll = _time } : null; ISearchResponse <T> respone_search = ElasticClient.LowLevel.Search <SearchResponse <T> >(GetIndex(indices), query_Dsl, requestParameters); recordCount = respone_search.Total; if (Transfinite) { while (true) { if (!respone_search.IsValid || respone_search.ScrollId == null) { if (isThrow) { throw new Exception(respone_search.ServerError.Error.Reason); } break; } if (!respone_search.Documents.Any_Ex()) { break; } data.AddRange(respone_search.Documents); respone_search = ElasticClient.Scroll <T>(_time, respone_search.ScrollId); } } else { if (!respone_search.IsValid) { if (isThrow) { throw new Exception(respone_search.ServerError.Error.Reason); } } else { data.AddRange(respone_search.Documents); } } ElasticClient.ClearScroll(new ClearScrollRequest(respone_search.ScrollId)); return(data); }
/// <summary> /// 查询(分页) /// </summary> /// <param name="recordCount">总数据量(不分页)</param> /// <param name="indices">指定索引(null:默认,[]:全部,["i_0","i_1","i_2"]:指定)</param> /// <param name="query"></param> /// <param name="sort">排序</param> /// <param name="pageIndex">指定页码</param>//(仅支持在数据量小于等于10000时进行分页) /// <param name="pageRows">每页数据量</param> /// <param name="time">快照保存时间(秒,默认60)</param> /// <param name="isThrow">抛出异常</param> /// <returns></returns> public List <T> SearchPaging <T>( out long recordCount, string[] indices = null, Func <QueryContainerDescriptor <T>, QueryContainer> query = null, Func <SortDescriptor <T>, IPromise <IList <ISort> > > sort = null, int?pageIndex = null, int?pageRows = null, int time = 60, bool isThrow = false) where T : class { //if (pageIndex != null && pageIndex * pageRows > 10000) // throw new Exception("数据量过大"); var indices_ = GetIndices(indices); var _time = TimeSpan.FromSeconds(time); List <T> data = new List <T>(); bool Transfinite = false; var response = ElasticClient.Search <T>(s => { s = s.Index(indices_); if (query != null) { s = s.Query(query); } else { s = s.MatchAll(); } if (pageIndex != null) { s = s.Size(pageRows) .From((pageIndex - 1) * pageRows); } else { Transfinite = Total <T>() > 10000; } if (sort != null) { s = s.Sort(sort); } if (Transfinite) { s = s.Scroll(_time); } return(s); }); recordCount = response.Total; if (Transfinite) { while (true) { if (!response.IsValid || response.ScrollId == null) { if (isThrow) { throw new ElasticsearchException(response); } break; } if (!response.Documents.Any_Ex()) { break; } data.AddRange(response.Documents); response = ElasticClient.Scroll <T>(_time, response.ScrollId); } } else { if (!response.IsValid) { if (isThrow) { throw new ElasticsearchException(response); } } else { data.AddRange(response.Documents); } } ElasticClient.ClearScroll(new ClearScrollRequest(response.ScrollId)); return(data); }
/// <summary> /// 查询(游标) /// </summary> /// <param name="indices">指定索引(null:默认,[]:全部,["i_0","i_1","i_2"]:指定)</param> /// <param name="query"></param> /// <param name="scrollId">滚动ID</param> /// <param name="time">快照保存时间(秒,默认60)</param> /// <param name="size">数据量上限</param> /// <param name="sortField">排序字段</param> /// <param name="sortType">排序类型(desc,asc[默认])</param> /// <param name="isThrow">抛出异常</param> /// <returns></returns> public List <T> SearchScroll <T>( string[] indices = null, Func <QueryContainerDescriptor <T>, QueryContainer> query = null, string scrollId = null, int time = 60, int?size = null, string sortField = null, string sortType = "asc", bool isThrow = false) where T : class { var indices_ = GetIndices(indices); string _time = ConvertTime(time); List <T> data = new List <T>(); bool Transfinite = size > 10000 && Total <T>() > 10000; var response = scrollId != null?ElasticClient.Scroll <T>(_time, scrollId) : ElasticClient.Search <T>(s => { s = s.Index(indices_); if (query != null) { s = s.Query(query); } else { s = s.MatchAll(); } if (size != null) { s = s.Size(size) .Sort(so => so.Field(typeof(T).GetProperty(sortField), sortType == "asc" ? SortOrder.Ascending : SortOrder.Descending)); } if (Transfinite) { s = s.Scroll(_time); } return(s); }); if (Transfinite) { while (true) { if (!response.IsValid || response.ScrollId == null) { if (isThrow) { throw new ElasticsearchException(response); } break; } if (!response.Documents.Any_Ex()) { break; } data.AddRange(response.Documents); response = ElasticClient.Scroll <T>(_time, response.ScrollId); } } else { if (!response.IsValid) { if (isThrow) { throw new ElasticsearchException(response); } } else { data.AddRange(response.Documents); } } ElasticClient.ClearScroll(new ClearScrollRequest(response.ScrollId)); return(data); }
public async Task <ApiResult> WriteFileAsync <TDoc>(string index, int PageSize = 10000) where TDoc : class { int CurrentCount = 0; //reads for that day from elastic //reads in chunks of 100, can be changed var docCount = _client.Count <TDoc>(c => c.Index(_currentIndex)); _logger.LogInformation($"WriteFileAsync - Inspecting {docCount.Count} documents for changes..."); var searchResponse = _client.Search <TDoc>(s => s .Size(100) .Scroll("1m") .Index(_currentIndex)); List <POChinoOutput> finalChinoOutput = new List <POChinoOutput>(); var outputType = _outputSettings.Value.Type.ToLower(); while (searchResponse.Documents.Any()) { switch (outputType) { case "xml": foreach (var document in searchResponse.Documents) { var pochinooutputcurrent = document as POChinoOutput; if (pochinooutputcurrent.ForceInclude) { finalChinoOutput.Add(pochinooutputcurrent); } else { //Only send the PO if it's 'Open' if (pochinooutputcurrent.StatusCode == "OP") { //Read from previous index for the document. If sent before, we don't send it again var doc = await _client.GetAsync <TDoc>(new DocumentPath <TDoc>(pochinooutputcurrent.OrderId), g => g.Index(_previousIndex)); if (doc.Source == null) { var podatanew = document as POChinoOutput; finalChinoOutput.Add(podatanew); } } } //save it to archive index before delete from current var archiveresponse = await _client.UpdateAsync <POChinoOutput, object>(pochinooutputcurrent, u => u.Doc(pochinooutputcurrent).Index(_archiveIndex).DocAsUpsert()); //Save to Previous Index from Current var previousresponse = await _client.UpdateAsync <POChinoOutput, object>(pochinooutputcurrent, u => u.Doc(pochinooutputcurrent).Index(_previousIndex).DocAsUpsert()); //delete from current var deletedcurrent = await _client.DeleteAsync <POChinoOutput>(pochinooutputcurrent, u => u.Index(_currentIndex)); _logger.LogInformation($"WriteFileAsync - Successfully Processed PONumber -- {pochinooutputcurrent.OrderId} ."); } _poChinoutilities.WriteChinoFile(finalChinoOutput); break; default: _logger.LogError($"WriteFileAsync - Invalid output type {outputType} specified."); return(new ApiResult <string>()); } CurrentCount++; searchResponse = _client.Scroll <TDoc>("1m", searchResponse.ScrollId); } _client.ClearScroll(c => c.ScrollId(searchResponse.ScrollId)); //ReIndex<TDoc>(); return(new ApiResult <string>()); }