public IEnumerable <IEnumerable <Adjacent> > GetOutputAdjacent(Guid id)
{
NumberOfRequests++;
var search = _client.Search <OutputAdjacent>(s => s
.Source(src => src
.Includes(i => i.Fields(
f => f.TargetId,
f => f.Weight
))
)
.Query(q => q
.Match(m => m
.Field(t => t.SourceId)
.Query(id.ToString())
)
)
.Size(Size)
.Scroll(Scroll)
).Validate();
var remaining = search.Total - search.Hits.Count;
yield return(search.Documents);
while (remaining > 0)
{
search = _client.Scroll <OutputAdjacent>(Scroll, search.ScrollId).Validate();
remaining -= search.Hits.Count;
yield return(search.Documents);
}
_client.ClearScroll(c => c.ScrollId(search.ScrollId)).Validate();
}
public void WriteFile <TDoc>(string index, int PageSize = 10000) where TDoc : class
{
var docCount = _client.Count <TDoc>(c => c.Index(index));
//int Start = 0;
int CurrentCount = 0;
var searchResponse = _client.Search <TDoc>(s => s
.Size(1000)
.Scroll("1m")
.Index(index));
while (searchResponse.Documents.Any())
{
switch (_outputSettings.Value.Type)
{
case "Txt":
foreach (var document in searchResponse.Documents)
{
//this will have to enabled after elastic read implementation
// WriteFile<TDoc>(document, CurrentCount);
}
break;
}
CurrentCount++;
searchResponse = _client.Scroll <TDoc>("1m", searchResponse.ScrollId);
}
_client.ClearScroll(c => c.ScrollId(searchResponse.ScrollId));
}
protected static List <Indberetning> GetAllDocumentsInIndex(DateTime from, DateTime to, bool index, string scrollTimeout = "2m", int scrollSize = 2000)
{
//The thing to know about scrollTimeout is that it resets after each call to the scroll so it only needs to be big enough to stay alive between calls.
//when it expires, elastic will delete the entire scroll.
var initialResponse = (index) ? UseIndex(from, to, scrollTimeout, scrollSize): UsePublic(from, to, scrollTimeout, scrollSize);
List <Indberetning> results = new List <Indberetning>();
if (!initialResponse.IsValid || string.IsNullOrEmpty(initialResponse.ScrollId))
{
throw new Exception(initialResponse.ServerError.Error.Reason);
}
if (initialResponse.Documents.Any())
{
results.AddRange(initialResponse.Documents);
}
string scrollid = initialResponse.ScrollId;
bool isScrollSetHasData = true;
while (isScrollSetHasData)
{
ISearchResponse <Indberetning> loopingResponse = es_client.Scroll <Indberetning>(scrollTimeout, scrollid);
if (loopingResponse.IsValid)
{
results.AddRange(loopingResponse.Documents);
scrollid = loopingResponse.ScrollId;
}
isScrollSetHasData = loopingResponse.Documents.Any();
}
//This would be garbage collected on it's own after scrollTimeout expired from it's last call but we'll clean up our room when we're done per best practice.
es_client.ClearScroll(new ClearScrollRequest(scrollid));
return(results);
}
/// <summary>
/// 获取数据
/// </summary>
/// <param name="search"></param>
/// <param name="max"></param>
/// <param name="isThrow"></param>
/// <returns></returns>
public List <T> GetList <T>(
ISearchResponse <T> search,
int?max = null,
bool isThrow = false) where T : class
{
List <T> data = new List <T>();
bool Transfinite = Total <T>() > 10000;
if (Transfinite)
{
while (true)
{
if (!search.IsValid || search.ScrollId == null)
{
if (isThrow)
{
throw new Exception(search.ServerError.Error.Reason);
}
break;
}
if (!search.Documents.Any_Ex())
{
break;
}
data.AddRange(search.Documents);
if (max != null && data.Count >= max)
{
break;
}
search = ElasticClient.Scroll <T>("2m", search.ScrollId);
}
}
else
{
if (!search.IsValid)
{
if (isThrow)
{
throw new Exception(search.ServerError.Error.Reason);
}
}
else
{
data.AddRange(search.Documents);
}
}
ElasticClient.ClearScroll(new ClearScrollRequest(search.ScrollId));
return(data);
}
public ConcurrentBag <IHit <T> > GetObjectScroll <T>(string _default_index, QueryContainer query, SourceFilter so, string scroll_timeout = "5m", int scroll_pageize = 2000) where T : class
{
if (query == null)
{
query = new MatchAllQuery();
}
if (so == null)
{
so = new SourceFilter()
{
}
}
;
ConcurrentBag <IHit <T> > bag = new ConcurrentBag <IHit <T> >();
try
{
var searchResponse = client.Search <T>(sd => sd.Source(s => so).Index(_default_index).From(0).Take(scroll_pageize).Query(q => query).Scroll(scroll_timeout));
while (true)
{
if (!searchResponse.IsValid || string.IsNullOrEmpty(searchResponse.ScrollId))
{
break;
}
if (!searchResponse.Documents.Any())
{
break;
}
var tmp = searchResponse.Hits;
foreach (var item in tmp)
{
bag.Add(item);
}
searchResponse = client.Scroll <T>(scroll_timeout, searchResponse.ScrollId);
}
client.ClearScroll(new ClearScrollRequest(searchResponse.ScrollId));
}
catch (Exception)
{
}
finally
{
}
return(bag);
}
/// <summary>
///
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="_esClient"></param>
/// <param name="index"></param>
/// <param name="type"></param>
/// <param name="query"></param>
/// <param name="scrollTimeout"></param>
/// <param name="scrollSize"></param>
/// <returns></returns>
public static async Task <List <T> > GetAllDocuments <T>(this ElasticClient _esClient, string index, string type, QueryContainer query, string scrollTimeout = "2m", int scrollSize = 10000) where T : class
{
//The thing to know about scrollTimeout is that it resets after each call to the scroll so it only needs to be big enough to stay alive between calls.
//when it expires, elastic will delete the entire scroll.
ISearchResponse <T> initialResponse = await _esClient.SearchAsync <T>
(scr => scr
.Index(index)
.Type(type)
.From(0)
.Take(scrollSize)
.Query(q => { return(query); })
.Scroll(scrollTimeout));
List <T> results = new List <T>();
if (!initialResponse.IsValid || string.IsNullOrEmpty(initialResponse.ScrollId))
{
throw new Exception(initialResponse.ServerError.Error.Reason);
}
if (initialResponse.Documents.Any())
{
results.AddRange(initialResponse.Documents);
}
string scrollid = initialResponse.ScrollId;
bool isScrollSetHasData = true;
while (isScrollSetHasData)
{
ISearchResponse <T> loopingResponse = await _esClient.ScrollAsync <T>(scrollTimeout, scrollid);
if (loopingResponse.IsValid)
{
results.AddRange(loopingResponse.Documents);
scrollid = loopingResponse.ScrollId;
}
isScrollSetHasData = loopingResponse.Documents.Any();
}
//This would be garbage collected on it's own after scrollTimeout expired from it's last call but we'll clean up our room when we're done per best practice.
_esClient.ClearScroll(new ClearScrollRequest(scrollid));
return(results);
}
public IList <ElasticRawLogRecord> GetLogData(LogDataFilter filter)
{
var takeNumber = 10000;
var scrollTimeout = new Time("1m");
var retVal = new List <ElasticRawLogRecord>();
var query = BuildQuery(filter);
var initialResponse = client.Search <ElasticRawLogRecord>(s => s
.AllTypes()
.Query(q => query)
.Sort(sort => sort.Ascending($"@{nameof(ElasticRawLogRecord.Timestamp).ToLowerCamelCase()}"))
.From(0)
.Take(takeNumber)
.Scroll(scrollTimeout)
);
// Add the first batch to the result
retVal.AddRange(GetLogRecords(initialResponse));
// Get the rest of the data
var scrollid = initialResponse.ScrollId;
var isScrollSetHasData = true;
while (isScrollSetHasData)
{
var loopingResponse = client.Scroll <ElasticRawLogRecord>(scrollTimeout, scrollid);
if (loopingResponse.IsValid)
{
retVal.AddRange(GetLogRecords(loopingResponse));
scrollid = loopingResponse.ScrollId;
}
isScrollSetHasData = loopingResponse.Documents.Any();
}
client.ClearScroll(new ClearScrollRequest(scrollid));
return(retVal);
}
public async Task <IEnumerable <MessageModel> > GetMessages <T>(Expression <Func <MessageModel, T> > valuePath, IEnumerable <T> values, string environmentTldCsv, int count = -1)
{
List <MessageModel> toReturn = new List <MessageModel>();
if (count <= 0)
{
count = values.Count();
}
if (this.Nodes != null && this.Nodes.Any())
{
IElasticClient client = new ElasticClient(new ConnectionSettings(new StaticConnectionPool(this.Nodes.Select(n => new Uri(n.Trim())).ToArray()))
.EnableDebugMode()
.DisableDirectStreaming()
.PrettyJson());
for (int idx = 0; idx *this.MaxValuesPerQuery < count; idx++)
{
try
{
ISearchResponse <MessageModel> resp = await client.SearchAsync <MessageModel>(s =>
s.Index("*gwar*")
.From(0)
.Size(Math.Min(this.MaxValuesPerQuery, count))
.Query(q => this.GenerateQuery(valuePath, values.Skip(idx * this.MaxValuesPerQuery).Take(this.MaxValuesPerQuery), environmentTldCsv))
.Sort(ss => ss.Descending("@timestamp")));
while (resp.Documents.Any())
{
toReturn.AddRange(resp.Documents);
resp = client.Scroll <MessageModel>("10m", resp.ScrollId);
}
client.ClearScroll(new ClearScrollRequest(resp.ScrollId));
}
catch (UnexpectedElasticsearchClientException) { }
}
}
return(toReturn);
}
public async Task <ApiResult> WriteFileAsync <TDoc>() where TDoc : class
{
try
{
//Log message if directory not found and return
if (!Directory.Exists(Path.GetDirectoryName(_outputSettings.Value.OutputFilePath)))
{
_logger.LogError("WriteFileAsync - No Directory found with given path- POFineLine: {Reason}", _outputSettings.Value.OutputFilePath);
return(new ApiResult <string>());
}
string path = GetOutputFilePath();
if (string.IsNullOrEmpty(path))
{
_logger.LogError("WriteFileAsync - Output path not specified.");
return(new ApiResult <string>());
}
bool atLeastOneRecord = false;
//reads for that day from elastic
//reads in chunks of 100, can be configured
var docCount = _client.Count <TDoc>(c => c.Index(_currentIndex));
_logger.LogInformation($"WriteFileAsync - Inspecting {docCount.Count} documents for changes...");
var searchResponse = _client.Search <TDoc>(s => s
.Size(100)
.Scroll("1m")
.Index(_currentIndex));
var outputType = _outputSettings.Value.Type.ToLower();
switch (outputType)
{
case "text":
case "txt":
while (searchResponse.Documents.Any())
{
foreach (var document in searchResponse.Documents)
{
var pofloutputcurrent = document as POFineLineOutput;
//If we're supposed to include this no matter what, just do so
if (pofloutputcurrent.ForceInclude)
{
if (pofloutputcurrent.POSkus != null && pofloutputcurrent.POSkus.Count > 0)
{
pofloutputcurrent.POSkus.ForEach(posku =>
{
posku.ActivityCode = "";
});
WriteFile <POFineLineOutput>(pofloutputcurrent, path);
atLeastOneRecord = true;
pofloutputcurrent.ForceInclude = false;
var response = await _client.UpdateAsync <POFineLineOutput, object>(pofloutputcurrent, u => u.Doc(pofloutputcurrent).Index(_currentIndex).DocAsUpsert());
}
else
{
_logger.LogInformation($"WriteFileAsync - Skipping output for PO - {pofloutputcurrent.PurchaseOrder}; no SKUs were found.");
}
}
else
{
//Read from previous index for the document and compare with the current.
var doc = await _client.GetAsync <TDoc>(new DocumentPath <TDoc>(pofloutputcurrent.PurchaseOrder), g => g.Index(_previouIndex));
if (doc.Source == null)
{
//PO data does not exist in previous, so send
var podatanew = document as POFineLineOutput;
if (podatanew.POSkus != null && podatanew.POSkus.Count > 0)
{
_logger.LogInformation($"WriteFileAsync - Outputting {pofloutputcurrent.PurchaseOrder} to file as new purchase order.");
podatanew.POSkus.ForEach(x => { x.ActivityCode = ""; });
WriteFile <POFineLineOutput>(podatanew, path);
atLeastOneRecord = true;
}
else
{
_logger.LogInformation($"WriteFileAsync - Skipping output for PO - {podatanew.PurchaseOrder}; no SKUs were found.");
}
}
else
{
//See if there is any difference between PO and or the POskus; if so, send the current
var pofloutputprev = doc.Source as POFineLineOutput;
var compareddata = _pofinelineutilities.CompareFineLineData(pofloutputcurrent, pofloutputprev, _outputSettings.Value.MembersToInclude);
if (compareddata != null && compareddata.POSkus != null && compareddata.POSkus.Count > 0)
{
_logger.LogInformation($"WriteFileAsync - Outputting changes to {pofloutputcurrent.PurchaseOrder} to file.");
WriteFile <POFineLineOutput>(compareddata, path);
atLeastOneRecord = true;
}
else
{
_logger.LogInformation($"WriteFileAsync - Skipping output for {pofloutputcurrent.PurchaseOrder}; no changes were found.");
}
}
}
if (atLeastOneRecord)
{
//save it to archive index before delete from current
var archiveresponse = await _client.UpdateAsync <POFineLineOutput, object>(pofloutputcurrent, u => u.Doc(pofloutputcurrent).Index(_archiveIndex).DocAsUpsert());
//Save to Previous Index from Current
var previousresponse = await _client.UpdateAsync <POFineLineOutput, object>(pofloutputcurrent, u => u.Doc(pofloutputcurrent).Index(_previouIndex).DocAsUpsert());
//delete from current
var deletedcurrent = await _client.DeleteAsync <POFineLineOutput>(pofloutputcurrent, u => u.Index(_currentIndex));
_logger.LogInformation($"WriteFileAsync - Archived Fineline Index Successfully -- {pofloutputcurrent.PurchaseOrder}.");
}
else
{
_logger.LogInformation($"WriteFileAsync - No file updated. So, did not Archived Fineline Index -- {pofloutputcurrent.PurchaseOrder}.");
}
_logger.LogInformation($"WriteFileAsync - Successfully Processed PONumber -- {pofloutputcurrent.PurchaseOrder} .");
}
//CurrentCount++;
searchResponse = _client.Scroll <TDoc>("1m", searchResponse.ScrollId);
}
break;
default:
_logger.LogError($"WriteFileAsync - Invalid output type {outputType} specified.");
return(new ApiResult <string>());
}
_client.ClearScroll(c => c.ScrollId(searchResponse.ScrollId));
//Make sure the file was actually written to disk before finalizing
if (atLeastOneRecord && !File.Exists(path))
{
_logger.LogError($"WriteFileAsync - Output failed to write successfully to {path}");
}
//else {
// ReIndex<TDoc>();
//}
return(new ApiResult <string>());
}
catch (Exception ex)
{
_logger.LogError("Method Name -- WriteFileAsync: {Reason}", ex.Message);
return(new ApiResult <string>(new[] { ex.Message }));
}
}
public static void DoActionForQuery <T>(ElasticClient client,
Func <int, int, ISearchResponse <T> > searchFunc,
System.Func <IHit <T>, object, Devmasters.Core.Batch.ActionOutputData> action, object actionParameters,
System.Action <string> logOutputFunc,
System.Action <Devmasters.Core.Batch.ActionProgressData> progressOutputFunc,
bool parallel,
int blockSize = 500, int?maxDegreeOfParallelism = null, string prefix = ""
)
where T : class
{
DateTime started = DateTime.Now;
long total = 0;
int currIteration = 0;
int processedCount = 0;
ISearchResponse <T> result = default(ISearchResponse <T>);
//var total = NoveInzeraty.Lib.Data.Smlouva.Search.NumberOfDocs(null, null);
string scrollId = null;
//create scroll search context
bool firstResult = true;
if (maxDegreeOfParallelism <= 1)
{
parallel = false;
}
try
{
result = searchFunc(blockSize, currIteration);
if (result.IsValid == false)
{
Lib.ES.Manager.LogQueryError <T>(result);
}
}
catch (Exception e1)
{
System.Threading.Thread.Sleep(10000);
try
{
result = searchFunc(blockSize, currIteration);
}
catch (Exception e2)
{
System.Threading.Thread.Sleep(20000);
try
{
result = searchFunc(blockSize, currIteration);
}
catch (Exception ex)
{
HlidacStatu.Util.Consts.Logger.Error("Cannot read data from Elastic, skipping iteration" + currIteration, ex);
return;
}
}
}
scrollId = result.ScrollId;
do
{
DateTime iterationStart = DateTime.Now;
if (firstResult)
{
firstResult = false;
}
else
{
result = client.Scroll <T>(ScrollLifeTime, scrollId);
scrollId = result.ScrollId;
}
currIteration++;
if (result.Hits.Count() == 0)
{
break;
}
total = result.Total;
bool canceled = false;
if (parallel)
{
CancellationTokenSource cts = new CancellationTokenSource();
try
{
ParallelOptions pOptions = new ParallelOptions();
if (maxDegreeOfParallelism.HasValue)
{
pOptions.MaxDegreeOfParallelism = maxDegreeOfParallelism.Value;
}
pOptions.CancellationToken = cts.Token;
Parallel.ForEach(result.Hits, (hit) =>
{
if (action != null)
{
ActionOutputData cancel = null;
try
{
cancel = action(hit, actionParameters);
System.Threading.Interlocked.Increment(ref processedCount);
if (logOutputFunc != null && !string.IsNullOrEmpty(cancel.Log))
{
logOutputFunc(cancel.Log);
}
if (cancel.CancelRunning)
{
cts.Cancel();
}
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error("DoActionForAll action error", e);
cts.Cancel();
}
}
if (progressOutputFunc != null)
{
ActionProgressData apd = new ActionProgressData(total, processedCount, started, prefix);
progressOutputFunc(apd);
}
});
}
catch (OperationCanceledException e)
{
//Catestrophic Failure
canceled = true;
}
}
else
{
foreach (var hit in result.Hits)
{
if (action != null)
{
ActionOutputData cancel = action(hit, actionParameters);
System.Threading.Interlocked.Increment(ref processedCount);
if (logOutputFunc != null && !string.IsNullOrEmpty(cancel.Log))
{
logOutputFunc(cancel.Log);
}
if (cancel.CancelRunning)
{
canceled = true;
break;
}
}
if (progressOutputFunc != null)
{
ActionProgressData apd = new ActionProgressData(total, processedCount, started, prefix);
progressOutputFunc(apd);
}
}
}
if (canceled)
{
break;
}
} while (result.Hits.Count() > 0);
client.ClearScroll(c => c.ScrollId(scrollId));
if (logOutputFunc != null)
{
logOutputFunc("Done");
}
}
//public ConcurrentBag<T> GetObjectScroll<T>(QueryContainer qc, SourceFilter so, List<ISort> lSort = null) where T : class
//{
// return GetObjectScroll<T>(_DefaultIndex, qc, so, lSort);
//}
protected ConcurrentBag <T> GetObjectScroll <T>(string _DefaultIndex, QueryContainer query, SourceFilter so, List <ISort> lSort = null, string routing = "", int scroll_pageize = 2000) where T : class
{
try
{
string scroll_timeout = "5m";
//int scroll_pageize = 2000;
var seenSlices = new ConcurrentBag <int>();
ConcurrentBag <T> bag = new ConcurrentBag <T>();
if (query == null)
{
query = new MatchAllQuery();
}
if (so == null)
{
so = new SourceFilter()
{
}
}
;
try
{
var searchResponse = client.Search <T>(sd => sd.Source(s => so).Index(_DefaultIndex).From(0).Take(scroll_pageize).Query(q => query).Scroll(scroll_timeout));
while (true)
{
if (!searchResponse.IsValid || string.IsNullOrEmpty(searchResponse.ScrollId))
{
break;
}
if (!searchResponse.Documents.Any())
{
break;
}
var tmp = searchResponse.Hits;
Parallel.ForEach(tmp, new ParallelOptions {
MaxDegreeOfParallelism = 4
}, item =>
{
var doc = HitToDocument(item);
bag.Add(doc);
});
/*
* foreach (var item in tmp)
* {
* var doc = HitToDocument(item);
* bag.Add(doc);
* }
*/
searchResponse = client.Scroll <T>(scroll_timeout, searchResponse.ScrollId);
}
client.ClearScroll(new ClearScrollRequest(searchResponse.ScrollId));
}
catch (Exception)
{
}
finally
{
}
return(bag);
}
catch
{
}
return(null);
}
/// <summary>
/// 查询(sql)
/// </summary>
/// <param name="recordCount">总数据量(不分页)</param>
/// <param name="indices">指定索引(null:默认，[]:全部,["i_0","i_1","i_2"]:指定)</param>
/// <param name="query">sql查询语句</param>
/// <param name="size">数据量</param>
/// <param name="time">快照保存时间(秒,默认60)</param>
/// <param name="isThrow">抛出异常</param>
/// <returns></returns>
public List <T> SearchWithSql <T>(
out long recordCount,
string[] indices = null,
string query = null,
int?size = null,
int time = 60,
bool isThrow = false) where T : class
{
//if (pageIndex != null && pageIndex * pageRows > 10000)
// throw new Exception("数据量过大");
var indices_ = GetIndices(indices);
var _time = TimeSpan.FromSeconds(time);
List <T> data = new List <T>();
var sql = $"select * from {GetIndex(indices)} {query}";
if (size != null)
{
sql += $" limit {size}";
}
bool Transfinite = Total(indices_) > 10000;
var request = new TranslateSqlRequest()
{
Query = sql,
FetchSize = size
};
var respone_sql = ElasticClient.LowLevel.Sql.Translate <StringResponse>(PostData.Serializable(request));
var query_Dsl = respone_sql.Body;
var requestParameters = Transfinite ? new SearchRequestParameters()
{
Scroll = _time
} : null;
ISearchResponse <T> respone_search = ElasticClient.LowLevel.Search <SearchResponse <T> >(GetIndex(indices), query_Dsl, requestParameters);
recordCount = respone_search.Total;
if (Transfinite)
{
while (true)
{
if (!respone_search.IsValid || respone_search.ScrollId == null)
{
if (isThrow)
{
throw new Exception(respone_search.ServerError.Error.Reason);
}
break;
}
if (!respone_search.Documents.Any_Ex())
{
break;
}
data.AddRange(respone_search.Documents);
respone_search = ElasticClient.Scroll <T>(_time, respone_search.ScrollId);
}
}
else
{
if (!respone_search.IsValid)
{
if (isThrow)
{
throw new Exception(respone_search.ServerError.Error.Reason);
}
}
else
{
data.AddRange(respone_search.Documents);
}
}
ElasticClient.ClearScroll(new ClearScrollRequest(respone_search.ScrollId));
return(data);
}
/// <summary>
/// 查询(分页)
/// </summary>
/// <param name="recordCount">总数据量(不分页)</param>
/// <param name="indices">指定索引(null:默认，[]:全部,["i_0","i_1","i_2"]:指定)</param>
/// <param name="query"></param>
/// <param name="sort">排序</param>
/// <param name="pageIndex">指定页码</param>//(仅支持在数据量小于等于10000时进行分页)
/// <param name="pageRows">每页数据量</param>
/// <param name="time">快照保存时间(秒,默认60)</param>
/// <param name="isThrow">抛出异常</param>
/// <returns></returns>
public List <T> SearchPaging <T>(
out long recordCount,
string[] indices = null,
Func <QueryContainerDescriptor <T>, QueryContainer> query = null,
Func <SortDescriptor <T>, IPromise <IList <ISort> > > sort = null,
int?pageIndex = null,
int?pageRows = null,
int time = 60,
bool isThrow = false) where T : class
{
//if (pageIndex != null && pageIndex * pageRows > 10000)
// throw new Exception("数据量过大");
var indices_ = GetIndices(indices);
var _time = TimeSpan.FromSeconds(time);
List <T> data = new List <T>();
bool Transfinite = false;
var response = ElasticClient.Search <T>(s =>
{
s = s.Index(indices_);
if (query != null)
{
s = s.Query(query);
}
else
{
s = s.MatchAll();
}
if (pageIndex != null)
{
s = s.Size(pageRows)
.From((pageIndex - 1) * pageRows);
}
else
{
Transfinite = Total <T>() > 10000;
}
if (sort != null)
{
s = s.Sort(sort);
}
if (Transfinite)
{
s = s.Scroll(_time);
}
return(s);
});
recordCount = response.Total;
if (Transfinite)
{
while (true)
{
if (!response.IsValid || response.ScrollId == null)
{
if (isThrow)
{
throw new ElasticsearchException(response);
}
break;
}
if (!response.Documents.Any_Ex())
{
break;
}
data.AddRange(response.Documents);
response = ElasticClient.Scroll <T>(_time, response.ScrollId);
}
}
else
{
if (!response.IsValid)
{
if (isThrow)
{
throw new ElasticsearchException(response);
}
}
else
{
data.AddRange(response.Documents);
}
}
ElasticClient.ClearScroll(new ClearScrollRequest(response.ScrollId));
return(data);
}
/// <summary>
/// 查询(游标)
/// </summary>
/// <param name="indices">指定索引(null:默认，[]:全部,["i_0","i_1","i_2"]:指定)</param>
/// <param name="query"></param>
/// <param name="scrollId">滚动ID</param>
/// <param name="time">快照保存时间(秒,默认60)</param>
/// <param name="size">数据量上限</param>
/// <param name="sortField">排序字段</param>
/// <param name="sortType">排序类型(desc,asc[默认])</param>
/// <param name="isThrow">抛出异常</param>
/// <returns></returns>
public List <T> SearchScroll <T>(
string[] indices = null,
Func <QueryContainerDescriptor <T>, QueryContainer> query = null,
string scrollId = null,
int time = 60,
int?size = null,
string sortField = null,
string sortType = "asc",
bool isThrow = false) where T : class
{
var indices_ = GetIndices(indices);
string _time = ConvertTime(time);
List <T> data = new List <T>();
bool Transfinite = size > 10000 && Total <T>() > 10000;
var response = scrollId != null?ElasticClient.Scroll <T>(_time, scrollId) : ElasticClient.Search <T>(s =>
{
s = s.Index(indices_);
if (query != null)
{
s = s.Query(query);
}
else
{
s = s.MatchAll();
}
if (size != null)
{
s = s.Size(size)
.Sort(so => so.Field(typeof(T).GetProperty(sortField), sortType == "asc" ? SortOrder.Ascending : SortOrder.Descending));
}
if (Transfinite)
{
s = s.Scroll(_time);
}
return(s);
});
if (Transfinite)
{
while (true)
{
if (!response.IsValid || response.ScrollId == null)
{
if (isThrow)
{
throw new ElasticsearchException(response);
}
break;
}
if (!response.Documents.Any_Ex())
{
break;
}
data.AddRange(response.Documents);
response = ElasticClient.Scroll <T>(_time, response.ScrollId);
}
}
else
{
if (!response.IsValid)
{
if (isThrow)
{
throw new ElasticsearchException(response);
}
}
else
{
data.AddRange(response.Documents);
}
}
ElasticClient.ClearScroll(new ClearScrollRequest(response.ScrollId));
return(data);
}
public async Task <ApiResult> WriteFileAsync <TDoc>(string index, int PageSize = 10000) where TDoc : class
{
int CurrentCount = 0;
//reads for that day from elastic
//reads in chunks of 100, can be changed
var docCount = _client.Count <TDoc>(c => c.Index(_currentIndex));
_logger.LogInformation($"WriteFileAsync - Inspecting {docCount.Count} documents for changes...");
var searchResponse = _client.Search <TDoc>(s => s
.Size(100)
.Scroll("1m")
.Index(_currentIndex));
List <POChinoOutput> finalChinoOutput = new List <POChinoOutput>();
var outputType = _outputSettings.Value.Type.ToLower();
while (searchResponse.Documents.Any())
{
switch (outputType)
{
case "xml":
foreach (var document in searchResponse.Documents)
{
var pochinooutputcurrent = document as POChinoOutput;
if (pochinooutputcurrent.ForceInclude)
{
finalChinoOutput.Add(pochinooutputcurrent);
}
else
{
//Only send the PO if it's 'Open'
if (pochinooutputcurrent.StatusCode == "OP")
{
//Read from previous index for the document. If sent before, we don't send it again
var doc = await _client.GetAsync <TDoc>(new DocumentPath <TDoc>(pochinooutputcurrent.OrderId), g => g.Index(_previousIndex));
if (doc.Source == null)
{
var podatanew = document as POChinoOutput;
finalChinoOutput.Add(podatanew);
}
}
}
//save it to archive index before delete from current
var archiveresponse = await _client.UpdateAsync <POChinoOutput, object>(pochinooutputcurrent, u => u.Doc(pochinooutputcurrent).Index(_archiveIndex).DocAsUpsert());
//Save to Previous Index from Current
var previousresponse = await _client.UpdateAsync <POChinoOutput, object>(pochinooutputcurrent, u => u.Doc(pochinooutputcurrent).Index(_previousIndex).DocAsUpsert());
//delete from current
var deletedcurrent = await _client.DeleteAsync <POChinoOutput>(pochinooutputcurrent, u => u.Index(_currentIndex));
_logger.LogInformation($"WriteFileAsync - Successfully Processed PONumber -- {pochinooutputcurrent.OrderId} .");
}
_poChinoutilities.WriteChinoFile(finalChinoOutput);
break;
default:
_logger.LogError($"WriteFileAsync - Invalid output type {outputType} specified.");
return(new ApiResult <string>());
}
CurrentCount++;
searchResponse = _client.Scroll <TDoc>("1m", searchResponse.ScrollId);
}
_client.ClearScroll(c => c.ScrollId(searchResponse.ScrollId));
//ReIndex<TDoc>();
return(new ApiResult <string>());
}
