public void ProgressWriter(ActionProgressData data)
{
if (actionProgressFunctions != null)
{
foreach (var w in actionProgressFunctions)
{
if (w != null)
{
w(data);
}
}
}
}
public static void DoActionForQuery <T>(ElasticClient client,
Func <int, int, ISearchResponse <T> > searchFunc,
System.Func <IHit <T>, object, Devmasters.Core.Batch.ActionOutputData> action, object actionParameters,
System.Action <string> logOutputFunc,
System.Action <Devmasters.Core.Batch.ActionProgressData> progressOutputFunc,
bool parallel,
int blockSize = 500, int?maxDegreeOfParallelism = null, string prefix = ""
)
where T : class
{
DateTime started = DateTime.Now;
long total = 0;
int currIteration = 0;
int processedCount = 0;
ISearchResponse <T> result = default(ISearchResponse <T>);
//var total = NoveInzeraty.Lib.Data.Smlouva.Search.NumberOfDocs(null, null);
string scrollId = null;
//create scroll search context
bool firstResult = true;
if (maxDegreeOfParallelism <= 1)
{
parallel = false;
}
try
{
result = searchFunc(blockSize, currIteration);
if (result.IsValid == false)
{
Lib.ES.Manager.LogQueryError <T>(result);
}
}
catch (Exception e1)
{
System.Threading.Thread.Sleep(10000);
try
{
result = searchFunc(blockSize, currIteration);
}
catch (Exception e2)
{
System.Threading.Thread.Sleep(20000);
try
{
result = searchFunc(blockSize, currIteration);
}
catch (Exception ex)
{
HlidacStatu.Util.Consts.Logger.Error("Cannot read data from Elastic, skipping iteration" + currIteration, ex);
return;
}
}
}
scrollId = result.ScrollId;
do
{
DateTime iterationStart = DateTime.Now;
if (firstResult)
{
firstResult = false;
}
else
{
result = client.Scroll <T>(ScrollLifeTime, scrollId);
scrollId = result.ScrollId;
}
currIteration++;
if (result.Hits.Count() == 0)
{
break;
}
total = result.Total;
bool canceled = false;
if (parallel)
{
CancellationTokenSource cts = new CancellationTokenSource();
try
{
ParallelOptions pOptions = new ParallelOptions();
if (maxDegreeOfParallelism.HasValue)
{
pOptions.MaxDegreeOfParallelism = maxDegreeOfParallelism.Value;
}
pOptions.CancellationToken = cts.Token;
Parallel.ForEach(result.Hits, (hit) =>
{
if (action != null)
{
ActionOutputData cancel = null;
try
{
cancel = action(hit, actionParameters);
System.Threading.Interlocked.Increment(ref processedCount);
if (logOutputFunc != null && !string.IsNullOrEmpty(cancel.Log))
{
logOutputFunc(cancel.Log);
}
if (cancel.CancelRunning)
{
cts.Cancel();
}
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error("DoActionForAll action error", e);
cts.Cancel();
}
}
if (progressOutputFunc != null)
{
ActionProgressData apd = new ActionProgressData(total, processedCount, started, prefix);
progressOutputFunc(apd);
}
});
}
catch (OperationCanceledException e)
{
//Catestrophic Failure
canceled = true;
}
}
else
{
foreach (var hit in result.Hits)
{
if (action != null)
{
ActionOutputData cancel = action(hit, actionParameters);
System.Threading.Interlocked.Increment(ref processedCount);
if (logOutputFunc != null && !string.IsNullOrEmpty(cancel.Log))
{
logOutputFunc(cancel.Log);
}
if (cancel.CancelRunning)
{
canceled = true;
break;
}
}
if (progressOutputFunc != null)
{
ActionProgressData apd = new ActionProgressData(total, processedCount, started, prefix);
progressOutputFunc(apd);
}
}
}
if (canceled)
{
break;
}
} while (result.Hits.Count() > 0);
client.ClearScroll(c => c.ScrollId(scrollId));
if (logOutputFunc != null)
{
logOutputFunc("Done");
}
}