// public builderForLog loger { get; set; }
//public String InstanceHash
//{
// get
// {
// return name + "_" + GetHashCode();
// }
//}
public abstract void eventUniversal <TFirst, TSecond>(crawlJobEngineStageEnum stage, crawlerDomainTaskMachine __machine, TFirst __task, TSecond __resource);
public override void eventUniversal <TFirst, TSecond>(crawlJobEngineStageEnum stage, crawlerDomainTaskMachine __machine, TFirst __task, TSecond __resource)
{
switch (stage)
{
case crawlJobEngineStageEnum.statusReport:
//var tMemory = __machine.measureTaker.GetTrend(trendMemory);
//var tCPU = __machine.measureTaker.GetTrend(trendCPU);
//var tDataLoad = __machine.measureTaker.GetTrend(trendDataLoad);
//var tContentPages = __machine.dataLoadTaker.GetTrend(trendContentPages);
//var tContentTerms = __machine.dataLoadTaker.GetTrend(trendContentTerms);
//var tIterations = __machine.dataLoadTaker.GetTrend(trendIterations);
//loger.AppendLine(String.Format(TREND_LINE, tMemory.GetTrendInline(), tCPU.GetTrendInline(), tDataLoad.GetTrendInline()));
//loger.AppendLine(String.Format(TREND_LINE, tContentPages.GetTrendInline(), tContentTerms.GetTrendInline(), tIterations.GetTrendInline()));
int change = 0;
int newTC = __machine.maxThreads;
double maxLatencyToLimit = 0;
double maxLatency = 0;
bool doBoost = false;
string domainThatLates = "";
string threadId = "";
Thread criticalThread = null;
double average = CPUAverageLast;
double avgChange = average - CPUAverageLast;
double maxAge = 0;
crawlerDomainTask taskOldest = null;
var tasks = __machine.task_running.ToList();
foreach (Task task in tasks)
{
crawlerDomainTask taskInRun = task.AsyncState as crawlerDomainTask;
double since = taskInRun.sinceLastIterationStart;
double tage = DateTime.Now.Subtract(taskInRun.startTime).TotalMinutes;
maxLatency = Math.Max(maxLatency, since);
if (maxAge <= tage)
{
maxAge = tage;
taskOldest = taskInRun;
}
if (maxLatency <= since)
{
domainThatLates = taskInRun.wRecord.domain;
if (taskInRun?.executionThread != null)
{
threadId = taskInRun.executionThread.ManagedThreadId.ToString() + " [" + taskInRun.executionThread.Priority.ToString() + "]";
}
criticalThread = taskInRun.executionThread;
}
}
maxLatencyToLimit = maxLatency.GetRatio(__machine.TimeLimitForTask);
double maxAgeLimit = maxAge.GetRatio(__machine._timeLimitForDLC);
double totalAgeLimit = DateTime.Now.Subtract(__machine.startTime).TotalMinutes.GetRatio(__machine.TimeLimitForCompleteJob);
loger.log("Max. latency: [" + maxLatency.ToString("F2") + " min][" + maxLatencyToLimit.ToString("P2") + "] " + domainThatLates + " Thread: " + threadId);
if (taskOldest != null)
{
loger.log("Oldest DLC: [" + maxAge.ToString("F2") + " min][" + maxAgeLimit.ToString("P2") + "] " + taskOldest.wRecord.domain + " Thread: " + taskOldest.executionThread.ManagedThreadId.ToString() + " [" + taskOldest.executionThread.Priority.ToString() + "]");
}
#region TIMEOUT PREVENTION -----------------------------------------
if (imbWEMManager.settings.crawlerJobEngine.doTaskTimeOutPrevention)
{
if (totalAgeLimit > 0.9)
{
bool newDisable = false;
foreach (Task task in tasks)
{
crawlerDomainTask t = task.AsyncState as crawlerDomainTask;
if (!t.isLoaderDisabled)
{
t.isLoaderDisabled = true;
newDisable = true;
loger.log("Time Limit Critical: loader is disabled for: " + t.wRecord.domain + " due execution time limit for Thread: " + t.executionThread.ManagedThreadId.ToString());
}
}
if (newDisable)
{
aceTerminalInput.doBeepViaConsole(1200, 250, 5);
}
}
if (maxAgeLimit > 0.9)
{
if (!taskOldest.isLoaderDisabled)
{
taskOldest.isLoaderDisabled = true;
loger.consoleAltColorToggle();
loger.log("DLC Time Limit Critical: loader is disabled for: " + taskOldest.wRecord.domain + " due execution time limit for Thread: " + taskOldest.executionThread.ManagedThreadId.ToString());
loger.consoleAltColorToggle();
aceTerminalInput.doBeepViaConsole();
}
}
doBoost = false;
if (maxLatencyToLimit > 0.5)
{
if (criticalThread != null)
{
criticalThread.Priority = ThreadPriority.AboveNormal;
}
change = -2;
}
else if (maxLatencyToLimit > 0.70)
{
if (criticalThread != null)
{
criticalThread.Priority = ThreadPriority.Highest;
}
change = -4;
}
else if (maxLatencyToLimit > 0.90)
{
loger.log("Max. latency critical :: REDUCING TO SINGLE THREAD : ");
foreach (Task task in tasks)
{
crawlerDomainTask taskInRun = task.AsyncState as crawlerDomainTask;
if (taskInRun?.executionThread != null)
{
taskInRun.executionThread.Priority = ThreadPriority.BelowNormal;
}
}
if (criticalThread != null)
{
criticalThread.Priority = ThreadPriority.Highest;
}
newTC = 1;
}
else
{
foreach (Task task in tasks)
{
crawlerDomainTask taskInRun = task.AsyncState as crawlerDomainTask;
if (taskOldest == taskInRun)
{
if (taskInRun?.executionThread != null)
{
taskInRun.executionThread.Priority = ThreadPriority.AboveNormal;
}
}
else
{
if (taskInRun?.executionThread != null)
{
taskInRun.executionThread.Priority = ThreadPriority.Normal;
}
}
}
doBoost = true;
}
}
if (imbWEMManager.settings.crawlerJobEngine.doAutoAdjustTC)
{
#endregion --------------------------- ^ timeout prevention ^^
if (doBoost) // <------ TC adjust
{
var takes = __machine.cpuTaker.GetLastSamples(imbWEMManager.settings.crawlerJobEngine.CPUSampleForAutoAdjustMax);
if (takes.Count < imbWEMManager.settings.crawlerJobEngine.CPUSampleForAutoAdjust)
{
return;
}
average = (takes.Average(x => x.reading) / 100);
avgChange = average - CPUAverageLast;
double CPUMargin = imbWEMManager.settings.crawlerJobEngine.CPUMargin;
int dlc = __machine.taskRunning;
CPUAverageDefendLine = Math.Max(average, CPUAverageLast);
if (dlc < (__machine.maxThreads - 1))
{
return;
}
if (average < imbWEMManager.settings.crawlerJobEngine.CPUTarget)
{
if (average < (CPUAverageDefendLine - CPUMargin))
{
change = -1;
}
else
{
change = 1;
}
}
else if (average > imbWEMManager.settings.crawlerJobEngine.CPULimit)
{
change = -1;
}
newTC = Math.Min(__machine.maxThreads + change, imbWEMManager.settings.crawlerJobEngine.TCAutoLimit);
if (newTC < 0)
{
newTC = 1;
}
CPUAverageLast = average;
}
else
{
if (change != 0)
{
newTC = Math.Min(__machine.maxThreads + change, imbWEMManager.settings.crawlerJobEngine.TCAutoLimit);
}
if (newTC < 0)
{
newTC = 1;
}
}
}
int e_change = newTC - __machine.maxThreads;
__machine.maxThreads = newTC;
loger.log("CPU average [" + average.ToString("P2") + "][" + avgChange.ToString("P2") + "] - (change: " + e_change + ") TC: " + __machine.maxThreads.ToString("D3") + " DLC:[" + __machine.taskRunning.ToString("D3") + "]");
/*
* if (average < imbWEMManager.settings.crawlerJobEngine.CPUTarget)
* {
*
* ;
*
* }
* else if (average > imbWEMManager.settings.crawlerJobEngine.CPULimit)
* {
* __machine.maxThreads = Math.Min(__machine.maxThreads - 1, imbWEMManager.settings.crawlerJobEngine.TCAutoLimit);
* loger.log("CPU average [" + average.ToString("P2") + "] > reducing TC to: " + __machine.maxThreads.ToString("D3") + " DLC:[" + __machine.taskRunning.ToString("D3") + "]");
* }
* else
* {
* loger.log("CPU average [" + average.ToString("P2") + "] ---------- TC_max: " + __machine.maxThreads.ToString("D3") + " DLC:[" + __machine.taskRunning.ToString("D3") + "]");
* }
*/
break;
}
}
