private void _managedOneTask(object domainTask)
{
crawlerDomainTask taskToRun = (crawlerDomainTask)domainTask;
try
{
items.running.Add(taskToRun);
taskToRun.start();
Thread.Sleep(50);
crawlerDomainTask taskToRemove = taskToRun;
logger.log("[" + taskToRun.wProfile.domain + "] finished [tl:" + taskToRun.targetLoaded + "][td:" + taskToRun.targetDetected + "]");
items.done.AddUnique(taskToRun);
} catch (Exception ex)
{
crawlerDomainTask taskToRemove = taskToRun;
items.done.AddUnique(taskToRun);
crawlerErrorLog clog = crawlerErrorLog.CreateAndSave(ex, taskToRun.wRecord, taskToRun, crawlerErrorEnum.domainOneTaskError);
aceLog.log("[" + taskToRun.wProfile.domain + "] Thread crashed [" + ex.Message + ":" + ex.GetType().Name);
taskToRun.CallAbort(cancelTokens[taskToRun]);
}
}
private void _managedParallel()
{
cancelTokens = new Dictionary <crawlerDomainTask, CancellationTokenSource>();
try
{
foreach (var item in items.items)
{
cancelTokens.Add(item, new CancellationTokenSource());
Task task = new Task(_managedOneTask, item, cancelTokens[item].Token);
tasks.Add(task);
task_waiting.Add(task);
}
maxThreads = Math.Min(maxThreads, tasks.Count);
int rIndex = 0;
while (task_finished.Count < tasks.Count)
{
if (task_running.Count < maxThreads)
{
int toRun = maxThreads - task_running.Count;
for (int i = 0; i < toRun; i++)
{
Task task = task_waiting.FirstOrDefault();
if (task != null)
{
task_waiting.Remove(task);
task_running.Add(task);
task_started.Add(task);
task.Start();
}
else
{
}
}
rIndex = task_started.Count;
}
foreach (Task task in task_running.ToList())
{
switch (task.Status)
{
case TaskStatus.Faulted:
task_canceled.Add(task);
task_finished.Add(task);
task_running.Remove(task);
break;
case TaskStatus.RanToCompletion:
task_finished.Add(task);
task_running.Remove(task);
break;
case TaskStatus.Canceled:
task_canceled.Add(task);
task_finished.Add(task);
task_running.Remove(task);
break;
default:
break;
}
}
Thread.Sleep(imbWEMManager.settings.crawlerJobEngine.crawlerDomainCheckTickMs);
foreach (Task task in task_running.ToList())
{
bool abortTask = false;
crawlerDomainTask taskToRun = (crawlerDomainTask)task.AsyncState;
if (taskToRun.startTime != DateTime.MinValue)
{
if (DateTime.Now.Subtract(taskToRun.startTime).TotalMinutes > TimeLimitForDomainCrawlCancelation)
{
abortTask = true;
aceLog.log("Forced cancelation of [" + taskToRun.wProfile.domain + "] due double execution timeout [" + DateTime.Now.Subtract(taskToRun.startTime).TotalMinutes.ToString("#0.00") + "]", null, true);
}
else if (DateTime.Now.Subtract(taskToRun.startTime).TotalMinutes > _timeLimitForDLC)
{
abortTask = true;
taskToRun.isStageAborted = true;
}
}
if (!isEnabled)
{
aceLog.log("General Crawl Engine Abort Call --> " + taskToRun.wRecord.domain);
abortTask = true;
}
if (abortTask)
{
taskToRun.CallAbort(cancelTokens[taskToRun]);
// taskToRun.reporter.reportDomainFinished(taskToRun.wRecord);
task_finished.Add(task);
task_canceled.Add(task);
task_running.Remove(task);
}
if (!taskToRun.IsActive())
{
taskToRun.CallAbort(cancelTokens[taskToRun]);
aceLog.log("Task [" + taskToRun.wProfile.domain + "] became inactive on state [" + taskToRun.iterationStatus.ToString() + "] -- calling for abortion", null, true);
task_finished.Add(task);
task_running.Remove(task);
task_canceled.Add(task);
}
}
if (DateTime.Now.Subtract(startTime).TotalMinutes > TimeLimitForCompleteJob)
{
aceLog.log("Canceling any threads creation due time limit reached");
break;
}
}
} catch (Exception ex)
{
// plugins.eventUniversal(crawlJobEngineStageEnum.error, this);
var clog = crawlerErrorLog.CreateAndSave(ex, items.tRecord, null, crawlerErrorEnum.TaskMachineError);
}
if (task_finished.Count == tasks.Count)
{
allTaskDone = true;
}
else if (items.done.Count == items.items.Count)
{
allTaskDone = true;
}
else if (items.running.Count == 0)
{
allTaskDone = true;
}
}