private void ExecuteTask(ICrawlerTask task)
{
var tries = _options.Retries ?? 2;
try
{
var hndExc = Policy
.Handle <Exception>()
.WaitAndRetry(tries, (i) => { return(TimeSpan.FromMilliseconds(new Random().Next(500, 2000) * i)); }
, (ex, ts) =>
{
LogService.LogWarning($"Task failed, retrying in {ts}");
});
var timeout = _options.TaskTimeout ?? 600;
var hndTout = Policy.Timeout(TimeSpan.FromMinutes(timeout), TimeoutStrategy.Pessimistic);
var wrap = hndExc.Wrap(hndTout);
var result = wrap.ExecuteAndCapture(() => {
LogService.LogInformation($"Starting task [{task.TaskId}] {task.TaskName}");
task.RunAsync(Context).ConfigureAwait(false).GetAwaiter().GetResult();
});
if (result.Outcome == OutcomeType.Failure)
{
throw result.FinalException;
}
}
catch (TimeoutRejectedException ex)
{
LogService.LogError(ex, "Task timeout excedeed.");
ShutDown();
}
catch (UnavailableSourceException ex)
{
LogService.LogError(ex, "Task source unavailable");
ShutDown();
}
catch (Exception ex)
{
LogService.LogError(ex, "Error in task execution");
ShutDown();
}
finally
{
task.SaveTask(Context);
if (Context.Shutdown)
{
Dispose();
}
}
}
public Task ScheduleAsync(ICrawlerTask task)
{
var jobToBuild = JobBuilder.Create(task.TaskType);
task.ConfigureJob(jobToBuild);
var job = jobToBuild.Build();
var triggerToBuild = TriggerBuilder.Create();
task.ConfigureTrigger(triggerToBuild);
var trigger = triggerToBuild.Build();
_quartzConfiguration.Scheduler.ScheduleJob(job, trigger);
return(Task.FromResult(0));
}
public void StartTask(ICrawlerTask task)
{
Task = task;
}
public void ClearContext()
{
Task = null;
Metadata = null;
Result = null;
}
/// <inheritdoc />
public void Run()
{
_logger.LogInformation("后台任务已启动主循环");
if (_tasks.Count == 0)
{
_logger.LogWarning("没有任何的 Crawler 任务。");
return;
}
var ts = 0; // Current timestamp
var crawlerTaskId = 0;
while (true)
{
var interval = _tasks.MaxElement.NextRunTimestamp - ts;
if (interval > 0)
{
Thread.Sleep(interval * 60 * 1000);
}
if (interval >= 0)
{
ts += interval;
}
while (_tasks.MaxElement.NextRunTimestamp == ts)
{
var activeTask = _tasks.RemoveMax();
ICrawlerTask taskObject = null;
try
{
taskObject = activeTask.TaskObject.Value;
}
catch (Exception ex)
{
_logger.LogError(ex, "无法激活 Crawler 任务对象 {0}：{1}：{2}",
activeTask.Metadata.Type, ex.GetType(), ex.Message);
}
if (taskObject != null)
{
var taskId = ++crawlerTaskId;
_logger.LogInformation("启动 Crawler 任务 {0}，ID = {1}", activeTask.Metadata.Type, taskId);
Task.Run(taskObject.Run)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
_logger.LogError(t.Exception, "Crawler 任务 {0} 抛出了未经处理的异常。", taskId);
}
else
{
_logger.LogInformation("Crawler 任务 {0} 已退出", taskId);
}
});
}
// activeTask.Metadata.Annotation.Interval should be guaranteed to be greater than 0.
activeTask.NextRunTimestamp += activeTask.Metadata.Annotation.Interval;
_tasks.Add(activeTask);
}
}
}
