private void ExecuteTask(ICrawlerTask task) { var tries = _options.Retries ?? 2; try { var hndExc = Policy .Handle <Exception>() .WaitAndRetry(tries, (i) => { return(TimeSpan.FromMilliseconds(new Random().Next(500, 2000) * i)); } , (ex, ts) => { LogService.LogWarning($"Task failed, retrying in {ts}"); }); var timeout = _options.TaskTimeout ?? 600; var hndTout = Policy.Timeout(TimeSpan.FromMinutes(timeout), TimeoutStrategy.Pessimistic); var wrap = hndExc.Wrap(hndTout); var result = wrap.ExecuteAndCapture(() => { LogService.LogInformation($"Starting task [{task.TaskId}] {task.TaskName}"); task.RunAsync(Context).ConfigureAwait(false).GetAwaiter().GetResult(); }); if (result.Outcome == OutcomeType.Failure) { throw result.FinalException; } } catch (TimeoutRejectedException ex) { LogService.LogError(ex, "Task timeout excedeed."); ShutDown(); } catch (UnavailableSourceException ex) { LogService.LogError(ex, "Task source unavailable"); ShutDown(); } catch (Exception ex) { LogService.LogError(ex, "Error in task execution"); ShutDown(); } finally { task.SaveTask(Context); if (Context.Shutdown) { Dispose(); } } }
public Task ScheduleAsync(ICrawlerTask task) { var jobToBuild = JobBuilder.Create(task.TaskType); task.ConfigureJob(jobToBuild); var job = jobToBuild.Build(); var triggerToBuild = TriggerBuilder.Create(); task.ConfigureTrigger(triggerToBuild); var trigger = triggerToBuild.Build(); _quartzConfiguration.Scheduler.ScheduleJob(job, trigger); return(Task.FromResult(0)); }
public void StartTask(ICrawlerTask task) { Task = task; }
public void ClearContext() { Task = null; Metadata = null; Result = null; }
/// <inheritdoc /> public void Run() { _logger.LogInformation("后台任务已启动主循环"); if (_tasks.Count == 0) { _logger.LogWarning("没有任何的 Crawler 任务。"); return; } var ts = 0; // Current timestamp var crawlerTaskId = 0; while (true) { var interval = _tasks.MaxElement.NextRunTimestamp - ts; if (interval > 0) { Thread.Sleep(interval * 60 * 1000); } if (interval >= 0) { ts += interval; } while (_tasks.MaxElement.NextRunTimestamp == ts) { var activeTask = _tasks.RemoveMax(); ICrawlerTask taskObject = null; try { taskObject = activeTask.TaskObject.Value; } catch (Exception ex) { _logger.LogError(ex, "无法激活 Crawler 任务对象 {0}:{1}:{2}", activeTask.Metadata.Type, ex.GetType(), ex.Message); } if (taskObject != null) { var taskId = ++crawlerTaskId; _logger.LogInformation("启动 Crawler 任务 {0},ID = {1}", activeTask.Metadata.Type, taskId); Task.Run(taskObject.Run) .ContinueWith(t => { if (t.IsFaulted) { _logger.LogError(t.Exception, "Crawler 任务 {0} 抛出了未经处理的异常。", taskId); } else { _logger.LogInformation("Crawler 任务 {0} 已退出", taskId); } }); } // activeTask.Metadata.Annotation.Interval should be guaranteed to be greater than 0. activeTask.NextRunTimestamp += activeTask.Metadata.Annotation.Interval; _tasks.Add(activeTask); } } }