/// <summary>
/// Initializes a new instance of CrawlController with default (injected) dependencies.
/// </summary>
public CrawlController(
IConfiguration configuration,
ILogger <CrawlController> logger,
ICrawlerStrategy crawler
)
{
this.configuration = configuration;
this.logger = logger;
this.crawler = crawler;
}
/// <summary>
/// This method is invoked indeterministically from the Hangfire job server.
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public virtual async Task <Task> Process(CrawlRequestBase request)
{
// should never be null if a job has been enqueued, except for unit tests
if (_cancellationTokenSource == null)
{
_cancellationTokenSource = new CancellationTokenSource();
}
_strategy = _crawlerStrategyFactory.Value.Create(request);
if (request.CompletionWindow != Timeout.InfiniteTimeSpan)
{
_cancellationTokenSource.CancelAfter(
request.CompletionWindow.Add(TimeSpan.FromMilliseconds(200)));
}
await _strategy.ProcessRequestAsync(request, _cancellationTokenSource.Token);
RegisterTimer(
PersistStateCallback,
null,
TimeSpan.Zero,
_strategy.PersistStateInterval);
// Task.Factory.StartNew(
// async () =>
// {
// // while no cancellation has been requested AND the strategy
// // has not completed, periodically save the state.
// // IsCompleted here does not mean success, but may mean cancelled as well
// while (!_cancellationTokenSource.IsCancellationRequested &&
// !strategy.Completion.IsCompleted)
// {
// await PersistState(strategy.GetSnapshot());
// await Task.Delay(strategy.PersistStateInterval);
// }
// },
// CancellationToken.None,
// TaskCreationOptions.None,
// TaskScheduler.Current);
return(_strategy.Completion
.ContinueWith(_ => PersistState(_strategy.GetSnapshot()), TaskScheduler.Current)
.Unwrap());
}