private void Downloader_FetchFailed(object Sender, FetchFailEventArgs args)
{
lock (hExecuted)
{
hExecuted.Add(args.Link.Uri.ToString());
}
log.Error($"[ERR] {args.Error.Message} {args.Link}");
args.Source = FetchEventArgs.EventSource.Downloader;
if (args.HttpErrorCode == 404)
{
SpiderWorkData.Error404.Add(args.Link.Uri.ToString());
}
FetchFailed?.Invoke(this, args);
}
private void OnTimedEvent(object sender, ElapsedEventArgs e)
{
if (!workQueue.TryDequeue(out Link l))
{
return;
}
if (ShouldFetch != null)
{
var args = new ShouldFetchEventArgs(l);
ShouldFetch(this, args);
if (args.Cancel)
{
return; // do not fail/complete
}
}
switch (DiscardMode)
{
case DiscardModeTypes.Fail:
FetchFailed?.Invoke(this,
new FetchFailEventArgs(l,
0,
new Exception("NullDownloader:DiscardModeTypes.Fail"),
new HeaderCollection()));
break;
case DiscardModeTypes.CompleteEmpty:
FetchCompleted?.Invoke(this,
new FetchCompleteEventArgs(l,
new byte[0],
new HeaderCollection(),
new HeaderCollection()));
break;
case DiscardModeTypes.Ignore:
default:
// ignore ...
break;
}
}