Esempio n. 1
0
        private void Downloader_FetchFailed(object Sender, FetchFailEventArgs args)
        {
            lock (hExecuted)
            {
                hExecuted.Add(args.Link.Uri.ToString());
            }
            log.Error($"[ERR] {args.Error.Message} {args.Link}");
            args.Source = FetchEventArgs.EventSource.Downloader;

            if (args.HttpErrorCode == 404)
            {
                SpiderWorkData.Error404.Add(args.Link.Uri.ToString());
            }

            FetchFailed?.Invoke(this, args);
        }
Esempio n. 2
0
        private void OnTimedEvent(object sender, ElapsedEventArgs e)
        {
            if (!workQueue.TryDequeue(out Link l))
            {
                return;
            }

            if (ShouldFetch != null)
            {
                var args = new ShouldFetchEventArgs(l);
                ShouldFetch(this, args);
                if (args.Cancel)
                {
                    return;              // do not fail/complete
                }
            }

            switch (DiscardMode)
            {
            case DiscardModeTypes.Fail:
                FetchFailed?.Invoke(this,
                                    new FetchFailEventArgs(l,
                                                           0,
                                                           new Exception("NullDownloader:DiscardModeTypes.Fail"),
                                                           new HeaderCollection()));
                break;

            case DiscardModeTypes.CompleteEmpty:
                FetchCompleted?.Invoke(this,
                                       new FetchCompleteEventArgs(l,
                                                                  new byte[0],
                                                                  new HeaderCollection(),
                                                                  new HeaderCollection()));
                break;

            case DiscardModeTypes.Ignore:
            default:
                // ignore ...
                break;
            }
        }