private void HandleBadDownload(DownloaderObj obj) { DevourTarget target = obj.Arg as DevourTarget; if (obj.State == HttpDownloadResult.BadAddress || obj.State == HttpDownloadResult.DocumentUnavailable) { AddFaultTarget(target); //Document extremely guilty _proxies.Release(obj.PrxContainer as ProxyContainer, false); //Proxy simply had a bad day } else { if (obj.State == HttpDownloadResult.ProxyError) { _proxies.Fire(obj.PrxContainer as ProxyContainer); //Proxy extremely guilty _targets.Enqueue(target); //Document not in the business } else { if (--target.Lifes <= 0) { AddFaultTarget(target); //Sth going wrong } _proxies.Release(obj.PrxContainer as ProxyContainer, false); //Proxy simply had a bad day } } }
private void AddFaultTarget(DevourTarget target) { lock (sync) { _faultTargets.Add(target); } if (OnTargetError != null) { OnTargetError(target); } }
private void DevourOne(ProxyContainer proxyCont, DevourTarget target) { DownloaderObj obj = new DownloaderObj(target.Uri, DevourCallback, true, proxyCont, this.CookieOptions, target.Attempts, target, null, false, 1000, this.TimingParams, this.RequestParams); Downloader.Queue(obj); #if DEBUG Interlocked.Increment(ref __queuedObjects); GlobalLog.Write("__queuedObjects: {0}", __queuedObjects); #endif MoveObjQueued(); }
private bool TryDevourTarget(DevourTarget target) { ProxyContainer proxy; if (_proxies.TryGet(out proxy)) { DevourOne(proxy, target); return(true); } return(false); }
private void MoveReadComplete(DevourTarget target) { Interlocked.Increment(ref _readsComplete); if (OnReadProgress != null) { OnReadProgress(_readsComplete, _readsQueued); } #if DEBUG GlobalLog.Write("Downloader.Processing: {0}, Queued: {1}", Downloader.Processing, Downloader.Queued); #endif UnsubscribeDevourTarget(target); CheckWorkCompletion(); }
private void HandleSuccessDownload(DownloaderObj obj) { _proxies.Release(obj.PrxContainer as ProxyContainer, true); DevourTarget target = obj.Arg as DevourTarget; //MoveReadQueue(); try { MoveReadQueue(); target.Reader.ReadData(obj.DataStr, target); MoveReadComplete(target); } catch (Exception ex) { GlobalLog.Err(ex, "error while reading data in devourer"); } AddSuccessTarget(obj.Uri); }
//RequestParams _params; public Devourer(DevourTarget initialTarget, IProxyProvider proxies, IPageValidator validator = null) : this(new List<DevourTarget>(new DevourTarget[] { initialTarget }), proxies, validator) { }
void UnsubscribeDevourTarget(DevourTarget target) { target.Reader.OnNewTargets -= new NewTargetsDel(_reader_OnNewTargets); target.Reader.OnReadComplete -= new ReadCompleteDel(MoveReadComplete); }
private bool TryDevourTarget(DevourTarget target) { ProxyContainer proxy; if (_proxies.TryGet(out proxy)) { DevourOne(proxy, target); return true; } return false; }
private void MoveReadComplete(DevourTarget target) { Interlocked.Increment(ref _readsComplete); if (OnReadProgress!=null) { OnReadProgress(_readsComplete, _readsQueued); } #if DEBUG GlobalLog.Write("Downloader.Processing: {0}, Queued: {1}", Downloader.Processing, Downloader.Queued); #endif UnsubscribeDevourTarget(target); CheckWorkCompletion(); }
private void AddFaultTarget(DevourTarget target) { lock (sync) { _faultTargets.Add(target); } if (OnTargetError!=null) { OnTargetError(target); } }
ConcurrentQueue <DevourTarget> _targets; /* KEY -- faultCount, VALUE -- targetUri */ //RequestParams _params; public Devourer(DevourTarget initialTarget, IProxyProvider proxies, IPageValidator validator = null) : this(new List <DevourTarget>(new DevourTarget[] { initialTarget }), proxies, validator) { }