private void HandleBadDownload(DownloaderObj obj)
{
DevourTarget target = obj.Arg as DevourTarget;
if (obj.State == HttpDownloadResult.BadAddress ||
obj.State == HttpDownloadResult.DocumentUnavailable)
{
AddFaultTarget(target); //Document extremely guilty
_proxies.Release(obj.PrxContainer as ProxyContainer, false); //Proxy simply had a bad day
}
else
{
if (obj.State == HttpDownloadResult.ProxyError)
{
_proxies.Fire(obj.PrxContainer as ProxyContainer); //Proxy extremely guilty
_targets.Enqueue(target); //Document not in the business
}
else
{
if (--target.Lifes <= 0)
{
AddFaultTarget(target); //Sth going wrong
}
_proxies.Release(obj.PrxContainer as ProxyContainer, false); //Proxy simply had a bad day
}
}
}
private void AddFaultTarget(DevourTarget target)
{
lock (sync)
{
_faultTargets.Add(target);
}
if (OnTargetError != null)
{
OnTargetError(target);
}
}
private void DevourOne(ProxyContainer proxyCont, DevourTarget target)
{
DownloaderObj obj = new DownloaderObj(target.Uri, DevourCallback, true, proxyCont, this.CookieOptions, target.Attempts, target, null, false, 1000, this.TimingParams, this.RequestParams);
Downloader.Queue(obj);
#if DEBUG
Interlocked.Increment(ref __queuedObjects);
GlobalLog.Write("__queuedObjects: {0}", __queuedObjects);
#endif
MoveObjQueued();
}
private bool TryDevourTarget(DevourTarget target)
{
ProxyContainer proxy;
if (_proxies.TryGet(out proxy))
{
DevourOne(proxy, target);
return(true);
}
return(false);
}
private void MoveReadComplete(DevourTarget target)
{
Interlocked.Increment(ref _readsComplete);
if (OnReadProgress != null)
{
OnReadProgress(_readsComplete, _readsQueued);
}
#if DEBUG
GlobalLog.Write("Downloader.Processing: {0}, Queued: {1}", Downloader.Processing, Downloader.Queued);
#endif
UnsubscribeDevourTarget(target);
CheckWorkCompletion();
}
private void HandleSuccessDownload(DownloaderObj obj)
{
_proxies.Release(obj.PrxContainer as ProxyContainer, true);
DevourTarget target = obj.Arg as DevourTarget;
//MoveReadQueue();
try
{
MoveReadQueue();
target.Reader.ReadData(obj.DataStr, target);
MoveReadComplete(target);
}
catch (Exception ex)
{
GlobalLog.Err(ex, "error while reading data in devourer");
}
AddSuccessTarget(obj.Uri);
}
//RequestParams _params;
public Devourer(DevourTarget initialTarget, IProxyProvider proxies, IPageValidator validator = null)
: this(new List<DevourTarget>(new DevourTarget[] { initialTarget }), proxies, validator)
{
}
void UnsubscribeDevourTarget(DevourTarget target)
{
target.Reader.OnNewTargets -= new NewTargetsDel(_reader_OnNewTargets);
target.Reader.OnReadComplete -= new ReadCompleteDel(MoveReadComplete);
}
private bool TryDevourTarget(DevourTarget target)
{
ProxyContainer proxy;
if (_proxies.TryGet(out proxy))
{
DevourOne(proxy, target);
return true;
}
return false;
}
private void MoveReadComplete(DevourTarget target)
{
Interlocked.Increment(ref _readsComplete);
if (OnReadProgress!=null)
{
OnReadProgress(_readsComplete, _readsQueued);
}
#if DEBUG
GlobalLog.Write("Downloader.Processing: {0}, Queued: {1}", Downloader.Processing, Downloader.Queued);
#endif
UnsubscribeDevourTarget(target);
CheckWorkCompletion();
}
private void DevourOne(ProxyContainer proxyCont, DevourTarget target)
{
DownloaderObj obj = new DownloaderObj(target.Uri, DevourCallback, true, proxyCont, this.CookieOptions, target.Attempts, target, null, false, 1000, this.TimingParams, this.RequestParams);
Downloader.Queue(obj);
#if DEBUG
Interlocked.Increment(ref __queuedObjects);
GlobalLog.Write("__queuedObjects: {0}", __queuedObjects);
#endif
MoveObjQueued();
}
private void AddFaultTarget(DevourTarget target)
{
lock (sync)
{
_faultTargets.Add(target);
}
if (OnTargetError!=null)
{
OnTargetError(target);
}
}
void UnsubscribeDevourTarget(DevourTarget target)
{
target.Reader.OnNewTargets -= new NewTargetsDel(_reader_OnNewTargets);
target.Reader.OnReadComplete -= new ReadCompleteDel(MoveReadComplete);
}
ConcurrentQueue <DevourTarget> _targets; /* KEY -- faultCount, VALUE -- targetUri */
//RequestParams _params;
public Devourer(DevourTarget initialTarget, IProxyProvider proxies, IPageValidator validator = null)
: this(new List <DevourTarget>(new DevourTarget[] { initialTarget }), proxies, validator)
{
}
