/// <summary>
/// Downloads a set of URIs in parallel using a ThreadPool.
/// </summary>
/// <param name="parentRequest">Root request.</param>
/// <param name="childObjects">Children URIs to be downloaded.</param>
/// <returns>List of downloaded requests.</returns>
private LinkedList<RCRequest> DownloadObjectsInParallel(RCRequest parentRequest, LinkedList<Uri> childObjects)
{
LinkedList<RCRequest> addedObjects = new LinkedList<RCRequest>();
if (_killYourself || childObjects.Count == 0)
{
return addedObjects;
}
parentRequest.ResetEvents = new ManualResetEvent[childObjects.Count];
try
{
// queue up worker threads to download URIs
for (int i = 0; i < childObjects.Count; i++)
{
// create the RCRequest for the object
RCRequest currChildObject = new RCRequest(_proxy, (HttpWebRequest)WebRequest.Create(childObjects.ElementAt(i)));
currChildObject.ChildNumber = i;
// Set the root request.
currChildObject.RootRequest = parentRequest;
addedObjects.AddLast(currChildObject);
// set the resetEvent
currChildObject.ResetEvents = parentRequest.ResetEvents;
parentRequest.ResetEvents[i] = new ManualResetEvent(false);
// make sure we haven't downloaded this before
if (_package.RCRequests.Contains(currChildObject))
{
// skip it
currChildObject.SetDone();
continue;
}
// download the page
//LogDebug("queueing: " + currChild.ChildNumber + " " + currChild.Uri);
ThreadPool.QueueUserWorkItem(new WaitCallback(DownloadPageWorkerThread), (object)currChildObject);
}
// wait for timeout
Utils.WaitAll(parentRequest.ResetEvents);
addedObjects = _package.Pack(this, addedObjects, ref _quota);
}
catch (Exception e)
{
Logger.Warn("unable to download embeddedObjects.", e);
}
return addedObjects;
}
/// <summary>
/// Downloads a set of URIs in series.
/// </summary>
/// <param name="parentRequest">Root request.</param>
/// <param name="children">Children requests to be downloaded.</param>
/// <returns>List of downloaded requests.</returns>
private LinkedList<RCRequest> DownloadObjects(RCRequest parentRequest, LinkedList<RCRequest> children)
{
LinkedList<RCRequest> addedObjects = new LinkedList<RCRequest>();
if (children.Count == 0)
{
return addedObjects;
}
parentRequest.ResetEvents = new ManualResetEvent[children.Count];
try
{
// queue up worker threads to download URIs
for (int i = 0; i < children.Count; i++)
{
RCRequest currChild = children.ElementAt(i);
// make sure we haven't downloaded this before
if (_package.RCRequests.Contains(currChild))
{
// skip it
parentRequest.SetDone();
continue;
}
// reduce the timer
DateTime currTime = DateTime.Now;
DateTime endTime = parentRequest.StartTime.AddMilliseconds(RequestHandler.WEB_REQUEST_DEFAULT_TIMEOUT);
if (endTime.CompareTo(currTime) > 0)
{
currChild.GenericWebRequest.Timeout = (int)(endTime.Subtract(currTime)).TotalMilliseconds;
}
else
{
currChild.GenericWebRequest.Timeout = 0;
}
// download the page
currChild.DownloadToCache(false);
if (IsTimedOut())
{
break;
}
}
addedObjects = _package.Pack(this, children, ref _quota);
}
catch (Exception e)
{
LogDebug("unable to download embeddedObjects: " + e.StackTrace + " " + e.Message);
}
return addedObjects;
}
