//This method starts the processing. At least one fetcher should
//be already in queue before calling this method, otherwise
//the ProcessingOver event is fired immediately.
public void Process()
{
//Two threads are left spinning indefinitely - this one
//takes care of moving fetchers from the job queue to
//the executing queue and starting the fetchers.
ThreadStart enqueuerThread = () =>
{
while (true)
{
lock (_lockObject)
{
if (_queue.Any() && _executing.Count() < MAX_CONCURRENT_THREADS)
{
Fetcher nextFetcher = _queue.First();
_queue.Remove(nextFetcher);
_executing.Add(nextFetcher);
nextFetcher.Fetch();
}
}
}
};
//This thread checks which ones of the executing fetchers
//have completed. It then invokes the approriate callback.
ThreadStart dispatcherThread = () =>
{
while (true)
{
lock (_lockObject)
{
if (_executing.Any(x => x.Completed))
{
Fetcher completedFetcher = _executing.First();
if (completedFetcher.DownloadedPage != null)
{
OnCompleted(completedFetcher);
}
else
{
RemoveFetcher(completedFetcher);
}
}
}
}
};
new Thread(enqueuerThread).Start();
//The dispatcher thread's stack, due to the callback, can
//grow quite a lot - giving the thread stack more space
//(default is 1Mb) is not exactly elegant but effective in
//this instance. Running the queue on more than one
//system (as hypotized in the document) should mitigate
//the issue as well.
new Thread(dispatcherThread, DISPATCHER_THREAD_STACK_SIZE).Start();
}