private Uri DequeueURI()
{
KeyValuePair <DateTime, string> oldestAccessedDomain = backQueueHeap.DeleteMinimum();
DateTime nextPossibleAccessTime = oldestAccessedDomain.Key; //The next time we may access this domain
string domain = oldestAccessedDomain.Value; //Get the domain accessed most in the past
if (nextPossibleAccessTime > DateTime.UtcNow)
{
Thread.Sleep((nextPossibleAccessTime - DateTime.UtcNow).Milliseconds);
}
int queueIndex = domainBackQueues[domain]; //Retrieve the queue for the domain
Uri uri = backQueues[queueIndex].Dequeue(); //Get the next URI for the domain
if (backQueues[queueIndex].Count == 0)
{
domainBackQueues.Remove(domain); // Remove the old domain
if (frontier.Count == 0)
{
TrashEmptyQueue(queueIndex);
return(uri);
}
Uri newUri = frontier.Dequeue();
domain = newUri.Host; //Overwrite the previous domain, whose queue was emptied
while (domainBackQueues.ContainsKey(domain))
{
int oldDomainQueueIndex = domainBackQueues[domain]; //The new domain is actually not new and already has a queue
backQueues[oldDomainQueueIndex].Enqueue(newUri);
if (frontier.Count == 0)
{
TrashEmptyQueue(queueIndex);
return(uri);
}
newUri = frontier.Dequeue();
domain = newUri.Host;
}
domainBackQueues.Add(domain, queueIndex); //Add the new domain
backQueues[queueIndex].Enqueue(newUri);
}
backQueueHeap.Insert(DateTime.UtcNow.AddSeconds(4), domain);
return(uri);
}
