static string PoolManager(
CollectorPool pool,
VisitedUrls urlHistory,
VisitedDomains domainHistory)
{
string link = pool.Next();
Uri uri = link.ToUri();
if (uri.IsNull())
return string.Empty; // No more links in the pool
// Check if url has been visited within this session
while (
urlHistory.ContainsUrl(uri.ToString()) ||
domainHistory.ContainsDomain(uri.ToString()) ||
IsExcludedDomain(pool, uri))
{
link = pool.Next();
uri = link.ToUri();
if (uri.IsNull())
return string.Empty; // No more links in the pool
}
// Recursively check if host is still processed
//if (domainHistory.ContainsDomain(uri.ToString()))
//{
// // Store the link back to pool
// pool.Store(uri.ToString());
// return PoolManager(pool, urlHistory, domainHistory);
//}
//else
//{
// Link is ok, add to history
urlHistory.Add(uri.ToString());
domainHistory.Add(uri.ToString());
return uri.ToString();
//}
}
static List<Collector> CreateCollectorPool(
CollectorPool pool,
VisitedUrls history,
int parallelCount)
{
List<Collector> collectors = new List<Collector>(parallelCount);
int i = 0;
while (i < parallelCount)
{
string link = pool.Next();
if (!history.ContainsUrl(link.ToUri().ToString()))
{
collectors.Add(new Collector
{
SeqNo = i,
Link = link
});
history.Add(link.ToUri().ToString());
++i;
}
}
return collectors;
}
