private static List <MatchClassification> CrawlLocation(CrawlerLinkDetails locationLink, string season)
{
List <MatchClassification> matchClassifications = new List <MatchClassification>();
var taskQueue = new Queue <Task <MatchClassification> >();
foreach (var link in PageCrawler.CrawlLinksPage(locationLink))
{
CrawlerLinkDetails l = link;
taskQueue.Enqueue(Task <MatchClassification> .Factory.StartNew(() => CrawlClassification(l, l.LinkText, season)));
}
Task.Factory.ContinueWhenAll(taskQueue.ToArray(),
completedTasks =>
{
foreach (Task <MatchClassification> task in completedTasks)
{
if (task.Exception == null)
{
matchClassifications.Add(task.Result);
}
else
{
Log.Error("Unexpected exception",
task.Exception);
}
}
})
.Wait();
return(matchClassifications);
}
public CrawlResults Crawl(Season season)
{
Console.Write("Crawling " + season.Name + " ");
CrawlResults results = new CrawlResults {
Season = season.Name
};
CrawlerLinkDetails seasonPage = new CrawlerLinkDetails
{
SourcePageType = PageType.SeasonList,
SourcePageUrl = "http://cricketarchive.com/Archive/Seasons/index.html",
DestinationPageType = PageType.LocationList,
DestinationUrl = season.Url,
LinkText = season.Name
};
List <CrawlerLinkDetails> locationLinks = PageCrawler.CrawlLinksPage(seasonPage);
List <MatchClassification> classifications = new List <MatchClassification>();
var taskQueue = new Queue <Task <List <MatchClassification> > >();
foreach (var link in locationLinks)
{
CrawlerLinkDetails l = link;
taskQueue.Enqueue(Task <List <MatchClassification> > .Factory.StartNew(() => CrawlLocation(l, season.Name)));
}
Task.Factory.ContinueWhenAll(taskQueue.ToArray(),
completedTasks =>
{
foreach (Task <List <MatchClassification> > task in completedTasks)
{
if (task.Exception == null)
{
classifications.AddRange(task.Result);
}
else
{
Log.Error("Unexpected exception",
task.Exception);
}
}
})
.Wait();
results.Classifications = classifications;
Console.WriteLine(" done.");
return(results);
}