protected static string DumpResults(CrawlResults results)
{
if (results.Classifications.Count == 0)
{
return("No Somerset matches found in " + results.Season);
}
StringBuilder sb = new StringBuilder();
sb.AppendLine(new string('-', 120));
string title = "Somerset matches in " + results.Season;
sb.Append("| ").Append(title.PadRight(116)).AppendLine(" |");
sb.AppendLine(new string('-', 120));
var matches = from c in results.Classifications
from sc in c.Scorecards
select sc;
foreach (ScorecardDetails match in matches)
{
sb.Append("| ").Append(FormatValue(match, 116)).AppendLine(" |");
}
sb.AppendLine(new string('-', 120));
return(sb.ToString());
}
private static void DownloadScorecards(FileStore dataStore, Season season)
{
CrawlResults crawlResults = GetCrawlResultsForSeason(dataStore, season);
if (crawlResults == null)
{
Log.WarnFormat("Season {0} has not been crawled yet.", season.Name);
return;
}
Log.InfoFormat("Scorecard download started at {0} for season {1}", DateTime.Now.ToShortTimeString(), crawlResults.Season);
var matchRecords = crawlResults.Classifications.SelectMany(m => m.Scorecards);
Queue <Task> tasks = new Queue <Task>();
foreach (ScorecardDetails md in matchRecords)
{
if (md.ScorecardAvailable && !string.IsNullOrEmpty(md.ScorecardUrl))
{
Log.InfoFormat("Downloading scorecard for {0}", md);
tasks.Enqueue(DownloadScorecardAsync(md, dataStore));
}
}
Task.WaitAll(tasks.ToArray());
SaveCrawlerResults(crawlResults, dataStore);
Log.InfoFormat("Scorecard download finished at {0} for season {1}", DateTime.Now.ToShortTimeString(), crawlResults.Season);
}
private CrawlResults GetResults(HttpClient client, string statusUri)
{
CrawlResults results = null;
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, statusUri))
{
using (HttpResponseMessage response = client.SendAsync(request).Result)
{
switch (response.StatusCode)
{
case HttpStatusCode.Accepted:
break;
case HttpStatusCode.OK:
var rawResults = response.Content.ReadAsStringAsync().Result;
var statusResponse = JsonConvert.DeserializeObject <StatusResponse>(rawResults);
if (statusResponse.RuntimeStatus == "Completed")
{
results = statusResponse.Output;
}
else
{
throw new Exception($"Unexpected RuntimeStatus: {statusResponse.RuntimeStatus}");
}
break;
default:
throw new Exception($"Failed to get status for Crawl. Received error code {response.StatusCode}");
}
}
}
return(results);
}
protected static void SaveCrawlerResults(CrawlResults results, FileStore dataStore)
{
if (string.IsNullOrEmpty(results.Id))
{
results.Id = CrawlResults.GenerateKey(results);
}
dataStore.Save(results, results.Id);
}
public CrawlResults Recheck(CrawlResults crawlResults)
{
Console.Write("Rechecking {0} ", crawlResults.Season);
var classificationLinks = from c in crawlResults.Classifications
select
new CrawlerLinkDetails
{
DestinationPageType = PageType.MatchList,
DestinationUrl = c.Url,
LinkText = c.Name,
SourcePageType = PageType.MatchClassification,
SourcePageUrl = c.LocationIndexUrl
};
List <MatchClassification> matchClassifications = new List <MatchClassification>();
var taskQueue = new Queue <Task <MatchClassification> >();
foreach (var link in classificationLinks)
{
CrawlerLinkDetails l = link;
taskQueue.Enqueue(Task <MatchClassification> .Factory.StartNew(() => CrawlClassification(l, l.LinkText, crawlResults.Season)));
}
if (taskQueue.Count == 0)
{
Console.WriteLine(" done.");
Log.InfoFormat("No URLs to check for season {0}", crawlResults.Season);
return(crawlResults);
}
Task.Factory.ContinueWhenAll(taskQueue.ToArray(),
completedTasks =>
{
foreach (Task <MatchClassification> task in completedTasks)
{
if (task.Exception == null)
{
matchClassifications.Add(task.Result);
}
else
{
Log.Error("Unexpected exception",
task.Exception);
}
}
})
.Wait();
Console.WriteLine(" done.");
return(new CrawlResults
{
Id = crawlResults.Id,
Season = crawlResults.Season,
Classifications = matchClassifications
});
}
public void Crawl(CrawlRequest crawlRequest)
{
_results = null;
using (HttpClient client = new HttpClient())
{
var statusUri = Start(client, crawlRequest);
_results = GetResultsWithRetry(client, statusUri, RESULT_RETRIES);
}
}
public void Execute(FileStore dataStore)
{
foreach (Season season in GetSeasons(dataStore, StartSeason, EndSeason))
{
Log.DebugFormat("Crawling Season {0} at {1}", season.Name, season.Url);
CrawlResults results = RunCrawler(season);
SaveCrawlerResults(results, dataStore);
}
}
private static CrawlResults RunCrawler(Season season)
{
Log.InfoFormat("Crawler started at {0} for season {1}", DateTime.Now.ToShortTimeString(), season.Name);
Spider crawler = new Spider();
CrawlResults results = crawler.Crawl(season);
Log.InfoFormat("\n{0}", DumpResults(results));
Log.InfoFormat("Crawler finished at {0}.", DateTime.Now.ToShortTimeString());
return(results);
}
public CrawlResults Crawl(Season season)
{
Console.Write("Crawling " + season.Name + " ");
CrawlResults results = new CrawlResults {
Season = season.Name
};
CrawlerLinkDetails seasonPage = new CrawlerLinkDetails
{
SourcePageType = PageType.SeasonList,
SourcePageUrl = "http://cricketarchive.com/Archive/Seasons/index.html",
DestinationPageType = PageType.LocationList,
DestinationUrl = season.Url,
LinkText = season.Name
};
List <CrawlerLinkDetails> locationLinks = PageCrawler.CrawlLinksPage(seasonPage);
List <MatchClassification> classifications = new List <MatchClassification>();
var taskQueue = new Queue <Task <List <MatchClassification> > >();
foreach (var link in locationLinks)
{
CrawlerLinkDetails l = link;
taskQueue.Enqueue(Task <List <MatchClassification> > .Factory.StartNew(() => CrawlLocation(l, season.Name)));
}
Task.Factory.ContinueWhenAll(taskQueue.ToArray(),
completedTasks =>
{
foreach (Task <List <MatchClassification> > task in completedTasks)
{
if (task.Exception == null)
{
classifications.AddRange(task.Result);
}
else
{
Log.Error("Unexpected exception",
task.Exception);
}
}
})
.Wait();
results.Classifications = classifications;
Console.WriteLine(" done.");
return(results);
}
private static void RecheckSeason(FileStore dataStore, Season season)
{
CrawlResults existing = GetCrawlResultsForSeason(dataStore, season);
if (existing == null)
{
Log.WarnFormat("Season {0} has not been crawled yet.", season.Name);
return;
}
Spider spider = new Spider();
CrawlResults recheckResults = spider.Recheck(existing);
Log.InfoFormat("\n{0}", DumpResults(recheckResults));
Log.InfoFormat("Crawler finished at {0}.", DateTime.Now.ToShortTimeString());
SaveCrawlerResults(recheckResults, dataStore);
}
private static void ParseScorecards(FileStore dataStore, Season season)
{
CrawlResults crawlResults = GetCrawlResultsForSeason(dataStore, season);
if (crawlResults == null)
{
Log.WarnFormat("Season {0} has not been crawled yet.", season.Name);
return;
}
Log.InfoFormat("Scorecard parsing started at {0} for season {1}", DateTime.Now.ToShortTimeString(), crawlResults.Season);
var matchRecords = crawlResults.Classifications.SelectMany(m => m.Scorecards);
foreach (ScorecardDetails md in matchRecords)
{
ParseScorecard(dataStore, md);
}
Log.InfoFormat("Scorecard parsing finished at {0} for season {1}", DateTime.Now.ToShortTimeString(), season.Name);
}
private CrawlResults GetResultsWithRetry(HttpClient client, string statusUri, int retries)
{
CrawlResults results = null;
int attempts = 0;
while (results == null && attempts < retries)
{
// Sleep between each check
Thread.Sleep(SECONDS_TO_SLEEP_BETWEEN_RESULT_ATTEMPTS);
attempts++;
results = GetResults(client, statusUri);
}
// If attempts have hit the retries and still no results, then throw an error
if (results == null && attempts == retries)
{
throw new Exception(String.Format("Unable to received crawl results for {0} - have tried {1} times", statusUri, attempts));
}
return(results);
}
private void RunMapFunctions(FileStore dataStore, Season season)
{
CrawlResults crawlResults = GetCrawlResultsForSeason(dataStore, season);
if (crawlResults == null)
{
Log.WarnFormat("Season {0} has not been crawled yet.", season.Name);
return;
}
Log.InfoFormat("Map started at {0} for season {1}", DateTime.Now.ToShortTimeString(), crawlResults.Season);
var matchRecords = crawlResults.Classifications.SelectMany(m => m.Scorecards);
if (matchRecords.Count() == 0)
{
Log.InfoFormat("No match records found for {0}", season.Name);
return;
}
var battingTasks = new Queue <Task <List <BattingRecord> > >();
var bowlingTasks = new Queue <Task <List <BowlingRecord> > >();
var fieldingTasks = new Queue <Task <List <FieldingRecord> > >();
foreach (ScorecardDetails details in matchRecords)
{
string id = CricketMatch.GenerateId(details.Season, details.MatchCode);
CricketMatch match = dataStore.Load <CricketMatch>(id);
if (match != null)
{
battingTasks.Enqueue(Task <List <BattingRecord> > .Factory.StartNew(() => IndividualBattingMap.Run(match)));
bowlingTasks.Enqueue(Task <List <BowlingRecord> > .Factory.StartNew(() => IndividualBowlingMap.Run(match)));
fieldingTasks.Enqueue(Task <List <FieldingRecord> > .Factory.StartNew(() => IndividualFieldingMap.Run(match)));
}
}
List <BattingRecord> battingRecords = new List <BattingRecord>();
List <BowlingRecord> bowlingRecords = new List <BowlingRecord>();
List <FieldingRecord> fieldingRecords = new List <FieldingRecord>();
Task[] continuations = new[] {
Task.Factory.ContinueWhenAll(battingTasks.ToArray(),
completedTasks =>
{
foreach (var task in completedTasks)
{
if (task.Exception == null)
{
battingRecords.AddRange(task.Result);
}
else
{
Log.Error("Unexpected exception", task.Exception);
}
}
}),
Task.Factory.ContinueWhenAll(bowlingTasks.ToArray(),
completedTasks =>
{
foreach (var task in completedTasks)
{
if (task.Exception == null)
{
bowlingRecords.AddRange(task.Result);
}
else
{
Log.Error("Unexpected exception", task.Exception);
}
}
}),
Task.Factory.ContinueWhenAll(fieldingTasks.ToArray(),
completedTasks =>
{
foreach (var task in completedTasks)
{
if (task.Exception == null)
{
fieldingRecords.AddRange(task.Result);
}
else
{
Log.Error("Unexpected exception", task.Exception);
}
}
})
};
Task.WaitAll(continuations);
dataStore.Save(battingRecords, IndividualBattingMap.GenerateId(season.Name));
dataStore.Save(bowlingRecords, IndividualBowlingMap.GenerateId(season.Name));
dataStore.Save(fieldingRecords, IndividualFieldingMap.GenerateId(season.Name));
Log.InfoFormat("Scorecard parsing finished at {0} for season {1}", DateTime.Now.ToShortTimeString(), season.Name);
}
