public static void UpdateSeason([Optional] int year, [Optional] bool forceOverwrite)
{
year = NhlModelHelper.SetDefaultYear(year);
List <Nhl_Games_Rtss> models;
using (SportsDataContext db = new SportsDataContext())
{
models = (from m in db.Nhl_Games_Rtss_DbSet
where
m.Year == year
select m).ToList();
}
// Retrieve the links of the blobs
Dictionary <Uri, string> summaryItems = new Dictionary <Uri, string>();
Dictionary <Uri, string> rosterItems = new Dictionary <Uri, string>();
foreach (Nhl_Games_Rtss m in models)
{
// Game Summary
if (Uri.IsWellFormedUriString(m.GameLink, UriKind.Absolute))
{
summaryItems.Add(new Uri(m.GameLink), m.Id.ToString());
}
// Roster
if (Uri.IsWellFormedUriString(m.RosterLink, UriKind.Absolute))
{
rosterItems.Add(new Uri(m.RosterLink), m.Id.ToString());
}
}
HtmlBlob.GetAndStoreHtmlBlobs(HtmlBlobType.NhlGame, summaryItems, forceOverwrite);
HtmlBlob.GetAndStoreHtmlBlobs(HtmlBlobType.NhlRoster, rosterItems, forceOverwrite);
}
public static void SaveBlob(HtmlBlobType htmlBlobType, string htmlBlobId, Uri uri, string html)
{
string blobName = HtmlBlob.ConstructBlobName(htmlBlobType, htmlBlobId, uri);
CloudBlockBlob cloudBlockBlob = HtmlBlob.CloudBlobContainer.GetBlockBlobReference(blobName);
cloudBlockBlob.UploadText(html);
}
public static bool BlobExists(HtmlBlobType htmlBlobType, string htmlBlobId, Uri uri)
{
string blobName = HtmlBlob.ConstructBlobName(htmlBlobType, htmlBlobId, uri);
CloudBlockBlob cloudBlockBlob = HtmlBlob.CloudBlobContainer.GetBlockBlobReference(blobName);
return(cloudBlockBlob.Exists());
}
public static void GetAndStoreHtmlBlobs(HtmlBlobType htmlBlobType, Dictionary <Uri, string> items, [Optional] bool forceOverwrite)
{
foreach (Uri uri in items.Keys)
{
HtmlBlob.GetAndStoreHtmlBlob(htmlBlobType, items[uri], uri, forceOverwrite);
}
}
public static void GetAndStoreHtmlBlob(HtmlBlobType htmlBlobType, string htmlBlobId, Uri uri, [Optional] bool forceOverwrite)
{
if (forceOverwrite == false && HtmlBlob.BlobExists(htmlBlobType, htmlBlobId, uri))
{
// Blob exists and we don't want to force an overwrite so do nothing
return;
}
string htmlBlob = HtmlBlob.GetHtmlPage(uri);
if (!String.IsNullOrWhiteSpace(htmlBlob))
{
HtmlBlob.SaveBlob(htmlBlobType, htmlBlobId, uri, htmlBlob);
}
}
public static string RetrieveBlob(HtmlBlobType htmlBlobType, string htmlBlobId, Uri uri, bool getIfNotExists = false)
{
// Get the blob if it doesn't exist
if (getIfNotExists == true)
{
// This method will not do anything if the blob already exists
HtmlBlob.GetAndStoreHtmlBlob(htmlBlobType, htmlBlobId, uri);
}
string blobName = HtmlBlob.ConstructBlobName(htmlBlobType, htmlBlobId, uri);
CloudBlockBlob cloudBlockBlob = HtmlBlob.CloudBlobContainer.GetBlockBlobReference(blobName);
string result = null;
try
{
result = cloudBlockBlob.DownloadText();
}
catch (System.AggregateException e)
{
HtmlBlob.PrintGetHtmlPageException(cloudBlockBlob.SnapshotQualifiedUri, e);
bool is404Exception = e.InnerExceptions.Where(ie =>
ie.GetType() == typeof(HttpRequestException) &&
ie.Message.IndexOf("404 (Not Found).", StringComparison.InvariantCultureIgnoreCase) >= 0).Any();
if (!is404Exception)
{
//throw;
}
}
catch (Exception e)
{
HtmlBlob.PrintGetHtmlPageException(cloudBlockBlob.SnapshotQualifiedUri, e);
//throw;
}
return(result);
}
public static string GetHtmlPage(Uri uri)
{
string responseString = null;
try
{
HttpClient httpClient = new HttpClient();
//httpClient.DefaultRequestHeaders.Accept.ParseAdd("text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
//httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36");
Task <string> response = httpClient.GetStringAsync(uri);
responseString = response.Result;
}
catch (System.AggregateException e)
{
HtmlBlob.PrintGetHtmlPageException(uri, e);
bool is404Exception = e.InnerExceptions.Where(ie =>
ie.GetType() == typeof(HttpRequestException) &&
ie.Message.IndexOf("404 (Not Found).", StringComparison.InvariantCultureIgnoreCase) >= 0).Any();
if (!is404Exception)
{
//throw;
}
else
{
responseString = "404";
}
}
catch (Exception e)
{
HtmlBlob.PrintGetHtmlPageException(uri, e);
//throw;
}
return(responseString);
}
public static void UpdateSeason([Optional] int year, [Optional] DateTime fromDate, [Optional] bool forceOverwrite)
{
// Get the RtssReports for the specified year. Exclude recent games that may not be done (UtcNow - 1)
List <Nhl_Games_Rtss> models = NhlHtmlReportBase.GetRtssReports(year, fromDate).Where(m => m.Date < DateTime.UtcNow.AddDays(-1).Date).ToList();
List <Nhl_Games_Rtss_Summary> existingModels = null;
if (forceOverwrite == false)
{
// Only query for existing if we are not going to force overwrite all
existingModels = NhlGamesRtssSummary.GetHtmlSummaryReports(year, fromDate);
}
// For each report, get the html blob from blob storage and parse the blob to a report
List <Nhl_Games_Rtss_Summary> results = new List <Nhl_Games_Rtss_Summary>();
foreach (Nhl_Games_Rtss model in models)
{
if (forceOverwrite == false && existingModels.Exists(m => m.NhlRtssReportModelId == model.Id))
{
// In this case, only get data if it is not already populated
continue;
}
Nhl_Games_Rtss_Summary report = null;
if (!model.GameLink.Equals("#"))
{
string htmlBlob = HtmlBlob.RetrieveBlob(HtmlBlobType.NhlRoster, model.Id.ToString(), new Uri(model.GameLink), true);
report = NhlGamesRtssSummary.ParseHtmlBlob(model.Id, htmlBlob);
}
if (null != report)
{
results.Add(report);
}
}
// Save the reports to the db
using (SportsDataContext db = new SportsDataContext())
{
Console.WriteLine("Start saving {0} to {1}", results.Count, db.Database.Connection.ConnectionString);
db.Configuration.AutoDetectChangesEnabled = false;
db.Configuration.ValidateOnSaveEnabled = false;
int counter = 0;
int totalCounter = 0;
int batchSize = 10;
foreach (var model in results)
{
counter++;
totalCounter++;
if (model.Id != 0)
{
db.Nhl_Games_Rtss_Summary_DbSet.Attach(model);
db.Entry(model).State = EntityState.Modified;
}
else
{
db.Entry(model).State = EntityState.Added;
}
if (counter >= batchSize)
{
db.SaveChanges();
counter = 0;
Console.WriteLine("Saved {0} of {1}", totalCounter, results.Count);
}
}
db.SaveChanges();
Console.WriteLine("Saved {0} of {1}", totalCounter, results.Count);
}
}
private static void PrintGetHtmlPageException(Uri uri, Exception e)
{
HtmlBlob.PrintGetHtmlPageException(uri.AbsoluteUri, e);
}
