private static readonly string root_endpoint = "https://corona.azure-api.net"; // Must not have trailing slash.
/********************************************************************************/
static async Task Main(string[] args)
{
DateTime start;
List <Record> recs;
bool alreadyLoaded, success;
double time;
string delim = "";
for (int i = 0; i < 80; i++)
{
delim += "*";
}
/* Write delimiter to make new cycles easy to pick out in docker logs */
BoxWrite(delim);
BoxWrite(delim);
BoxWrite(delim);
BoxWrite("Begin:");
while (true)
{
start = DateTime.Now;
recs = new List <Record>();
time = 0;
/* Check for any records. */
alreadyLoaded = false;
try
{
using (RonaContext con = new RonaContext())
{
BoxWrite("Applying database migrations...");
await con.Database.MigrateAsync();
BoxWrite("Checking for existing records...");
alreadyLoaded = con.Records.Any();
}
}
catch (Exception ex)
{
PrintException(ex, false, "Hit an exception when checking for existing data. Did you scaffold the database? (and are you trying the correct host?)");
return;
}
if (!alreadyLoaded)
{
/* Looks like this is the first run. Grab old data and load the db. */
BoxWrite("No records found in database. Loading historical records from CSSE repo...");
recs = await GetHistoricalRecords();
if (recs == null)
{
BoxWrite($"Could not load historical data for initial database load.");
}
else if (recs.Count > 0)
{
BoxWrite("Loading records to database...");
success = LoadDbRecords(recs);
if (success)
{
BoxWrite($"Loaded {recs.Count():n0} historical records to the database.");
BoxWrite($"Current runtime: {(DateTime.Now - start).TotalMilliseconds / 1000:n3}s");
}
else
{
BoxWrite("There was a problem loading historical records to the database.");
}
}
}
/* Load today's data */
BoxWrite("Loading list of ISO2 country codes...");
List <string> countries = await GetCountries();
BoxWrite($"Found {countries.Count:n0} country codes to process.");
foreach (string country in countries)
{
// BoxWrite($"Gathering today's data for {country}...");
// recs = await GetNewRecordsAsync("US");
recs = await GetNewRecordsAsync(country);
if (recs != null && recs.Count > 0)
{
BoxWrite($"Loading {recs.Count:n0} record{(recs.Count == 1 ? "" : "s")} for {country} to database...");
LoadDbRecords(recs);
// BoxWrite($"{country}'s data has been loaded.");
}
else if (recs == null)
{
BoxWrite($"!!! There was an error retrieving data using ISO2 code \"{country}\".");
}
else
{
BoxWrite($"Retrieved 0 records for {country}.");
}
await Task.Delay(1000); //Avoid sending too many calls too fast if my code borks.
}
/* Finish up */
time = (DateTime.Now - start).TotalMilliseconds;
BoxWrite($"Done. Runtime: {(time/1000.0):n3}s");
BoxWrite($"Waiting {millisPerHour:n0}ms (1 hour) for the next data check.");
await Task.Delay(millisPerHour);
}
}//Main
private static bool LoadDbRecords(List <Record> recs)
{
/************************************************************
* Load Record objects to database. *
* The weird new-context loop is to prevent timeouts that *
* can result from loading hundreds of thousands of records *
* at once. *
************************************************************/
bool success;
List <string> existing;
IEnumerable <DateTime> dates = from r in recs select r.Last_Update;
DateTime minDate = dates.Min();
DateTime maxDate = dates.Max();
int startLen = 0, trimmedLen = 0, loadedSoFar = 0;
RonaContext con;
if (recs == null)
{
BoxWrite("Cannot load null list to database.");
return(false);
}
else if (recs.Count < 1)
{
BoxWrite("Cannot load empty list to database.");
return(false);
}
try
{
/* According to Some Guy On The Internet we should manually remove recs that conflict with db constraints. */
con = new RonaContext();
existing = (
from r in con.Records
where r.Last_Update >= minDate && r.Last_Update <= maxDate
select r.UniqueFields()
).ToList();
startLen = recs.Count;
recs = recs.Where(r => !existing.Contains(r.UniqueFields())).ToList();
trimmedLen = recs.Count;
if (trimmedLen < startLen)
{
BoxWrite($"After accounting for existing records, {trimmedLen:n0} records will be loaded.");
}
/* Break into subsections of recs to load quicker. */
var subLists = recs.Select((r, idx) => new { r, idx })
.GroupBy(a => a.idx / dbLoadInterval)
.Select((grp => grp.Select(g => g.r).ToList()))
.ToList();
foreach (List <Record> subList in subLists)
{
con = new RonaContext();
BoxWrite($"{loadedSoFar:n0} already loaded. Adding {subList.Count:n0} records to the database...");
con.Records.AddRange(subList);
con.SaveChanges();
loadedSoFar += subList.Count;
}
success = true;
}
catch (Exception ex)
{
success = false;
PrintException(ex, true, "Encountered an exception while writing records to database.");
}
return(success);
}
