private static readonly string root_endpoint = "https://corona.azure-api.net"; // Must not have trailing slash. /********************************************************************************/ static async Task Main(string[] args) { DateTime start; List <Record> recs; bool alreadyLoaded, success; double time; string delim = ""; for (int i = 0; i < 80; i++) { delim += "*"; } /* Write delimiter to make new cycles easy to pick out in docker logs */ BoxWrite(delim); BoxWrite(delim); BoxWrite(delim); BoxWrite("Begin:"); while (true) { start = DateTime.Now; recs = new List <Record>(); time = 0; /* Check for any records. */ alreadyLoaded = false; try { using (RonaContext con = new RonaContext()) { BoxWrite("Applying database migrations..."); await con.Database.MigrateAsync(); BoxWrite("Checking for existing records..."); alreadyLoaded = con.Records.Any(); } } catch (Exception ex) { PrintException(ex, false, "Hit an exception when checking for existing data. Did you scaffold the database? (and are you trying the correct host?)"); return; } if (!alreadyLoaded) { /* Looks like this is the first run. Grab old data and load the db. */ BoxWrite("No records found in database. Loading historical records from CSSE repo..."); recs = await GetHistoricalRecords(); if (recs == null) { BoxWrite($"Could not load historical data for initial database load."); } else if (recs.Count > 0) { BoxWrite("Loading records to database..."); success = LoadDbRecords(recs); if (success) { BoxWrite($"Loaded {recs.Count():n0} historical records to the database."); BoxWrite($"Current runtime: {(DateTime.Now - start).TotalMilliseconds / 1000:n3}s"); } else { BoxWrite("There was a problem loading historical records to the database."); } } } /* Load today's data */ BoxWrite("Loading list of ISO2 country codes..."); List <string> countries = await GetCountries(); BoxWrite($"Found {countries.Count:n0} country codes to process."); foreach (string country in countries) { // BoxWrite($"Gathering today's data for {country}..."); // recs = await GetNewRecordsAsync("US"); recs = await GetNewRecordsAsync(country); if (recs != null && recs.Count > 0) { BoxWrite($"Loading {recs.Count:n0} record{(recs.Count == 1 ? "" : "s")} for {country} to database..."); LoadDbRecords(recs); // BoxWrite($"{country}'s data has been loaded."); } else if (recs == null) { BoxWrite($"!!! There was an error retrieving data using ISO2 code \"{country}\"."); } else { BoxWrite($"Retrieved 0 records for {country}."); } await Task.Delay(1000); //Avoid sending too many calls too fast if my code borks. } /* Finish up */ time = (DateTime.Now - start).TotalMilliseconds; BoxWrite($"Done. Runtime: {(time/1000.0):n3}s"); BoxWrite($"Waiting {millisPerHour:n0}ms (1 hour) for the next data check."); await Task.Delay(millisPerHour); } }//Main
private static bool LoadDbRecords(List <Record> recs) { /************************************************************ * Load Record objects to database. * * The weird new-context loop is to prevent timeouts that * * can result from loading hundreds of thousands of records * * at once. * ************************************************************/ bool success; List <string> existing; IEnumerable <DateTime> dates = from r in recs select r.Last_Update; DateTime minDate = dates.Min(); DateTime maxDate = dates.Max(); int startLen = 0, trimmedLen = 0, loadedSoFar = 0; RonaContext con; if (recs == null) { BoxWrite("Cannot load null list to database."); return(false); } else if (recs.Count < 1) { BoxWrite("Cannot load empty list to database."); return(false); } try { /* According to Some Guy On The Internet we should manually remove recs that conflict with db constraints. */ con = new RonaContext(); existing = ( from r in con.Records where r.Last_Update >= minDate && r.Last_Update <= maxDate select r.UniqueFields() ).ToList(); startLen = recs.Count; recs = recs.Where(r => !existing.Contains(r.UniqueFields())).ToList(); trimmedLen = recs.Count; if (trimmedLen < startLen) { BoxWrite($"After accounting for existing records, {trimmedLen:n0} records will be loaded."); } /* Break into subsections of recs to load quicker. */ var subLists = recs.Select((r, idx) => new { r, idx }) .GroupBy(a => a.idx / dbLoadInterval) .Select((grp => grp.Select(g => g.r).ToList())) .ToList(); foreach (List <Record> subList in subLists) { con = new RonaContext(); BoxWrite($"{loadedSoFar:n0} already loaded. Adding {subList.Count:n0} records to the database..."); con.Records.AddRange(subList); con.SaveChanges(); loadedSoFar += subList.Count; } success = true; } catch (Exception ex) { success = false; PrintException(ex, true, "Encountered an exception while writing records to database."); } return(success); }