コード例 #1
0
        private static readonly string root_endpoint          = "https://corona.azure-api.net"; // Must not have trailing slash.
        /********************************************************************************/
        static async Task Main(string[] args)
        {
            DateTime      start;
            List <Record> recs;
            bool          alreadyLoaded, success;
            double        time;
            string        delim = "";

            for (int i = 0; i < 80; i++)
            {
                delim += "*";
            }

            /* Write delimiter to make new cycles easy to pick out in docker logs */
            BoxWrite(delim);
            BoxWrite(delim);
            BoxWrite(delim);
            BoxWrite("Begin:");

            while (true)
            {
                start = DateTime.Now;
                recs  = new List <Record>();
                time  = 0;

                /* Check for any records. */
                alreadyLoaded = false;
                try
                {
                    using (RonaContext con = new RonaContext())
                    {
                        BoxWrite("Applying database migrations...");
                        await con.Database.MigrateAsync();

                        BoxWrite("Checking for existing records...");
                        alreadyLoaded = con.Records.Any();
                    }
                }
                catch (Exception ex)
                {
                    PrintException(ex, false, "Hit an exception when checking for existing data. Did you scaffold the database? (and are you trying the correct host?)");
                    return;
                }

                if (!alreadyLoaded)
                {
                    /* Looks like this is the first run. Grab old data and load the db. */
                    BoxWrite("No records found in database. Loading historical records from CSSE repo...");
                    recs = await GetHistoricalRecords();

                    if (recs == null)
                    {
                        BoxWrite($"Could not load historical data for initial database load.");
                    }
                    else if (recs.Count > 0)
                    {
                        BoxWrite("Loading records to database...");
                        success = LoadDbRecords(recs);
                        if (success)
                        {
                            BoxWrite($"Loaded {recs.Count():n0} historical records to the database.");
                            BoxWrite($"Current runtime: {(DateTime.Now - start).TotalMilliseconds / 1000:n3}s");
                        }
                        else
                        {
                            BoxWrite("There was a problem loading historical records to the database.");
                        }
                    }
                }

                /* Load today's data */
                BoxWrite("Loading list of ISO2 country codes...");
                List <string> countries = await GetCountries();

                BoxWrite($"Found {countries.Count:n0} country codes to process.");
                foreach (string country in countries)
                {
                    // BoxWrite($"Gathering today's data for {country}...");
                    // recs = await GetNewRecordsAsync("US");
                    recs = await GetNewRecordsAsync(country);

                    if (recs != null && recs.Count > 0)
                    {
                        BoxWrite($"Loading {recs.Count:n0} record{(recs.Count == 1 ? "" : "s")} for {country} to database...");
                        LoadDbRecords(recs);
                        // BoxWrite($"{country}'s data has been loaded.");
                    }
                    else if (recs == null)
                    {
                        BoxWrite($"!!! There was an error retrieving data using ISO2 code \"{country}\".");
                    }
                    else
                    {
                        BoxWrite($"Retrieved 0 records for {country}.");
                    }
                    await Task.Delay(1000); //Avoid sending too many calls too fast if my code borks.
                }

                /* Finish up */
                time = (DateTime.Now - start).TotalMilliseconds;
                BoxWrite($"Done. Runtime: {(time/1000.0):n3}s");
                BoxWrite($"Waiting {millisPerHour:n0}ms (1 hour) for the next data check.");
                await Task.Delay(millisPerHour);
            }
        }//Main
コード例 #2
0
        private static bool LoadDbRecords(List <Record> recs)
        {
            /************************************************************
            * Load Record objects to database.                         *
            * The weird new-context loop is to prevent timeouts that   *
            * can result from loading hundreds of thousands of records *
            * at once.                                                 *
            ************************************************************/
            bool                   success;
            List <string>          existing;
            IEnumerable <DateTime> dates = from r in recs select r.Last_Update;
            DateTime               minDate = dates.Min();
            DateTime               maxDate = dates.Max();
            int         startLen = 0, trimmedLen = 0, loadedSoFar = 0;
            RonaContext con;

            if (recs == null)
            {
                BoxWrite("Cannot load null list to database.");
                return(false);
            }
            else if (recs.Count < 1)
            {
                BoxWrite("Cannot load empty list to database.");
                return(false);
            }
            try
            {
                /* According to Some Guy On The Internet we should manually remove recs that conflict with db constraints. */
                con      = new RonaContext();
                existing = (
                    from r in con.Records
                    where r.Last_Update >= minDate && r.Last_Update <= maxDate
                    select r.UniqueFields()
                    ).ToList();
                startLen   = recs.Count;
                recs       = recs.Where(r => !existing.Contains(r.UniqueFields())).ToList();
                trimmedLen = recs.Count;

                if (trimmedLen < startLen)
                {
                    BoxWrite($"After accounting for existing records, {trimmedLen:n0} records will be loaded.");
                }

                /* Break into subsections of recs to load quicker. */
                var subLists = recs.Select((r, idx) => new { r, idx })
                               .GroupBy(a => a.idx / dbLoadInterval)
                               .Select((grp => grp.Select(g => g.r).ToList()))
                               .ToList();
                foreach (List <Record> subList in subLists)
                {
                    con = new RonaContext();
                    BoxWrite($"{loadedSoFar:n0} already loaded. Adding {subList.Count:n0} records to the database...");
                    con.Records.AddRange(subList);
                    con.SaveChanges();
                    loadedSoFar += subList.Count;
                }
                success = true;
            }
            catch (Exception ex)
            {
                success = false;
                PrintException(ex, true, "Encountered an exception while writing records to database.");
            }
            return(success);
        }