Beispiel #1
0
        public List <PrecipitationRecord> Process()
        {
            if (!File.Exists(filename))
            {
                throw new FileNotFoundException(filename);
            }

            //Clear database if it already exists
            using (JBADbContext db = new JBADbContext(filename))
            {
                db.Database.EnsureDeleted();
                db.Database.EnsureCreated();
                db.SaveChanges();
            }

            using (StreamReader file =
                       new StreamReader(filename))
            {
                if (TryReadHeader(file))
                {
                    ParseRecords(file);
                    return(recordsToAdd);
                }

                throw new ParseException("Unable to read file header.");
            }
        }
Beispiel #2
0
        private static void WriteRecords(List <PrecipitationRecord> records, bool rawSql, string filename)
        {
            if (rawSql)
            {
                //need to manually increment ids since EF is not used
                int idCounter = 0;
                //batch inserts in chunks to avoid massive query size
                using (JBADbContext context = new JBADbContext(filename))
                {
                    context.ChangeTracker.AutoDetectChangesEnabled = false;
                    for (int i = 0; i < Math.Ceiling((double)records.Count / 10000); i++)
                    {
                        StringBuilder builder = new StringBuilder("INSERT INTO \"PrecipitationRecords\" VALUES ");
                        //safe to append values directly as the were parsed into primitives previously
                        builder.AppendJoin(",", records.Skip(10000 * i).Take(10000).Select(
                                               x =>
                                               $"({idCounter++}, {x.Xref}, {x.Yref}, \"{x.Date:yyyy-MM-dd HH:mm:ss}\", {x.Value})"));
                        context.Database.ExecuteSqlCommand(builder.ToString());
                        Console.WriteLine(
                            $"Inserted {idCounter} records into database for {filename}");
                    }

                    context.SaveChanges();
                }
            }

            else
            {
                //insert using EFCore, slower as each record is inserted individually so requires a round trip
                using (JBADbContext context = new JBADbContext(filename))
                {
                    //batch adding to provide status updates
                    for (int i = 0; i < Math.Ceiling((double)records.Count / 1000); i++)
                    {
                        context.ChangeTracker.AutoDetectChangesEnabled = false;
                        List <PrecipitationRecord> recordsToInsert = records.Skip(1000 * i).Take(1000).ToList();
                        context.AddRange(recordsToInsert);
                        context.SaveChanges();
                        Console.WriteLine(
                            $"Inserted {i * 1000 + recordsToInsert.Count} records into database for {filename}");
                    }
                }
            }
        }