public void NoErrorHandling()
        {
            //Arrange
            MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();

            source.DataAsList = new List <MySimpleRow>()
            {
                new MySimpleRow()
                {
                    Col1 = "X"
                },
                new MySimpleRow()
                {
                    Col1 = "1"
                },
                new MySimpleRow()
                {
                    Col1 = null
                }
            };
            CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFileNoError.csv");

            //Act
            //Assert
            Assert.ThrowsAny <Exception>(() =>
            {
                source.LinkTo(dest);
                source.Execute();
                dest.Wait();
            });
        }
Example #2
0
        public void TestCsvDestination_Tabs()
        {
            var outDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);

            var opts = new IsIdentifiableRelationalDatabaseOptions
            {
                // This is slash t, not an tab
                DestinationCsvSeparator = "\\t",
                DestinationNoWhitespace = true,
                DestinationCsvFolder    = outDir.FullName
            };

            var dest = new CsvDestination(opts, "test", false);

            var report = new TestFailureReport(dest);

            report.WriteToDestinations();
            report.CloseReport();

            string fileCreatedContents = File.ReadAllText(Path.Combine(outDir.FullName, "test.csv"));

            fileCreatedContents = fileCreatedContents.Replace("\r\n", Environment.NewLine);

            TestHelpers.AreEqualIgnoringLineEndings(@"col1	col2
cell1 with some new lines and tabs	cell2
", fileCreatedContents);
        }
Example #3
0
        public static async Task <HttpResponseMessage> Run(
            [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
            ILogger log)
        {
            log.LogInformation("Received a json file to transform.");

            string filename = Guid.NewGuid().ToString() + ".csv";

            log.LogInformation("Temp file used to hold data: " + filename);

            var jsonSource = new JsonSource()
            {
                CreateStreamReader = _ => new StreamReader(req.Body)
            };
            var csvFileDest = new CsvDestination(filename);

            jsonSource.LinkTo(csvFileDest);
            await Network.ExecuteAsync(jsonSource);

            log.LogInformation("Successfully stored data in file - now returning the content as response.");

            return(new HttpResponseMessage()
            {
                StatusCode = HttpStatusCode.OK,
                Content = new StreamContent(new FileStream(filename, FileMode.Open))
            });
        }
Example #4
0
        public void IgnoreWithObject()
        {
            //Arrange
            MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();

            source.DataAsList = new List <MySimpleRow>()
            {
                null,
                new MySimpleRow()
                {
                    Col1 = 1, Col2 = "Test1"
                },
                null,
                new MySimpleRow()
                {
                    Col1 = 2, Col2 = "Test2"
                },
                new MySimpleRow()
                {
                    Col1 = 3, Col2 = "Test3"
                },
                null
            };

            //Act
            CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./IgnoreNullValues.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(File.ReadAllText("./IgnoreNullValues.csv"),
                         File.ReadAllText("res/CsvDestination/TwoColumns.csv"));
        }
Example #5
0
        public void SerializingDateTime()
        {
            int rowCount = 0;
            //Arrange
            CustomSource <MySeriRow> source = new CustomSource <MySeriRow>(
                () =>
                new MySeriRow()
            {
                Col1 = 1,
                Col2 = new DateTime(2010, 02, 05)
            },
                () => rowCount++ == 1);


            //Act
            CsvDestination <MySeriRow> dest = new CsvDestination <MySeriRow>("./DateTimeSerialization.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(File.ReadAllText("./DateTimeSerialization.csv"),
                         File.ReadAllText("res/CsvDestination/DateTimeSerialization.csv"));
        }
Example #6
0
        public void IgnoreWithStringArray()
        {
            //Arrange
            MemorySource <string[]> source = new MemorySource <string[]>();

            source.DataAsList = new List <string[]>()
            {
                null,
                new string[] { "1", "Test1" },
                null,
                new string[] { "2", "Test2" },
                new string[] { "3", "Test3" },
                null
            };

            //Act
            CsvDestination <string[]> dest = new CsvDestination <string[]>("./IgnoreNullValuesStringArray.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.csv"),
                         File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv"));
        }
Example #7
0
        static void Main(string[] args)
        {
            var source = new CsvSource <InputData>("Accounts_Quartal1.csv");

            source.Configuration.Delimiter = ";";

            var trans = new RowMultiplication <InputData, PivotedOutput>();

            trans.MultiplicationFunc = row =>
            {
                List <PivotedOutput> result = new List <PivotedOutput>();
                result.Add(new PivotedOutput()
                {
                    Account      = row.Account,
                    Month        = nameof(InputData.January),
                    MonthlyValue = row.January
                });
                result.Add(new PivotedOutput()
                {
                    Account      = row.Account,
                    Month        = nameof(InputData.February),
                    MonthlyValue = row.February
                });
                result.Add(new PivotedOutput()
                {
                    Account      = row.Account,
                    Month        = nameof(InputData.March),
                    MonthlyValue = row.March
                });
                return(result);
            };
            var dest = new CsvDestination <PivotedOutput>("AccountNumbers_Pivoted.csv");

            dest.Configuration.HasHeaderRecord = false;

            source.LinkTo(trans);
            trans.LinkTo(dest);

            Network.Execute(source);

            /* AccountNumbers_Pivoted.csv output:
             *  4711,January,10
             *  4711,February,11
             *  4711,March,12
             *  4712,January,20
             *  4712,February,21
             *  4712,March,22
             *  4713,January,30
             *  4713,February,31
             *  4713,March,32
             */
        }
        public void RedirectSingleRecordWithObject()
        {
            //Arrange
            MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();

            source.DataAsList = new List <MySimpleRow>()
            {
                new MySimpleRow()
                {
                    Col1 = "X"
                },
                new MySimpleRow()
                {
                    Col1 = "1"
                },
                new MySimpleRow()
                {
                    Col1 = "2"
                },
                new MySimpleRow()
                {
                    Col1 = null
                },
                new MySimpleRow()
                {
                    Col1 = "3"
                },
            };
            CsvDestination <MySimpleRow>    dest      = new CsvDestination <MySimpleRow>("ErrorFile.csv");
            MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();

            //Act
            source.LinkTo(dest);
            dest.LinkErrorTo(errorDest);
            source.Execute();
            dest.Wait();
            errorDest.Wait();

            //Assert
            Assert.Equal(File.ReadAllText("./ErrorFile.csv"),
                         File.ReadAllText("res/CsvDestination/TwoColumnsErrorLinking.csv"));
            Assert.Collection <ETLBoxError>(errorDest.Data,
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
                                            );
        }
Example #9
0
        public static void Main(string[] args)
        {
            PrepareSqlLiteDestination();

            var currentYear = StartYear;

            var source = new CustomBatchSource <Accident>();

            source.ReadBatchFunc = _ => {
                var accidents = ParseAccidentsFromUrl($"https://aviation-safety.net/database/dblist.php?Year={currentYear}");
                currentYear++;
                return(accidents);
            };
            source.ReadingCompleted = _ => currentYear > EndYear;

            var filter = new FilterTransformation <Accident>();

            filter.FilterPredicate = accident => accident.Year <= 1;

            var multicast = new Multicast <Accident>();

            var memDest = new MemoryDestination <Accident>();

            var sqlLiteDest = new DbDestination <Accident>(SQLiteConnection, "Accidents");

            var aggregation = new Aggregation <Accident, AccidentsPerYear>();
            var csvDest     = new CsvDestination <AccidentsPerYear>("aggregated.csv");

            source.LinkTo(filter);
            filter.LinkTo(multicast);
            multicast.LinkTo(memDest);
            multicast.LinkTo(sqlLiteDest);

            multicast.LinkTo(aggregation, row => row.Year > 1);
            aggregation.LinkTo(csvDest);

            Network.Execute(source);

            Console.WriteLine($"Imported {memDest.Data.Count} rows from aviation-safety.net");
            for (int year = StartYear; year <= EndYear; year++)
            {
                Console.WriteLine($"There were {memDest.Data.Where(a => a.Year == year).Count()} accidents in {year}");
            }
        }
        public void SimpleFlow()
        {
            //Arrange
            TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestDynamicObject");

            s2C.InsertTestDataSet3();
            DbSource <ExpandoObject> source = new DbSource <ExpandoObject>(SqlConnection, "CSVDestDynamicObject");

            //Act
            CsvDestination <ExpandoObject> dest = new CsvDestination <ExpandoObject>("./SimpleWithDynamicObject.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(File.ReadAllText("./SimpleWithDynamicObject.csv"),
                         File.ReadAllText("res/CsvDestination/TwoColumnsSet3DynamicObject.csv"));
        }
Example #11
0
        public void SimpleFlowWithObject()
        {
            //Arrange
            TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestSimple");

            s2C.InsertTestDataSet3();
            DbSource <MySimpleRow> source = new DbSource <MySimpleRow>("CSVDestSimple", SqlConnection);

            //Act
            CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./SimpleWithObject.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(File.ReadAllText("./SimpleWithObject.csv"),
                         File.ReadAllText("res/CsvDestination/TwoColumnsSet3.csv"));
        }
        public void SimpleNonGeneric()
        {
            //Arrange
            TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestSimpleNonGeneric");

            s2C.InsertTestDataSet3();
            DbSource <string[]> source = new DbSource <string[]>(SqlConnection, "CSVDestSimpleNonGeneric");

            //Act
            CsvDestination <string[]> dest = new CsvDestination <string[]>("./SimpleNonGeneric.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            //Assert
            Assert.Equal(File.ReadAllText("./SimpleNonGeneric.csv"),
                         File.ReadAllText("res/CsvDestination/TwoColumnsSet3NoHeader.csv"));
        }
Example #13
0
        public void Run()
        {
            PostgresConnectionManager conMan = new PostgresConnectionManager(PostgresConnectionString);
            //Import CSV
            CsvSource     sourceCSV  = new CsvSource("NameList.csv");
            DbDestination importDest = new DbDestination(conMan, "NameTable");

            sourceCSV.LinkTo(importDest);
            sourceCSV.Execute();
            importDest.Wait();

            //Export again
            DbSource <NameListElement>       sourceTable = new DbSource <NameListElement>(conMan, "NameTable");
            CsvDestination <NameListElement> destCSV     = new CsvDestination <NameListElement>("Export.csv");

            destCSV.Configuration.Delimiter = ";";
            sourceTable.LinkTo(destCSV);
            sourceTable.Execute();
            destCSV.Wait();
        }
Example #14
0
        public void DisableHeader()
        {
            //Arrange
            TwoColumnsTableFixture s2c = new TwoColumnsTableFixture("CsvSourceNoHeader");

            s2c.InsertTestData();
            DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "CsvSourceNoHeader");

            //Act
            CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./ConfigurationNoHeader.csv");

            dest.Configuration.HasHeaderRecord = false;
            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(File.ReadAllText("./ConfigurationNoHeader.csv"),
                         File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv"));
        }
Example #15
0
        /// <summary>
        /// Creates report destinations. Can be overriden to add headers or to initialize the destination in some way.
        /// </summary>
        /// <param name="opts"></param>
        public virtual void AddDestinations(IsIdentifiableAbstractOptions opts)
        {
            IReportDestination destination;

            // Default is to write out CSV results
            if (!string.IsNullOrWhiteSpace(opts.DestinationCsvFolder))
            {
                destination = new CsvDestination(opts, _reportName);
            }
            else if (!string.IsNullOrWhiteSpace(opts.DestinationConnectionString))
            {
                destination = new DatabaseDestination(opts, _reportName);
            }
            else
            {
                opts.DestinationCsvFolder = Environment.CurrentDirectory;
                destination = new CsvDestination(opts, _reportName);
            }

            Destinations.Add(destination);
        }
            public void should_write_the_data_to_a_file_using_the_specified_field_separator()
            {
                var result = new StringBuilder();
                Func <string, TextWriter> writerCreator = s => new StringWriter(result);
                var destination = new CsvDestination(";", writerCreator);

                var data = new[] {
                    new TestData {
                        Bar = 1, Foo = "Hello"
                    },
                    new TestData {
                        Bar = 2, Foo = "World"
                    },
                };

                destination.Load(data);

                Check.That(result.ToString())
                .Contains("Hello;1")
                .And.Contains("World;2");
            }
Example #17
0
        public void UsingGetInputRecordKeyFunc()
        {
            var source = new CsvSource <InputRow>("InputData.csv");

            source.Configuration.MissingFieldFound = null;

            var lookupSource = new MemorySource <LookupRow>();

            lookupSource.DataAsList = new List <LookupRow>()
            {
                new LookupRow()
                {
                    LookupId = "idstringa", LookupValue = "A"
                },
                new LookupRow()
                {
                    LookupId = "idstringb", LookupValue = "B"
                },
                new LookupRow()
                {
                    LookupId = "idstringc", LookupValue = "C"
                }
            };

            var lookup = new LookupTransformation <InputRow, LookupRow>();

            lookup.Source = lookupSource;
            lookup.GetInputRecordKeyFunc  = row => row.Id.ToLower();
            lookup.GetSourceRecordKeyFunc = row => row.LookupId;
            var dest = new CsvDestination <InputRow>("output1.csv");

            source.LinkTo(lookup).LinkTo(dest);

            Network.Execute(source);

            PrintFile("InputData.csv");
            PrintFile("output1.csv");
        }
Example #18
0
        public void WriteIntoMultipleDestinations()
        {
            //Arrange
            var source = new MemorySource <string[]>();

            source.DataAsList.Add(new string[] { "Test" });
            var trans = new RowTransformation <string[]>();

            trans.TransformationFunc = r => throw new Exception();
            var dest = new MemoryDestination <string[]>();

            CreateErrorTableTask.Create(SqlConnection, "error_log");
            var mc       = new Multicast <ETLBoxError>();
            var errorMem = new MemoryDestination <ETLBoxError>();
            var errorDb  = new DbDestination <ETLBoxError>(SqlConnection, "error_log");
            var errorCsv = new CsvDestination <ETLBoxError>("error_csv.csv");

            source.LinkTo(trans);
            trans.LinkTo(dest);

            //Act
            trans.LinkErrorTo(mc);
            mc.LinkTo(errorMem);
            mc.LinkTo(errorDb);
            mc.LinkTo(errorCsv);

            source.Execute();
            dest.Wait();
            errorMem.Wait();
            errorDb.Wait();
            errorCsv.Wait();

            //Assert
            Assert.True(errorMem.Data.Count > 0);
            Assert.True(RowCountTask.Count(SqlConnection, "error_log") > 0);
            Assert.True(File.ReadAllText("error_csv.csv").Length > 0);
        }
Example #19
0
        public void OverwritingComparisonInObject()
        {
            var source = new CsvSource <MyInputRow>("InputData.csv");

            source.Configuration.MissingFieldFound = null;

            var lookupSource = new MemorySource <MyLookupRow>();

            lookupSource.DataAsList = new List <MyLookupRow>()
            {
                new MyLookupRow()
                {
                    LookupId = new ComparableObject("idstringa"), LookupValue = "A"
                },
                new MyLookupRow()
                {
                    LookupId = new ComparableObject("idstringb"), LookupValue = "B"
                },
                new MyLookupRow()
                {
                    LookupId = new ComparableObject("idstringc"), LookupValue = "C"
                }
            };

            var lookup = new LookupTransformation <MyInputRow, MyLookupRow>();

            lookup.Source = lookupSource;

            var dest = new CsvDestination <MyInputRow>("output2.csv");

            source.LinkTo(lookup).LinkTo(dest);

            Network.Execute(source);

            PrintFile("InputData.csv");
            PrintFile("output1.csv");
        }
Example #20
0
        public void WriteAsyncAndCheckLock(string filename, int noRecords)
        {
            //Arrange
            if (File.Exists(filename))
            {
                File.Delete(filename);
            }
            MemorySource <string[]> source = new MemorySource <string[]>();

            for (int i = 0; i < noRecords; i++)
            {
                source.DataAsList.Add(new string[] { HashHelper.RandomString(100) });
            }
            CsvDestination <string[]> dest = new CsvDestination <string[]>(filename);
            bool onCompletionRun           = false;

            //Act
            source.LinkTo(dest);
            Task sT = source.ExecuteAsync();
            Task dt = dest.Completion;

            while (!File.Exists(filename))
            {
                Task.Delay(10).Wait();
            }

            //Assert
            dest.OnCompletion = () =>
            {
                Assert.False(IsFileLocked(filename), "StreamWriter should be disposed and file unlocked");
                onCompletionRun = true;
            };

            Assert.True(IsFileLocked(filename), "Right after  start the file should still be locked.");
            dt.Wait();
            Assert.True(onCompletionRun, "OnCompletion action and assertion did run");
        }
Example #21
0
        public void ErrorIntoCsv()
        {
            var source = new MemorySource <MyXmlRow>();

            source.DataAsList.Add(new MyXmlRow()
            {
                Xml = _validXml
            });
            source.DataAsList.Add(new MyXmlRow()
            {
                Xml = _invalidXml
            });
            source.DataAsList.Add(new MyXmlRow()
            {
                Xml = _validXml
            });

            XmlSchemaValidation <MyXmlRow> valid   = new XmlSchemaValidation <MyXmlRow>(xsdMarkup, i => i.Xml);
            CsvDestination <ETLBoxError>   errDest = new CsvDestination <ETLBoxError>("res/XmlValidation/Error.csv");
            CsvDestination <MyXmlRow>      dest    = new CsvDestination <MyXmlRow>("res/XmlValidation/NoError.csv");

            source.LinkTo(valid);
            valid.LinkTo(dest);
            valid.LinkErrorTo(errDest);
            source.Execute();

            dest.Wait();
            errDest.Wait();

            Assert.Equal(File.ReadAllText("res/XmlValidation/ValidXml.csv"),
                         File.ReadAllText("res/XmlValidation/NoError.csv"), ignoreCase: true, ignoreLineEndingDifferences: true, ignoreWhiteSpaceDifferences: true);

            Assert.StartsWith(@"ErrorText,ReportTime,ExceptionType,RecordAsJson
The element 'Root' has invalid child element 'Child3'. List of possible elements expected: 'Child2'.",
                              File.ReadAllText("res/XmlValidation/Error.csv"));
        }
Example #22
0
        /// <summary>
        /// Connects to the database and runs all failures through the rules base performing redactions as required
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            //In RulesOnly mode this will be null
            var server = _target?.Discover();
            List <Exception> errors = new List <Exception>();

            var storeReport = new FailureStoreReport(_outputFile.Name, 100);

            Stopwatch sw = new Stopwatch();

            sw.Start();

            using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile))
            {
                IsIdentifiableRule updateRule;

                storeReport.AddDestination(storeReportDestination);

                while (_reportReader.Next())
                {
                    bool noUpdate;

                    try
                    {
                        noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule);
                    }
                    catch (Exception e)
                    {
                        errors.Add(e);
                        continue;
                    }

                    //is it novel for updater
                    if (noUpdate)
                    {
                        //is it novel for ignorer
                        if (_ignorer.OnLoad(_reportReader.Current, out IsIdentifiableRule ignoreRule))
                        {
                            //we can't process it unattended
                            storeReport.Add(_reportReader.Current);
                            Unresolved++;
                        }
                        else
                        {
                            if (!_ignoreRulesUsed.ContainsKey(ignoreRule))
                            {
                                _ignoreRulesUsed.Add(ignoreRule, 1);
                            }
                            else
                            {
                                _ignoreRulesUsed[ignoreRule]++;
                            }

                            Ignores++;
                        }
                    }
                    else
                    {
                        if (!_updateRulesUsed.ContainsKey(updateRule))
                        {
                            _updateRulesUsed.Add(updateRule, 1);
                        }
                        else
                        {
                            _updateRulesUsed[updateRule]++;
                        }

                        Updates++;
                    }

                    Total++;

                    if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000)
                    {
                        Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}", true);
                        sw.Restart();
                    }
                }

                storeReport.CloseReport();
            }

            Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
                                                                          _ignoreRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);

            Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
                                                                          _updateRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);

            Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => e.ToString())), false);

            Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}", true);
            return(0);
        }
Example #23
0
        static void Main(string[] args)
        {
            var scraper = new Scraper();
            var terms   = scraper.GetTerms().ToList();

            Console.Clear();
            Console.WriteLine("Select a term to Scrape:");
            int t    = ChooseOptions(terms.Select(x => x.Name).ToArray());
            var term = terms[t - 1];

            Console.Clear();
            Console.WriteLine("Will scrape " + term.Name);
            Console.WriteLine("Select a data destination:");
            int d = ChooseOptions(
                "Scrape to Blank SQL Database",
                "Scrape to JSON file",
                "Scrape to XML file",
                "Scrape to CSV");

            ISectionDestination dest;
            string path = null;

            if (d > 1)
            {
                Console.Write("Enter Output File Path: ");
                path = Console.ReadLine();
            }

            switch (d)
            {
            case 1:
                dest = new DatabaseContext();
                break;

            case 2:
                dest = new JsonDestination(path);
                break;

            case 3:
                dest = new XmlDestination(path);
                break;

            case 4:
                dest = new CsvDestination(path);
                break;

            default:
                throw new Exception("Invalid Option");
            }

            Console.Clear();

            var watch = Stopwatch.StartNew();

            double prog = 0;

            var    schools       = scraper.GetSchools(term).ToList();
            double progPerSchool = 1.0 / schools.Count;

            foreach (var school in schools)
            {
                var    subjects       = scraper.GetSubjects(school).ToList();
                double progPerSubject = progPerSchool / subjects.Count;
                foreach (var subject in subjects)
                {
                    var    crns       = scraper.GetCrns(subject).ToList();
                    double progPerCrn = progPerSubject / crns.Count;
                    foreach (var crn in crns)
                    {
                        var cd = scraper.GetDetailedClass(crn);

                        var cs = new ClassSection(cd)
                        {
                            School  = school.Name,
                            Term    = term.Name,
                            Subject = subject.Name
                        };

                        dest.Add(cs);
                        prog += progPerCrn;

                        Console.Write("{0:P2} ({1:F2}s)\r", prog, watch.ElapsedMilliseconds / 1000.0);
                    }
                    dest.CommitChanges();
                }
            }
            Console.WriteLine("\nClosing...");
            dest.Close();
            watch.Stop();
            Console.WriteLine("Finished ({0}ms)", watch.ElapsedMilliseconds);
        }