예제 #1
0
        private int LoadState(TextReader textReader, int skipRows)
        {
            var rowCount         = 0;
            var updateErrorCount = 0;

            using (var csvReader = new CsvReader(textReader, true))
            {
                var headers = csvReader.GetFieldHeaders();

                // process skips
                if (skipRows > 0)
                {
                    AppendStatusText("Skipping {0} rows", skipRows);
                    while (skipRows > 0 && csvReader.ReadNextRecord())
                    {
                        skipRows--;
                        rowCount++;
                    }
                }

                if (ToCsv)
                {
                    var csvWriter = new SimpleCsvWriter();
                    while (csvReader.ReadNextRecord())
                    {
                        var error = false;
                        rowCount++;
                        try
                        {
                            foreach (var field in headers)
                            {
                                var value = csvReader[field];
                                // The incoming data has a few anomolies...
                                value = FixupField(field, value);
                                //if (field == "AddressPrimaryLowNumber" ||
                                //  field == "AddressPrimaryHighNumber")
                                //  value = FixNumericPrimaryAddress(value);
                                csvWriter.AddField(value);
                            }
                        }
                        catch (Exception ex)
                        {
                            error = true;
                            AppendStatusText("Row {0} skipped due to {1}", rowCount, ex.Message);
                        }
                        if (!error)
                        {
                            csvWriter.Write(_TextWriter);
                        }
                        else
                        {
                            csvWriter.Clear();
                        }
                        if (rowCount % 50000 == 0) // report every 50000 rows
                        {
                            AppendStatusText("{0} rows written", rowCount);
                        }
                    }
                }
                else
                {
                    var table = new DB.VoteZipNew.ZipStreetsDownloadedTable();
                    while (csvReader.ReadNextRecord())
                    {
                        var row   = table.NewRow();
                        var error = false;
                        rowCount++;
                        try
                        {
                            foreach (var field in headers)
                            {
                                var value = csvReader[field];
                                // The incoming data has a few anomolies...
                                value = FixupField(field, value);
                                //if (field == "AddressPrimaryLowNumber" ||
                                //  field == "AddressPrimaryHighNumber")
                                //  value = FixNumericPrimaryAddress(field, value);
                                row[field] = value;
                            }
                        }
                        catch (Exception ex)
                        {
                            error = true;
                            AppendStatusText("Row {0} skipped due to {1}", rowCount, ex.Message);
                        }
                        if (!error)
                        {
                            table.AddRow(row);
                        }
                        if (rowCount % 1000 == 0) // flush every 1000 rows
                        {
                            UpdateZipStreetsDownloaded(table, ref updateErrorCount);
                            table = new DB.VoteZipNew.ZipStreetsDownloadedTable();
                        }
                        if (rowCount % 50000 == 0) // report every 50000 rows
                        {
                            AppendStatusText("{0} rows written", rowCount);
                        }
                    }
                    UpdateZipStreetsDownloaded(table, ref updateErrorCount);
                }
            }
            return(rowCount);
        }