public void BadCSV_TooManyCellsInRow(BadDataHandlingStrategy strategy) { var file = CreateTestFile( "Name,Description,Age", "Frank,Is the greatest,100", "Bob,He's also dynamite, seen him do a lot of good work,30", "Dennis,Hes ok,35"); switch (strategy) { case BadDataHandlingStrategy.ThrowException: var ex = Assert.Throws <FlatFileLoadException>(() => RunGetChunk(file, strategy, true)); StringAssert.StartsWith("Bad data found on line 3", ex.Message); break; case BadDataHandlingStrategy.IgnoreRows: var dt = RunGetChunk(file, strategy, true); Assert.AreEqual(2, dt.Rows.Count); break; case BadDataHandlingStrategy.DivertRows: var dt2 = RunGetChunk(file, strategy, true); Assert.AreEqual(2, dt2.Rows.Count); AssertDivertFileIsExactly("Bob,He's also dynamite, seen him do a lot of good work,30" + Environment.NewLine); break; default: throw new ArgumentOutOfRangeException("strategy"); } }
protected DataTable RunGetChunk(FlatFileToLoad file, BadDataHandlingStrategy strategy, bool throwOnEmpty) { return(RunGetChunk(file, s => { s.BadDataHandlingStrategy = strategy; s.ThrowOnEmptyFiles = throwOnEmpty; })); }
public FlatFileEventHandlers(FlatFileToLoad fileToLoad, FlatFileToDataTablePusher dataPusher, bool throwOnEmptyFiles, BadDataHandlingStrategy strategy, IDataLoadEventListener listener, int maximumErrorsToReport, bool ignoreBadDataEvents) { _fileToLoad = fileToLoad; _dataPusher = dataPusher; _throwOnEmptyFiles = throwOnEmptyFiles; _strategy = strategy; _listener = listener; _maximumErrorsToReport = maximumErrorsToReport; _ignoreBadDataEvents = ignoreBadDataEvents; }
public void BadCSV_UnclosedQuote(BadDataHandlingStrategy strategy) { var file = CreateTestFile( "Name,Description,Age", "Frank,\"Is, the greatest\",100", //<---- how you should be doing it "Frank,Is the greatest,100", "Frank,\"Is the greatest,100", //<----- no closing quote! i.e. read the rest of the file! "Frank,Is the greatest,100", "Frank,Is the greatest,100", "Frank,Is the greatest,100", "Frank,Is the greatest,100"); Action <DelimitedFlatFileDataFlowSource> adjust = (a) => { a.BadDataHandlingStrategy = strategy; a.ThrowOnEmptyFiles = true; a.IgnoreQuotes = false; }; switch (strategy) { case BadDataHandlingStrategy.ThrowException: var ex = Assert.Throws <FlatFileLoadException>(() => RunGetChunk(file, adjust)); Assert.AreEqual("Bad data found on line 9", ex.Message); break; case BadDataHandlingStrategy.IgnoreRows: var dt = RunGetChunk(file, adjust); Assert.AreEqual(2, dt.Rows.Count); //reads first 2 rows and chucks the rest! break; case BadDataHandlingStrategy.DivertRows: //read 2 rows and rejected the rest var dt2 = RunGetChunk(file, adjust); Assert.AreEqual(2, dt2.Rows.Count); AssertDivertFileIsExactly($"Frank,\"Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}"); break; default: throw new ArgumentOutOfRangeException("strategy"); } }
public void BadCSV_TooFewCellsInRow(BadDataHandlingStrategy strategy, bool tryToResolve) { var file = CreateTestFile( "Name,Description,Age", "Frank,Is the greatest,100", "", "Other People To Investigate", "Dennis,Hes ok,35"); Action <DelimitedFlatFileDataFlowSource> adjust = (a) => { a.BadDataHandlingStrategy = strategy; a.AttemptToResolveNewLinesInRecords = tryToResolve; a.ThrowOnEmptyFiles = true; }; switch (strategy) { case BadDataHandlingStrategy.ThrowException: var ex = Assert.Throws <FlatFileLoadException>(() => RunGetChunk(file, adjust)); StringAssert.StartsWith("Bad data found on line 4", ex.Message); break; case BadDataHandlingStrategy.IgnoreRows: var dt = RunGetChunk(file, adjust); Assert.AreEqual(2, dt.Rows.Count); break; case BadDataHandlingStrategy.DivertRows: var dt2 = RunGetChunk(file, adjust); Assert.AreEqual(2, dt2.Rows.Count); AssertDivertFileIsExactly(Environment.NewLine + "Other People To Investigate" + Environment.NewLine); break; default: throw new ArgumentOutOfRangeException("strategy"); } }
public void NewLinesInConstantString_NotEscaped(BadDataHandlingStrategy strategy) { if (File.Exists(filename)) { File.Delete(filename); } StringBuilder sb = new StringBuilder(); sb.AppendLine("CHI,StudyID,Date"); sb.AppendLine(@"0101010101,5 The first,2001-01-05"); sb.AppendLine("0101010101,5,2001-01-05"); sb.AppendLine("0101010101,5,2001-01-05"); sb.AppendLine("0101010101,5,2001-01-05"); sb.AppendLine("0101010101,5,2001-01-05"); File.WriteAllText(filename, sb.ToString()); var testFile = new FileInfo(filename); DelimitedFlatFileDataFlowSource source = new DelimitedFlatFileDataFlowSource(); source.PreInitialize(new FlatFileToLoad(testFile), new ThrowImmediatelyDataLoadEventListener()); source.Separator = ","; source.MaxBatchSize = 10000; source.StronglyTypeInput = true;//makes the source interpret the file types properly source.BadDataHandlingStrategy = strategy; try { switch (strategy) { case BadDataHandlingStrategy.ThrowException: var ex = Assert.Throws <FlatFileLoadException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken())); StringAssert.Contains("line 2", ex.Message); break; case BadDataHandlingStrategy.IgnoreRows: var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); Assert.IsNotNull(dt); Assert.AreEqual(4, dt.Rows.Count); break; case BadDataHandlingStrategy.DivertRows: var dt2 = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); Assert.AreEqual(4, dt2.Rows.Count); Assert.IsNotNull(source.EventHandlers.DivertErrorsFile); Assert.AreEqual(@"0101010101,5 The first,2001-01-05 ", File.ReadAllText(source.EventHandlers.DivertErrorsFile.FullName)); break; default: throw new ArgumentOutOfRangeException("strategy"); } } finally { source.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); } }