public void BadCSV_TooManyCellsInRow(BadDataHandlingStrategy strategy)
{
var file = CreateTestFile(
"Name,Description,Age",
"Frank,Is the greatest,100",
"Bob,He's also dynamite, seen him do a lot of good work,30",
"Dennis,Hes ok,35");
switch (strategy)
{
case BadDataHandlingStrategy.ThrowException:
var ex = Assert.Throws <FlatFileLoadException>(() => RunGetChunk(file, strategy, true));
StringAssert.StartsWith("Bad data found on line 3", ex.Message);
break;
case BadDataHandlingStrategy.IgnoreRows:
var dt = RunGetChunk(file, strategy, true);
Assert.AreEqual(2, dt.Rows.Count);
break;
case BadDataHandlingStrategy.DivertRows:
var dt2 = RunGetChunk(file, strategy, true);
Assert.AreEqual(2, dt2.Rows.Count);
AssertDivertFileIsExactly("Bob,He's also dynamite, seen him do a lot of good work,30" + Environment.NewLine);
break;
default:
throw new ArgumentOutOfRangeException("strategy");
}
}
protected DataTable RunGetChunk(FlatFileToLoad file, BadDataHandlingStrategy strategy, bool throwOnEmpty)
{
return(RunGetChunk(file, s =>
{
s.BadDataHandlingStrategy = strategy;
s.ThrowOnEmptyFiles = throwOnEmpty;
}));
}
public FlatFileEventHandlers(FlatFileToLoad fileToLoad, FlatFileToDataTablePusher dataPusher, bool throwOnEmptyFiles, BadDataHandlingStrategy strategy, IDataLoadEventListener listener, int maximumErrorsToReport, bool ignoreBadDataEvents)
{
_fileToLoad = fileToLoad;
_dataPusher = dataPusher;
_throwOnEmptyFiles = throwOnEmptyFiles;
_strategy = strategy;
_listener = listener;
_maximumErrorsToReport = maximumErrorsToReport;
_ignoreBadDataEvents = ignoreBadDataEvents;
}
public void BadCSV_UnclosedQuote(BadDataHandlingStrategy strategy)
{
var file = CreateTestFile(
"Name,Description,Age",
"Frank,\"Is, the greatest\",100", //<---- how you should be doing it
"Frank,Is the greatest,100",
"Frank,\"Is the greatest,100", //<----- no closing quote! i.e. read the rest of the file!
"Frank,Is the greatest,100",
"Frank,Is the greatest,100",
"Frank,Is the greatest,100",
"Frank,Is the greatest,100");
Action <DelimitedFlatFileDataFlowSource> adjust = (a) =>
{
a.BadDataHandlingStrategy = strategy;
a.ThrowOnEmptyFiles = true;
a.IgnoreQuotes = false;
};
switch (strategy)
{
case BadDataHandlingStrategy.ThrowException:
var ex = Assert.Throws <FlatFileLoadException>(() => RunGetChunk(file, adjust));
Assert.AreEqual("Bad data found on line 9", ex.Message);
break;
case BadDataHandlingStrategy.IgnoreRows:
var dt = RunGetChunk(file, adjust);
Assert.AreEqual(2, dt.Rows.Count); //reads first 2 rows and chucks the rest!
break;
case BadDataHandlingStrategy.DivertRows:
//read 2 rows and rejected the rest
var dt2 = RunGetChunk(file, adjust);
Assert.AreEqual(2, dt2.Rows.Count);
AssertDivertFileIsExactly($"Frank,\"Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}Frank,Is the greatest,100{Environment.NewLine}");
break;
default:
throw new ArgumentOutOfRangeException("strategy");
}
}
public void BadCSV_TooFewCellsInRow(BadDataHandlingStrategy strategy, bool tryToResolve)
{
var file = CreateTestFile(
"Name,Description,Age",
"Frank,Is the greatest,100",
"",
"Other People To Investigate",
"Dennis,Hes ok,35");
Action <DelimitedFlatFileDataFlowSource> adjust = (a) =>
{
a.BadDataHandlingStrategy = strategy;
a.AttemptToResolveNewLinesInRecords = tryToResolve;
a.ThrowOnEmptyFiles = true;
};
switch (strategy)
{
case BadDataHandlingStrategy.ThrowException:
var ex = Assert.Throws <FlatFileLoadException>(() => RunGetChunk(file, adjust));
StringAssert.StartsWith("Bad data found on line 4", ex.Message);
break;
case BadDataHandlingStrategy.IgnoreRows:
var dt = RunGetChunk(file, adjust);
Assert.AreEqual(2, dt.Rows.Count);
break;
case BadDataHandlingStrategy.DivertRows:
var dt2 = RunGetChunk(file, adjust);
Assert.AreEqual(2, dt2.Rows.Count);
AssertDivertFileIsExactly(Environment.NewLine + "Other People To Investigate" + Environment.NewLine);
break;
default:
throw new ArgumentOutOfRangeException("strategy");
}
}
public void NewLinesInConstantString_NotEscaped(BadDataHandlingStrategy strategy)
{
if (File.Exists(filename))
{
File.Delete(filename);
}
StringBuilder sb = new StringBuilder();
sb.AppendLine("CHI,StudyID,Date");
sb.AppendLine(@"0101010101,5
The first,2001-01-05");
sb.AppendLine("0101010101,5,2001-01-05");
sb.AppendLine("0101010101,5,2001-01-05");
sb.AppendLine("0101010101,5,2001-01-05");
sb.AppendLine("0101010101,5,2001-01-05");
File.WriteAllText(filename, sb.ToString());
var testFile = new FileInfo(filename);
DelimitedFlatFileDataFlowSource source = new DelimitedFlatFileDataFlowSource();
source.PreInitialize(new FlatFileToLoad(testFile), new ThrowImmediatelyDataLoadEventListener());
source.Separator = ",";
source.MaxBatchSize = 10000;
source.StronglyTypeInput = true;//makes the source interpret the file types properly
source.BadDataHandlingStrategy = strategy;
try
{
switch (strategy)
{
case BadDataHandlingStrategy.ThrowException:
var ex = Assert.Throws <FlatFileLoadException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()));
StringAssert.Contains("line 2", ex.Message);
break;
case BadDataHandlingStrategy.IgnoreRows:
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
Assert.IsNotNull(dt);
Assert.AreEqual(4, dt.Rows.Count);
break;
case BadDataHandlingStrategy.DivertRows:
var dt2 = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
Assert.AreEqual(4, dt2.Rows.Count);
Assert.IsNotNull(source.EventHandlers.DivertErrorsFile);
Assert.AreEqual(@"0101010101,5
The first,2001-01-05
", File.ReadAllText(source.EventHandlers.DivertErrorsFile.FullName));
break;
default:
throw new ArgumentOutOfRangeException("strategy");
}
}
finally
{
source.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
}
}
