private bool CheckRowOverflow(RowData rowData)
{
bool rowHandled = false;
if (rowData.ColumnCount > this.bufferService.ColumnCount)
{
string errorMessage = MessageStrings.RowOveflow(this.currentRowCount, rowData.ColumnCount, this.bufferService.ColumnCount);
if (this.bufferService.ErrorOutputUsed)
{
if (this.output.TruncationRowDisposition == DTSRowDisposition.RD_RedirectRow)
{
this.bufferService.AddErrorRow(errorMessage, string.Empty, rowData.RowText);
rowHandled = true;
}
else if (this.output.TruncationRowDisposition == DTSRowDisposition.RD_IgnoreFailure)
{
rowHandled = true;
}
}
if (!rowHandled)
{
throw new BufferSinkException(errorMessage);
}
}
// If the row is handled here it will be ignored in the main output.
return rowHandled;
}
public void AddRow(RowData rowData)
{
this.currentRowCount++;
if (!this.CheckRowOverflow(rowData))
{
this.bufferService.AddRow();
AddColumns(rowData);
}
}
internal static void VerifyParsedRow(RowData rowData, string expectedRowText, string[] expectedValues)
{
Assert.AreEqual<int>(expectedValues.Length, rowData.ColumnCount);
if (!string.IsNullOrEmpty(expectedRowText))
{
Assert.AreEqual<string>(expectedRowText, rowData.RowText);
}
for (int i = 0; i < rowData.ColumnCount && i < expectedValues.Length; i++)
{
Assert.AreEqual<string>(expectedValues[i], rowData.GetColumnData(i));
}
}
public void ParseNextRow(IFileReader reader, RowData rowData)
{
ArgumentVerifier.CheckObjectArgument(reader, "reader");
if (rowData != null)
{
rowData.ResetRowData();
}
this.fieldParser.ResetParsingState();
if (this.singleColumn)
{
fieldParser.ParseNext(reader, rowData);
if (rowData != null)
{
string columnData = fieldParser.CurrentText;
if (!reader.IsEOF || !string.IsNullOrEmpty(columnData))
{
rowData.AddColumnData(fieldParser.CurrentText);
}
}
}
else
{
while (!reader.IsEOF && !this.fieldParser.RowDelimiterMatch)
{
this.fieldParser.ParseNext(reader, rowData);
if (rowData != null)
{
string columnData = fieldParser.CurrentText;
if (!reader.IsEOF || rowData.ColumnCount > 0 || !string.IsNullOrEmpty(columnData))
{
if (MaxColumnNumber == rowData.ColumnCount)
{
throw new RowColumnNumberOverflow();
}
// Add data if this is not the last and empty row.
rowData.AddColumnData(fieldParser.CurrentText);
}
}
}
}
}
public void ParseNextRow(IFileReader reader, RowData rowData)
{
ArgumentVerifier.CheckObjectArgument(reader, "reader");
if (rowData != null)
{
rowData.ResetRowData();
}
this.fieldParser.ResetParsingState();
if (this.singleColumn)
{
fieldParser.ParseNext(reader, rowData);
if (rowData != null)
{
string columnData = fieldParser.CurrentText;
if (!reader.IsEOF || !string.IsNullOrEmpty(columnData))
{
rowData.AddColumnData(fieldParser.CurrentText);
}
}
}
else
{
while (!reader.IsEOF && !this.fieldParser.RowDelimiterMatch)
{
this.fieldParser.ParseNext(reader, rowData);
if (rowData != null)
{
string columnData = fieldParser.CurrentText;
if (!reader.IsEOF || rowData.ColumnCount > 0 || !string.IsNullOrEmpty(columnData))
{
if (MaxColumnNumber == rowData.ColumnCount)
{
throw new RowColumnNumberOverflow();
}
// Add data if this is not the last and empty row.
rowData.AddColumnData(fieldParser.CurrentText);
}
}
}
}
}
private void AddColumns(RowData rowData)
{
for (int i = 0; i < bufferService.ColumnCount; i++)
{
if (i < rowData.ColumnCount)
{
string columnData = rowData.GetColumnData(i);
if (this.treatEmptyStringAsNull && string.IsNullOrEmpty(columnData))
{
this.bufferService.SetNull(i);
}
else
{
try
{
this.bufferService.SetColumnData(i, columnData);
}
catch (Exception ex)
{
IDTSOutputColumn100 outputColumn = this.output.OutputColumnCollection[i];
if (ex is DoesNotFitBufferException ||
ex is OverflowException ||
ex is System.Data.SqlTypes.SqlTruncateException)
{
this.HandleColumnErrorDistribution(outputColumn.TruncationRowDisposition, outputColumn.IdentificationString, columnData, rowData.RowText, ex);
}
else
{
this.HandleColumnErrorDistribution(outputColumn.ErrorRowDisposition, outputColumn.IdentificationString, columnData, rowData.RowText, ex);
}
// If we get this far, it means the error row is redirected or ignored. Stop the loop.
break;
}
}
}
else
{
this.bufferService.SetNull(i);
}
}
}
private void AddColumns(RowData rowData)
{
for (int i = 0; i < bufferService.ColumnCount; i++)
{
if (i < rowData.ColumnCount)
{
string columnData = rowData.GetColumnData(i);
if (this.treatEmptyStringAsNull && string.IsNullOrEmpty(columnData))
{
this.bufferService.SetNull(i);
}
else
{
try
{
this.bufferService.SetColumnData(i, columnData);
}
catch (Exception ex)
{
IDTSOutputColumn100 outputColumn = this.output.OutputColumnCollection[i];
if (ex is DoesNotFitBufferException ||
ex is OverflowException ||
ex is System.Data.SqlTypes.SqlTruncateException)
{
this.HandleColumnErrorDistribution(outputColumn.TruncationRowDisposition, outputColumn.IdentificationString, columnData, rowData.RowText, ex);
}
else
{
this.HandleColumnErrorDistribution(outputColumn.ErrorRowDisposition, outputColumn.IdentificationString, columnData, rowData.RowText, ex);
}
// If we get this far, it means the error row is redirected or ignored. Stop the loop.
break;
}
}
}
else
{
this.bufferService.SetNull(i);
}
}
}
public void RowDataConstructorTest()
{
RowData target = new RowData();
}
//
//You can use the following additional attributes as you write your tests:
//
//Use ClassInitialize to run code before running the first test in the class
//[ClassInitialize()]
//public static void MyClassInitialize(TestContext testContext)
//{
//}
//
//Use ClassCleanup to run code after all tests in a class have run
//[ClassCleanup()]
//public static void MyClassCleanup()
//{
//}
//
//Use TestInitialize to run code before running each test
//[TestInitialize()]
//public void MyTestInitialize()
//{
//}
//
//Use TestCleanup to run code after each test has run
//[TestCleanup()]
//public void MyTestCleanup()
//{
//}
//
#endregion
internal static void VerifyParsedRow(RowData rowData, string[] expectedValues)
{
VerifyParsedRow(rowData, null, expectedValues);
}
public void ReadColumnNamesTest()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("A,B,C\r\n1,2,3");
DelimitedFileParser parser = new DelimitedFileParser(",", "\r\n");
parser.ColumnNameInFirstRow = true;
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "A", "B", "C" });
}
public void AddColumnDataTest()
{
RowData target = new RowData();
target.AddColumnData("ABC");
Assert.AreEqual<int>(1, target.ColumnCount);
target.AddColumnData("DEF");
Assert.AreEqual<int>(2, target.ColumnCount);
Assert.AreEqual<string>("ABC", target.GetColumnData(0));
Assert.AreEqual<string>("DEF", target.GetColumnData(1));
}
public void RowTextTest()
{
RowData target = new RowData();
Assert.AreEqual<string>(string.Empty, target.RowText);
}
public void ReadSingleColumnRows()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("1,2,3");
DelimitedFileParser parser = new DelimitedFileParser(string.Empty, ",");
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "1" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "2" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "3" });
}
public void ResetRowDataTest()
{
RowData target = new RowData();
target.AddColumnData("A");
target.AddColumnData("B");
Assert.AreEqual<int>(2, target.ColumnCount);
target.ResetRowData();
Assert.AreEqual<int>(0, target.ColumnCount);
}
public void ParseNextRowBadReaderTest()
{
RowData rowData = new RowData();
RowParser target = new RowParser(string.Empty, ",", string.Empty);
target.ParseNextRow(null, rowData);
}
public void ParseSingleQualifiedColumn()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("\"A,B\r\n\",C\r\n1,2,3");
RowParser rowParser = new RowParser(string.Empty, "\r\n", "\"");
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, "\"A,B\r\n\",C\r\n", new string[] { "A,B\r\n,C" });
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, "1,2,3", new string[] { "1,2,3" });
}
public void ReadQualifiedColumnNamesAfterHeaderTest()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("\r\n\r\nHeader text\r\n\"A\",B,\"C\"\r\n1,2,3");
DelimitedFileParser parser = new DelimitedFileParser(",", "\r\n");
parser.ColumnNameInFirstRow = true;
parser.HeaderRowDelimiter = "\r\n";
parser.HeaderRowsToSkip = 3;
parser.TextQualifier = "\"";
parser.SkipHeaderRows(reader);
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "A", "B", "C" });
}
public void ParseRowsWithUnevenFields()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("A,B,C,D\r\n1,2,3");
RowParser rowParser = new RowParser(",", "\r\n", "\"");
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "A", "B", "C", "D" });
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "1", "2", "3" });
}
public void ReadMultipleDataRowsAfterHeader()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("\r\n\r\nHeader text\r\n1,1,1,1\r\n2,2,2,2\r\n3,3,3,3\r\n4,4,4,4");
DelimitedFileParser parser = new DelimitedFileParser(",", "\r\n");
parser.HeaderRowDelimiter = "\r\n";
parser.HeaderRowsToSkip = 3;
parser.SkipHeaderRows(reader);
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "1", "1", "1", "1" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "2", "2", "2", "2" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "3", "3", "3", "3" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "4", "4", "4", "4" });
}
public void ReadSingleColumnRows3()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("\"ABC\"\r\n\"DE\"F\r\n\"G\"HI");
DelimitedFileParser parser = new DelimitedFileParser(string.Empty, "\r\n");
parser.TextQualifier = "\"";
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "ABC" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "DEF" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "GHI" });
}
public void ReadSingleColumnRows2()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("ABC\r\nDEF\r\nGHI");
DelimitedFileParser parser = new DelimitedFileParser(string.Empty, "\r\n");
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "ABC" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "DEF" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "GHI" });
}
public void ColumnCountTest()
{
RowData target = new RowData();
Assert.AreEqual<int>(0, target.ColumnCount);
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
PipelineBuffer mainBuffer = null;
PipelineBuffer errorBuffer = null;
IDTSOutput100 mainOutput = null;
int errorOutID = 0;
int errorOutIndex = 0;
// If there is an error output, figure out which output is the main
// and which is the error
if (outputs == 2)
{
GetErrorOutputInfo(ref errorOutID, ref errorOutIndex);
if (outputIDs[0] == errorOutID)
{
mainBuffer = buffers[1];
errorBuffer = buffers[0];
mainOutput = this.ComponentMetaData.OutputCollection[1];
}
else
{
mainBuffer = buffers[0];
errorBuffer = buffers[1];
mainOutput = this.ComponentMetaData.OutputCollection[0];
}
}
else
{
mainBuffer = buffers[0];
mainOutput = this.ComponentMetaData.OutputCollection[0];
}
bool firstRowColumnNames = (bool)this.GetComponentPropertyValue(PropertiesManager.ColumnNamesInFirstRowPropName);
bool treatNulls = (bool)this.GetComponentPropertyValue(PropertiesManager.TreatEmptyStringsAsNullPropName);
FileReader reader = new FileReader(this.fileName, this.GetEncoding());
DelimitedFileParser parser = this.CreateParser();
ComponentBufferService bufferService = new ComponentBufferService(mainBuffer, errorBuffer);
BufferSink bufferSink = new BufferSink(bufferService, mainOutput, treatNulls);
bufferSink.CurrentRowCount = parser.HeaderRowsToSkip + parser.DataRowsToSkip + (firstRowColumnNames ? 1 : 0);
try
{
parser.SkipInitialRows(reader);
RowData rowData = new RowData();
while (!reader.IsEOF)
{
parser.ParseNextRow(reader, rowData);
if (rowData.ColumnCount == 0)
{
// Last row with no data will be ignored.
break;
}
bufferSink.AddRow(rowData);
}
}
catch (ParsingBufferOverflowException ex)
{
this.PostErrorAndThrow(MessageStrings.ParsingBufferOverflow(bufferSink.CurrentRowCount + 1, ex.ColumnIndex + 1, FieldParser.ParsingBufferMaxSize));
}
catch (RowColumnNumberOverflow)
{
this.PostErrorAndThrow(MessageStrings.MaximumColumnNumberOverflow(bufferSink.CurrentRowCount + 1, RowParser.MaxColumnNumber));
}
finally
{
reader.Close();
}
foreach (PipelineBuffer buffer in buffers)
{
buffer.SetEndOfRowset();
}
}
public void ReadMultipleDataRowsAndBlankLastRow()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("1,1,1,1\r\n2,2,2,2\r\n3,3,3,3\r\n4,4,4,4\r\n");
DelimitedFileParser parser = new DelimitedFileParser(",", "\r\n");
parser.SkipInitialRows(reader);
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "1", "1", "1", "1" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "2", "2", "2", "2" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "3", "3", "3", "3" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "4", "4", "4", "4" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[0]);
}
public void ReadSingleDataRow()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("1,2,3");
DelimitedFileParser parser = new DelimitedFileParser(",", "\r\n");
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "1", "2", "3" });
}
public void ParseQualifiedRowsWithError()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("A,\"B\" \r\n1,2,3");
RowParser rowParser = new RowParser(",", "\r\n", "\"");
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "A", "B "});
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "1", "2", "3" });
}
public void TestSkippingRows()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("\r\n\r\nHeader\r\nA,B,C,D\r\n1,1,1,1\r\n2,2,2,2\r\n3,3,3,3\r\n4,4,4,4\r\n");
DelimitedFileParser parser = new DelimitedFileParser(",", "\r\n");
parser.HeaderRowDelimiter = "\r\n";
parser.HeaderRowsToSkip = 3;
parser.DataRowsToSkip = 3;
parser.ColumnNameInFirstRow = true;
parser.SkipInitialRows(reader);
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "4", "4", "4", "4" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[0]);
Assert.AreEqual<bool>(true, reader.IsEOF);
}
public void ParseRowWithTooManyColumns()
{
RowData rowData = new RowData();
// It will repeat the given string up to 8000 characters nad that
// will pass the maximum number of columns.
IFileReader reader = new FileReaderTestImpl("ABC,", 2000*4);
RowParser rowParser = new RowParser(",", "\r\n", string.Empty);
rowParser.ParseNextRow(reader, rowData);
}
public void TestSkippingTooManyHeaderRows()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("\r\n\r\nHeader\r\nA,B,C,D\r\n1,1,1,1\r\n2,2,2,2\r\n3,3,3,3\r\n4,4,4,4\r\n");
DelimitedFileParser parser = new DelimitedFileParser(",", "\r\n");
parser.HeaderRowDelimiter = "\r\n";
parser.HeaderRowsToSkip = 12;
parser.DataRowsToSkip = 15;
parser.ColumnNameInFirstRow = true;
parser.SkipInitialRows(reader);
Assert.AreEqual<bool>(true, reader.IsEOF);
}
public void ParseSimpleRows()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("A,B,C\r\n1,2,3");
RowParser rowParser = new RowParser(",", "\r\n", string.Empty);
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, "A,B,C\r\n", new string[] { "A", "B", "C" });
rowParser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, "1,2,3", new string[] { "1", "2", "3" });
}
public void ReadMultipleDataRowsAfterHeaderWithMessyParsing()
{
RowData rowData = new RowData();
IFileReader reader = FileReaderTest.GetReader("\r\n\r\nHeader text\r\n A)::-)brr\"-:-)\r\n-\r\nasdfghj\"-\"\"-\":-)\"-\":-)\"-\"\"-\"\"-\":--\r");
DelimitedFileParser parser = new DelimitedFileParser(":-)", "-\r\n");
parser.HeaderRowDelimiter = "\r\n";
parser.HeaderRowsToSkip = 3;
parser.TextQualifier = "\"-\"";
parser.SkipHeaderRows(reader);
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { " A):", "brr\"-", "\r\n" });
parser.ParseNextRow(reader, rowData);
RowDataTest.VerifyParsedRow(rowData, new string[] { "asdfghj\"-\"\"-\"", ":-)\"-\":-" });
}
public void ParseNextRow(IFileReader reader, RowData rowData)
{
if (this.rowParser == null)
{
InitializeDataRowParsing();
}
this.rowParser.ParseNextRow(reader, rowData);
}
public void GetColumnDataBadArgTest2()
{
RowData target = new RowData();
target.GetColumnData(-3);
}