private void ProcessRecordsExcel(object x)
{
ConcurrentExcelReader reader = x as ConcurrentExcelReader;
SimpleSqlTableLoader writer = new SimpleSqlTableLoader(m_Context);
string[] line;
int rowsProcessed = 0;
int numColumns = m_Context.ColumnNames.Count();
while (reader.TryExtractRecord(out line))
{
var newRow = writer.GetEmptyRow();
for (int i = 0; i < numColumns; i++)
{
newRow[i] = line[i];
}
writer.PostRecord(newRow);
if (++rowsProcessed % numberOfLines == 0)
{
OnRecordsProcessed(Thread.CurrentThread.Name);
}
}
//flush final records and trigger last event
writer.WriteRecords();
OnRecordsProcessed(Thread.CurrentThread.Name, rowsProcessed % numberOfLines);
}
public void WriteDbNullValuesTest()
{
var cont = new PipelineContext()
{
FirstLineContainsHeaders = false,
SourceFilePath = @"..\..\..\D2S.LibraryTests\TestDataFiveColumns.txt",
Delimiter = "||",
TotalObjectsInSequentialPipe = 10,
DestinationTableName = "dbo.WriteDbNullValuesTest",
DbNullStringValue = "nullerino"
};
DestinationTableCreator table = new DestinationTableCreator(cont);
table.CreateTable();
SimpleSqlTableLoader loader = new SimpleSqlTableLoader(cont);
var row = loader.GetEmptyRow();
for (int j = 0; j < cont.ColumnNamesSelection.Count(); j++)
{
row[j] = "nullerino";
}
loader.PostRecord(row);
try
{
loader.WriteRecords();
}
finally
{
var val = ReadScalar(1, 1, "dbo.WriteDbNullValuesTest");
Assert.AreEqual(expected: DBNull.Value, actual: val);
DropTableAndReturnRows("dbo.WriteDbNullValuesTest");
}
}
public void WriteRecordNoHeaderTest()
{
var cont = new PipelineContext()
{
FirstLineContainsHeaders = false,
SourceFilePath = @"..\..\..\D2S.LibraryTests\TestDataFiveColumns.txt",
Delimiter = "||",
TotalObjectsInSequentialPipe = 10,
DestinationTableName = "dbo.SimpleSqlTableLoaderTest"
};
DestinationTableCreator table = new DestinationTableCreator(cont);
table.CreateTable();
SimpleSqlTableLoader loader = new SimpleSqlTableLoader(cont);
for (int i = 0; i < 5; i++)
{
var row = loader.GetEmptyRow();
for (int j = 0; j < cont.ColumnNamesSelection.Count(); j++)
{
row[j] = $"col{j}";
}
loader.PostRecord(row);
}
try
{
loader.WriteRecords();
}
finally
{
int rowCount = DropTableAndReturnRows(cont.DestinationTableName);
Assert.IsTrue(rowCount == 5);
}
}
public void PostRecordTest()
{
SimpleSqlTableLoader loader = new SimpleSqlTableLoader(context);
for (int i = 0; i < 10; i++)
{
var row = loader.GetEmptyRow();
foreach (var column in context.ColumnNames)
{
row[column] = $"Value{i}";
}
loader.PostRecord(row);
}
//if no errors we should be okay
}
public void WriteRecordsTest()
{
DestinationTableCreator destinationTableCreator = new DestinationTableCreator(context);
destinationTableCreator.CreateTable();
SimpleSqlTableLoader loader = new SimpleSqlTableLoader(context);
int numberOfRows = 100;
for (int i = 0; i < numberOfRows; i++)
{
var row = loader.GetEmptyRow();
foreach (var column in context.ColumnNames)
{
row[column] = $"Value{i}";
}
loader.PostRecord(row);
}
loader.WriteRecords();
int rowCount = DropTableAndReturnRows(context.DestinationTableName);
Assert.AreEqual(expected: numberOfRows, actual: rowCount);
}
private void ProcessRecords(object x)
{
ConcurrentFlatFileExtractor reader = x as ConcurrentFlatFileExtractor;
SimpleSqlTableLoader writer = new SimpleSqlTableLoader(m_Context);
string line;
int rowsProcessed = 0;
int numColumns = m_Context.ColumnNames.Count();
//if a selection is made on the source columns we will compute the ordinal rankings we require here
int[] ordinalRankings = null;
//if these are not equal a selection is made.
if (numColumns != m_Context.ColumnNamesSelection.Count())
{
ordinalRankings = new int[m_Context.ColumnNamesSelection.Count()];
int indexRankings = 0;
//for every name in the total list, check if it is present in the selection and if so write its ordinal ranking to the array.
//the rankings will be sorted low to high by design which also suits the simplesqlWriter in case it is in ordinal mode.
for (int i = 0; i < numColumns; i++)
{
if (m_Context.ColumnNamesSelection.Any(
selectedName => selectedName.Equals(m_Context.ColumnNames[i], StringComparison.InvariantCultureIgnoreCase)))
{
ordinalRankings[indexRankings++] = i;
}
}
}
while (reader.TryExtractLine(out line))
{
string[] record = StringAndText.SplitRow(line, m_Context.Delimiter, m_Context.Qualifier, true);
//assume the orindal rankings are identical (if all the pieces use the context.columnsnames property that will be the case
//check the column count tho
if (record.Count() != numColumns)
{
var errorMsg = $"A row was skipped over because it had too many or too few columns, expected: {numColumns}, actual: {record.Count()}";
if (m_Context.IsSkippingError)
{
LogService.Instance.Warn(errorMsg);
}
else
{
Exception ex = new Exception(errorMsg);
LogService.Instance.Error(ex);
throw ex;
}
}
else
{
var newRow = writer.GetEmptyRow();
//write all columns
if (ordinalRankings == null)
{
for (int i = 0; i < numColumns; i++)
{
newRow[i] = record[i];
}
}
//else write only selected columns (the indices we want are in the ordinalrankings array)
else
{
for (int i = 0; i < ordinalRankings.Count(); i++)
{
newRow[i] = record[ordinalRankings[i]];
}
}
writer.PostRecord(newRow);
if (++rowsProcessed % numberOfLines == 0)
{
OnRecordsProcessed(Thread.CurrentThread.Name);
}
}
}
//flush final records and trigger last event
writer.WriteRecords();
OnRecordsProcessed(Thread.CurrentThread.Name, rowsProcessed % numberOfLines);
}