internal ScalingSequentialPipeline(PipelineContext context)
{
m_Pause = new ManualResetEvent(true);
m_LatestPauseState = false;
m_DummyProgress = new Progress <int>();
m_ActualProgress = new Progress <int>();
m_PauseSyncRoot = new object();
m_Context = context;
m_LineBuffers = new List <BoundedConcurrentQueu <string> >();
m_RecordBuffers = new List <BoundedConcurrentQueu <object[]> >();
m_RowBuffers = new List <BoundedConcurrentQueu <Row> >();
if (context.SourceFilePath.Contains(".xls"))
{
m_ExcelReader = new ExcelDataExtractor();
}
else
{
m_Reader = new DIALFlatFileExtractor();
m_StringSplitter = new StringSplitter(context.Qualifier)
{
Delimiter = context.Delimiter
};
}
m_RowBuilder = new RecordToRowTransformer(context.ColumnNames, context.IsSkippingError);
m_Loader = new SQLTableLoader();
//register event
m_ActualProgress.ProgressChanged += OnReaderEvent;
}
public void RecordToRowTransformerTest()
{
string[] columnNames = new string[] { "First Name", "Surname", "Age" };
RecordToRowTransformer rowFactory = new RecordToRowTransformer(columnNames);
object[] data = new object[3]
{
new object[] { "John", "Doe", 35 },
new object[] { "Jane", "Doe", 36 },
new object[] { "Frenk", "Tank, de", 19 }
};
ConcurrentBag <object[]> input = new ConcurrentBag <object[]>();
for (int i = 0; i < 3; i++)
{
input.Add((object[])data[i]);
}
ConcurrentBag <Row> output = new ConcurrentBag <Row>();
ManualResetEvent pause = new ManualResetEvent(true);
Progress <int> progress = new Progress <int>();
progress.ProgressChanged += progresshandler;
var action = rowFactory.GetReportingPausableWorkItem();
Task work = Task.Factory.StartNew(() => action(input, output, pause, progress));
while (!input.IsEmpty)
{
Task.Delay(100).Wait();
}
rowFactory.SignalCompletion();
work.Wait();
int sumTotalAge = 0;
foreach (Row r in output)
{
sumTotalAge += (int)r["Age"].Item1;
}
Assert.AreEqual(expected: 35 + 36 + 19, actual: sumTotalAge);
//check errorskipping
rowFactory = new RecordToRowTransformer(columnNames, true);
data = new object[3]
{
new object[] { "John", "Doe", 35, "" }, //this row has too many columns!
new object[] { "Jane", "Doe", 36 },
new object[] { "Frenk", "Tank, de", 19 }
};
for (int i = 0; i < 3; i++)
{
input.Add((object[])data[i]);
}
output = new ConcurrentBag <Row>();
action = rowFactory.GetReportingPausableWorkItem();
work = Task.Factory.StartNew(() => action(input, output, pause, progress));
while (!input.IsEmpty)
{
Task.Delay(100).Wait();
}
rowFactory.SignalCompletion();
work.Wait();
Assert.IsTrue(output.Count == 2);
}
