public void TryExtractRecordTest()
{
try
{
CreateTestTable();
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context);
BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>();
Action action = () =>
{
object[] currObject = null;
while (reader.TryExtractRecord(out currObject))
{
output.TryAdd(currObject);
}
};
List <Task> tasks = new List <Task>();
for (int i = 0; i < 3; i++)
{
tasks.Add(Task.Factory.StartNew(action));
}
Task.WhenAll(tasks).Wait();
Assert.IsTrue(output.Count == 5);
}
finally
{
DestroyTestTable();
}
}
public void DbNullValueTest()
{
string OmegaLul = "HoHoHaHa";
try
{
CreateTestTable();
UpdateTestTable();
PipelineContext context = new PipelineContext()
{
SourceTableName = "dbo.ConcurrentSqlExtractorTest"
, DbNullStringValue = OmegaLul
};
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context);
BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>();
object[] row = null;
while (reader.TryExtractRecord(out row))
{
output.TryAdd(row);
}
Assert.IsTrue(output.Count == 5);
object[] val = null;
output.TryTake(out val);
Assert.AreEqual(expected: OmegaLul, actual: val[0]);
}
finally
{
DestroyTestTable();
}
}
public override async Task StartAsync()
{
//handle some pre-start conditions
OutputToConsoleAndLog("Pipeline is starting");
//create shared objects
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(m_Context);
ConcurrentFlatFileWriter writer = new ConcurrentFlatFileWriter(m_Context);
Tuple <ConcurrentSqlExtractor, ConcurrentFlatFileWriter, string> state = new Tuple <ConcurrentSqlExtractor, ConcurrentFlatFileWriter, string>(reader, writer, m_Context.Delimiter);
int NumTasks = m_Context.CpuCountUsedToComputeParallalism;
Task[] tasks = new Task[NumTasks];
OutputToConsoleAndLog($"Creating {NumTasks} tasks");
//create tasks, to future editors, ensure that the state object is the correct type or create a new processrecords method
for (int i = 0; i < NumTasks; i++)
{
tasks[i] = new Task(new Action <object>(ProcessRecords), state);
}
//write header line if needed
if (m_Context.FirstLineContainsHeaders)
{
WriteHeaderLine(reader, writer);
}
OutputToConsoleAndLog("Starting tasks");
//start tasks
for (int i = 0; i < NumTasks; i++)
{
tasks[i].Start();
}
OutputToConsoleAndLog("Awaiting tasks");
//await the completion of asynchronous tasks
await Task.WhenAll(tasks);
writer.Close();
OutputToConsoleAndLog($"{ NumTasks} have finished completing. Pipeline finished");
}
private void WriteHeaderLine(ConcurrentSqlExtractor reader, ConcurrentFlatFileWriter writer)
{
var HeaderLineObject = reader.GetDataTable();
StringBuilder builder = new StringBuilder();
int count = HeaderLineObject.Rows.Count;
string delim = m_Context.Delimiter;
for (int i = 0; i < count - 1; i++)
{
builder.Append(HeaderLineObject.Rows[i]["ColumnName"]);
builder.Append(delim);
}
builder.Append(HeaderLineObject.Rows[count - 1]["ColumnName"]);
writer.WriteLine(builder.ToString());
}
public void GetDataTableTest()
{
try
{
CreateTestTable();
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context);
var result = reader.GetDataTable();
Assert.AreEqual(expected: 3, actual: result.Rows.Count);
Assert.AreEqual(expected: "col1", actual: result.Rows[0]["ColumnName"]);
}
finally
{
DestroyTestTable();
}
}
public void SkipperinoCappucinoMachiatoTest()
{
try
{
CreateTestTable();
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context);
BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>();
object[] row = null;
Assert.IsTrue(reader.TrySkipRecord());
Assert.IsTrue(reader.TrySkipRecord());
while (reader.TryExtractRecord(out row))
{
output.TryAdd(row);
}
Assert.IsTrue(output.Count == 3);
}
finally
{
DestroyTestTable();
}
}
