public override async Task StartAsync()
{
//handle some pre-start conditions
OutputToConsoleAndLog("Pipeline is starting");
//create shared objects
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(m_Context);
ConcurrentFlatFileWriter writer = new ConcurrentFlatFileWriter(m_Context);
Tuple <ConcurrentSqlExtractor, ConcurrentFlatFileWriter, string> state = new Tuple <ConcurrentSqlExtractor, ConcurrentFlatFileWriter, string>(reader, writer, m_Context.Delimiter);
int NumTasks = m_Context.CpuCountUsedToComputeParallalism;
Task[] tasks = new Task[NumTasks];
OutputToConsoleAndLog($"Creating {NumTasks} tasks");
//create tasks, to future editors, ensure that the state object is the correct type or create a new processrecords method
for (int i = 0; i < NumTasks; i++)
{
tasks[i] = new Task(new Action <object>(ProcessRecords), state);
}
//write header line if needed
if (m_Context.FirstLineContainsHeaders)
{
WriteHeaderLine(reader, writer);
}
OutputToConsoleAndLog("Starting tasks");
//start tasks
for (int i = 0; i < NumTasks; i++)
{
tasks[i].Start();
}
OutputToConsoleAndLog("Awaiting tasks");
//await the completion of asynchronous tasks
await Task.WhenAll(tasks);
writer.Close();
OutputToConsoleAndLog($"{ NumTasks} have finished completing. Pipeline finished");
}
public void WriteLineTest()
{
try
{
int rowcountPerWriter = 2000;
int writersThreads = 3;
ConcurrentFlatFileWriter writer = new ConcurrentFlatFileWriter(context);
Task[] tasklist = new Task[writersThreads];
for (int i = 0; i < writersThreads; i++)
{
tasklist[i] = Task.Factory.StartNew(
() =>
{
for (int x = 0; x < rowcountPerWriter; x++)
{
writer.WriteLine($"Thread with ID: {Thread.CurrentThread.ManagedThreadId} is writing its {x} line: {TestLine}");
}
});
}
Task.WaitAll(tasklist);
writer.Close();
var result = File.ReadAllLines(context.DestinationFilePath);
Assert.AreEqual(expected: writersThreads * rowcountPerWriter, actual: result.Count());
}
finally
{
File.Delete(context.DestinationFilePath);
}
}
private void WriteHeaderLine(ConcurrentSqlExtractor reader, ConcurrentFlatFileWriter writer)
{
var HeaderLineObject = reader.GetDataTable();
StringBuilder builder = new StringBuilder();
int count = HeaderLineObject.Rows.Count;
string delim = m_Context.Delimiter;
for (int i = 0; i < count - 1; i++)
{
builder.Append(HeaderLineObject.Rows[i]["ColumnName"]);
builder.Append(delim);
}
builder.Append(HeaderLineObject.Rows[count - 1]["ColumnName"]);
writer.WriteLine(builder.ToString());
}
public void CloseTest()
{
try
{
ConcurrentFlatFileWriter writer = new ConcurrentFlatFileWriter(context);
//just testing the closing here when the number of lines written to the buffer is less than the buffersize, not testing concurrent writing here
int rowcount = 10;
for (int i = 0; i < rowcount; i++)
{
writer.WriteLine($"{i}{TestLine}");
}
writer.Close();
var result = File.ReadAllLines(context.DestinationFilePath);
Assert.AreEqual(expected: rowcount, actual: result.Count());
}
finally
{
File.Delete(context.DestinationFilePath);
}
}
