public override async Task StartAsync()
{
//create, truncate, drop tables if specified
HandeTableOptions();
List <Task> Tasklist = new List <Task>();
if (Regex.IsMatch(m_Context.SourceFilePath, @"\.xls"))
{
ConcurrentExcelReader reader = new ConcurrentExcelReader(m_Context);
for (int i = 0; i < m_Context.CpuCountUsedToComputeParallalism; i++)
{
Tasklist.Add(
Task.Factory.StartNew(
x => ProcessRecordsExcel(x), reader));
}
}
else
{
ConcurrentFlatFileExtractor reader = new ConcurrentFlatFileExtractor(m_Context);
for (int i = 0; i < m_Context.CpuCountUsedToComputeParallalism; i++)
{
Tasklist.Add(
Task.Factory.StartNew(
x => ProcessRecords(x), reader));
}
}
await Task.WhenAll(Tasklist);
}
public void TryExtractLineConcurrentlyDialTest()
{
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
ConcurrentFlatFileExtractor reader = new ConcurrentFlatFileExtractor(dialContext);
Task[] tasklist = new Task[10];
for (int i = 0; i < 10; i++)
{
object[] ob = new object[] { reader, output };
tasklist[i] = new Task(obj
=>
{
object[] objarray = obj as object[];
ConcurrentFlatFileExtractor r = objarray[0] as ConcurrentFlatFileExtractor;
BoundedConcurrentQueu <string> o = objarray[1] as BoundedConcurrentQueu <string>;
string l;
while (r.TryExtractLine(out l))
{
o.TryAdd(l);
Thread.Sleep(100);
}
}, ob);
}
for (int i = 0; i < 10; i++)
{
tasklist[i].Start();
}
Task.WaitAll(tasklist);
Assert.IsTrue(output.Count == 3);
}
public void TryExtractLineDialTest()
{
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
ConcurrentFlatFileExtractor reader = new ConcurrentFlatFileExtractor(dialContext);
string line;
while (reader.TryExtractLine(out line))
{
output.TryAdd(line);
}
Assert.IsTrue(output.Count == 3);
}
public void TryExtractLineDataLake()
{
//Code is made unreachable so unit test can run on a remote machine. If you wish to test the login prompt, simple remove the inconclusive assert and the return statement and run the test.
Assert.Inconclusive();
return;
PipelineContext context = new PipelineContext()
{
SourceFilePath = @"qrm02-p-01\04_Sourcefiles_Archive\201902\FT004408\Sophis_Trigger_20190227.csv"
,
FirstLineContainsHeaders = true
,
SourceFileIsSourcedFromDial = false
,
PromptAzureLogin = true
,
IsReadingFromDataLake = true
,
DataLakeAdress = ConfigurationManager.AppSettings.Get("DatalakeAdress")
};
ConcurrentFlatFileExtractor reader = new ConcurrentFlatFileExtractor(context);
string snap = "20190227";
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
string line;
while (reader.TryExtractLine(out line))
{
output.TryAdd(line);
}
Assert.IsTrue(output.Count == 1);
string result;
output.TryTake(out result);
Console.WriteLine(result);
Assert.IsTrue(result.Equals("Sophis|14298|-677451144.84"));
}
private void ProcessRecords(object x)
{
ConcurrentFlatFileExtractor reader = x as ConcurrentFlatFileExtractor;
SimpleSqlTableLoader writer = new SimpleSqlTableLoader(m_Context);
string line;
int rowsProcessed = 0;
int numColumns = m_Context.ColumnNames.Count();
//if a selection is made on the source columns we will compute the ordinal rankings we require here
int[] ordinalRankings = null;
//if these are not equal a selection is made.
if (numColumns != m_Context.ColumnNamesSelection.Count())
{
ordinalRankings = new int[m_Context.ColumnNamesSelection.Count()];
int indexRankings = 0;
//for every name in the total list, check if it is present in the selection and if so write its ordinal ranking to the array.
//the rankings will be sorted low to high by design which also suits the simplesqlWriter in case it is in ordinal mode.
for (int i = 0; i < numColumns; i++)
{
if (m_Context.ColumnNamesSelection.Any(
selectedName => selectedName.Equals(m_Context.ColumnNames[i], StringComparison.InvariantCultureIgnoreCase)))
{
ordinalRankings[indexRankings++] = i;
}
}
}
while (reader.TryExtractLine(out line))
{
string[] record = StringAndText.SplitRow(line, m_Context.Delimiter, m_Context.Qualifier, true);
//assume the orindal rankings are identical (if all the pieces use the context.columnsnames property that will be the case
//check the column count tho
if (record.Count() != numColumns)
{
var errorMsg = $"A row was skipped over because it had too many or too few columns, expected: {numColumns}, actual: {record.Count()}";
if (m_Context.IsSkippingError)
{
LogService.Instance.Warn(errorMsg);
}
else
{
Exception ex = new Exception(errorMsg);
LogService.Instance.Error(ex);
throw ex;
}
}
else
{
var newRow = writer.GetEmptyRow();
//write all columns
if (ordinalRankings == null)
{
for (int i = 0; i < numColumns; i++)
{
newRow[i] = record[i];
}
}
//else write only selected columns (the indices we want are in the ordinalrankings array)
else
{
for (int i = 0; i < ordinalRankings.Count(); i++)
{
newRow[i] = record[ordinalRankings[i]];
}
}
writer.PostRecord(newRow);
if (++rowsProcessed % numberOfLines == 0)
{
OnRecordsProcessed(Thread.CurrentThread.Name);
}
}
}
//flush final records and trigger last event
writer.WriteRecords();
OnRecordsProcessed(Thread.CurrentThread.Name, rowsProcessed % numberOfLines);
}
