public void DataLakeFlatFileExtractorTest()
{
//Code is made unreachable so unit test can run on a remote machine. If you wish to test the login prompt, simple remove the inconclusive assert and the return statement and run the test.
Assert.Inconclusive();
return;
DataLakeFlatFileExtractor dataLakeFlatFileExtractor = new DataLakeFlatFileExtractor(ConfigurationManager.AppSettings.Get("DatalakeAdress"));
var method = dataLakeFlatFileExtractor.GetPausableReportingWorkItem();
string snap = "20190227";
PipelineContext context = new PipelineContext()
{
SourceFilePath = @"qrm02-p-01\04_Sourcefiles_Archive\201902\FT004408\Sophis_Trigger_" + snap + ".csv"
, FirstLineContainsHeaders = true
, SourceFileIsSourcedFromDial = false
, PromptAzureLogin = true
};
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
Progress <int> prog = new Progress <int>();
method(context, output, null, prog);
Assert.IsTrue(output.Count == 1);
string result;
output.TryTake(out result);
Console.WriteLine(result);
Assert.IsTrue(result.Equals("Sophis|14298|-677451144.84"));
}
internal BasicSequentialPipeline(PipelineContext context)
{
m_Context = context;
//decide which reader to use
if (context.IsReadingFromDataLake)
{
m_Reader = new DataLakeFlatFileExtractor(context.DataLakeAdress);
}
else if (context.SourceFilePath.Contains(".xls"))
{
m_ExcelReader = new ExcelDataExtractor();
}
else
{
m_Reader = new DIALFlatFileExtractor();
}
m_StringSplitter = new StringSplitter(context.Qualifier)
{
Delimiter = context.Delimiter
};
m_RowBuilder = new RecordToRowTransformer(context.ColumnNames, context.IsSkippingError);
m_Loader = new SQLTableLoader();
m_LineBuffers = new List <BoundedConcurrentQueu <string> > {
new BoundedConcurrentQueu <string>(context.TotalObjectsInSequentialPipe / NumberOfBuffers)
};
m_RecordBuffers = new List <BoundedConcurrentQueu <object[]> > {
new BoundedConcurrentQueu <object[]>(context.TotalObjectsInSequentialPipe / NumberOfBuffers)
};
m_RowBuffers = new List <BoundedConcurrentQueu <Row> > {
new BoundedConcurrentQueu <Row>(context.TotalObjectsInSequentialPipe / NumberOfBuffers)
};
m_Pause = new ManualResetEvent(true);
m_LatestPauseState = false;
m_DummyProgress = new Progress <int>();
m_ActualProgress = new Progress <int>();
m_PauseSyncRoot = new object();
//register event
m_ActualProgress.ProgressChanged += OnReaderEvent;
}
