public void DataLakeFlatFileExtractorTest()
{
//Code is made unreachable so unit test can run on a remote machine. If you wish to test the login prompt, simple remove the inconclusive assert and the return statement and run the test.
Assert.Inconclusive();
return;
DataLakeFlatFileExtractor dataLakeFlatFileExtractor = new DataLakeFlatFileExtractor(ConfigurationManager.AppSettings.Get("DatalakeAdress"));
var method = dataLakeFlatFileExtractor.GetPausableReportingWorkItem();
string snap = "20190227";
PipelineContext context = new PipelineContext()
{
SourceFilePath = @"qrm02-p-01\04_Sourcefiles_Archive\201902\FT004408\Sophis_Trigger_" + snap + ".csv"
, FirstLineContainsHeaders = true
, SourceFileIsSourcedFromDial = false
, PromptAzureLogin = true
};
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
Progress <int> prog = new Progress <int>();
method(context, output, null, prog);
Assert.IsTrue(output.Count == 1);
string result;
output.TryTake(out result);
Console.WriteLine(result);
Assert.IsTrue(result.Equals("Sophis|14298|-677451144.84"));
}
public void ReportingMmfExtractorTest()
{
MmfExtractor extractor = new MmfExtractor();
PipelineContext pipelineContext = new PipelineContext()
{
FirstLineContainsHeaders = false,
SourceFilePath = @"..\..\..\D2S.LibraryTests\ExtractorTests.txt"
};
ManualResetEvent pause = new ManualResetEvent(true);
BoundedConcurrentQueu <string[]> output = new BoundedConcurrentQueu <string[]>();
Progress <int> prog = new Progress <int>();
var action = extractor.GetPausableReportingWorkItem();
action(pipelineContext, output, pause, prog);
Assert.IsTrue(output.Count == 3);
List <string> ResultList = new List <string>();
for (int i = 0; i < 3; i++)
{
string[] s;
output.TryTake(out s);
ResultList.Add(s[0]);
}
Console.WriteLine(ResultList[0]);
Console.WriteLine("This is a line");
Assert.IsTrue(ResultList[0].Normalize().Equals("This is a line", StringComparison.InvariantCulture));
}
public void DbNullValueTest()
{
string OmegaLul = "HoHoHaHa";
try
{
CreateTestTable();
UpdateTestTable();
PipelineContext context = new PipelineContext()
{
SourceTableName = "dbo.ConcurrentSqlExtractorTest"
, DbNullStringValue = OmegaLul
};
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context);
BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>();
object[] row = null;
while (reader.TryExtractRecord(out row))
{
output.TryAdd(row);
}
Assert.IsTrue(output.Count == 5);
object[] val = null;
output.TryTake(out val);
Assert.AreEqual(expected: OmegaLul, actual: val[0]);
}
finally
{
DestroyTestTable();
}
}
public void TryExtractRecordTest()
{
try
{
CreateTestTable();
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context);
BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>();
Action action = () =>
{
object[] currObject = null;
while (reader.TryExtractRecord(out currObject))
{
output.TryAdd(currObject);
}
};
List <Task> tasks = new List <Task>();
for (int i = 0; i < 3; i++)
{
tasks.Add(Task.Factory.StartNew(action));
}
Task.WhenAll(tasks).Wait();
Assert.IsTrue(output.Count == 5);
}
finally
{
DestroyTestTable();
}
}
public void RowDuplicatorTests()
{
BoundedConcurrentQueu <Row> input = new BoundedConcurrentQueu <Row>();
BoundedConcurrentQueu <Row> firstOut = new BoundedConcurrentQueu <Row>();
BoundedConcurrentQueu <Row> secondOut = new BoundedConcurrentQueu <Row>();
BoundedConcurrentQueu <Row>[] outputArray = new BoundedConcurrentQueu <Row>[] { firstOut, secondOut };
ManualResetEvent pause = new ManualResetEvent(true);
Progress <int> progress = new Progress <int>();
RowDuplicater rowDuplicater = new RowDuplicater();
var action = rowDuplicater.GetReportingPausableWorkItem();
input.TryAdd(new Row());
Task t = Task.Factory.StartNew(() => action(input, outputArray, pause, progress));
Task.Delay(10).Wait();
rowDuplicater.SignalCompletion();
Assert.IsTrue(input.Count == 0);
foreach (var queue in outputArray)
{
Assert.IsTrue(queue.Count == 1);
}
}
public void TryExtractLineConcurrentlyDialTest()
{
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
ConcurrentFlatFileExtractor reader = new ConcurrentFlatFileExtractor(dialContext);
Task[] tasklist = new Task[10];
for (int i = 0; i < 10; i++)
{
object[] ob = new object[] { reader, output };
tasklist[i] = new Task(obj
=>
{
object[] objarray = obj as object[];
ConcurrentFlatFileExtractor r = objarray[0] as ConcurrentFlatFileExtractor;
BoundedConcurrentQueu <string> o = objarray[1] as BoundedConcurrentQueu <string>;
string l;
while (r.TryExtractLine(out l))
{
o.TryAdd(l);
Thread.Sleep(100);
}
}, ob);
}
for (int i = 0; i < 10; i++)
{
tasklist[i].Start();
}
Task.WaitAll(tasklist);
Assert.IsTrue(output.Count == 3);
}
public void BoundedConcurrentQueuTest()
{
int boundedCapacity = 10;
BoundedConcurrentQueu <int> bc = new BoundedConcurrentQueu <int>(boundedCapacity);
//ensure basic adding and taking works within the boundery
var t = Parallel.For(0, 5, inte => bc.TryAdd(1)); // add 5 integers
while (!t.IsCompleted)
{
Task.Delay(10).Wait();
}
if (bc.Count != 5)
{
Assert.Fail("Failed to insert all 5 integers");
}
int result;
for (int i = 0; i < 5; i++)
{
if (bc.TryTake(out result))
{
continue;
}
else
{
Assert.Fail("failed to retrieve all 5 integers");
}
}
//fill up the collection to the max capacity
for (int i = 0; i < 10; i++)
{
bc.TryAdd(1);
}
//try to add one more item and check that the call to tryadd is successfully blocked
Task attemptToExceedCapacity = Task.Factory.StartNew(
() => bc.TryAdd(1));
Task.Delay(50).Wait();
//ensure task is started and is executing (i.e. waiting for the semaphore to be released)
if (attemptToExceedCapacity.IsCompleted || attemptToExceedCapacity.Status != TaskStatus.Running)
{
Assert.Fail("Test managed to exceed the capacitiy of the collection or did not start");
}
//dequeu one item and check if the task succeeded
bc.TryTake(out result);
Task.Delay(50).Wait();
if (!attemptToExceedCapacity.IsCompleted)
{
Assert.Fail("could not add new item after releasing the semaphore");
}
Assert.IsTrue(bc.Count == 10);
}
public void TryExtractLineDialTest()
{
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
ConcurrentFlatFileExtractor reader = new ConcurrentFlatFileExtractor(dialContext);
string line;
while (reader.TryExtractLine(out line))
{
output.TryAdd(line);
}
Assert.IsTrue(output.Count == 3);
}
public void TryExtractLineDataLake()
{
//Code is made unreachable so unit test can run on a remote machine. If you wish to test the login prompt, simple remove the inconclusive assert and the return statement and run the test.
Assert.Inconclusive();
return;
PipelineContext context = new PipelineContext()
{
SourceFilePath = @"qrm02-p-01\04_Sourcefiles_Archive\201902\FT004408\Sophis_Trigger_20190227.csv"
,
FirstLineContainsHeaders = true
,
SourceFileIsSourcedFromDial = false
,
PromptAzureLogin = true
,
IsReadingFromDataLake = true
,
DataLakeAdress = ConfigurationManager.AppSettings.Get("DatalakeAdress")
};
ConcurrentFlatFileExtractor reader = new ConcurrentFlatFileExtractor(context);
string snap = "20190227";
BoundedConcurrentQueu <string> output = new BoundedConcurrentQueu <string>();
string line;
while (reader.TryExtractLine(out line))
{
output.TryAdd(line);
}
Assert.IsTrue(output.Count == 1);
string result;
output.TryTake(out result);
Console.WriteLine(result);
Assert.IsTrue(result.Equals("Sophis|14298|-677451144.84"));
}
public void SkipperinoCappucinoMachiatoTest()
{
try
{
CreateTestTable();
ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context);
BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>();
object[] row = null;
Assert.IsTrue(reader.TrySkipRecord());
Assert.IsTrue(reader.TrySkipRecord());
while (reader.TryExtractRecord(out row))
{
output.TryAdd(row);
}
Assert.IsTrue(output.Count == 3);
}
finally
{
DestroyTestTable();
}
}
public void PausingMmfExtractorTest()
{
MmfExtractor extractor = new MmfExtractor();
PipelineContext pipelineContext = new PipelineContext()
{
FirstLineContainsHeaders = false,
SourceFilePath = @"..\..\..\D2S.LibraryTests\ExtractorTests.txt",
Delimiter = " "
};
ManualResetEvent pause = new ManualResetEvent(true);
BoundedConcurrentQueu <string[]> output = new BoundedConcurrentQueu <string[]>();
//fun fact: this extractor gives zero f's about the amount of columns per row so were gonna try with a space delimiter.
var action = extractor.GetPausableWorkItem();
action(pipelineContext, output, pause);
Assert.IsTrue(output.Count == 3);
List <string[]> ResultList = new List <string[]>();
for (int i = 0; i < 3; i++)
{
string[] s;
output.TryTake(out s);
ResultList.Add(s);
}
if (ResultList[0][0].Equals("This", StringComparison.InvariantCulture))
{
Assert.IsTrue(true);
}
else
{
Assert.Fail();
}
}
