public void testTNMBigIntegrationTestBad()
{
if (DebugSettings.RUN_HUGE_BAD_TNM_TESTS)
{
List <String> _SCHEMA_FILES = new List <String>();
IntegrationUtils.WritelineToLog("Starting TNMBigIntegrationTest::testBigIntegrationTestBad...");
TNMStagingCSharp.Src.Staging.Staging staging = TNMStagingCSharp.Src.Staging.Staging.getInstance(TnmDataProvider.getInstance(TnmVersion.LATEST));
// hard-code data directory based on Windows vs Linux
String dataDirectory;
if (Environment.OSVersion.Platform != PlatformID.Unix)
{
dataDirectory = "\\\\omni\\btp\\csb\\Staging\\TNM_ErrorCases\\";
}
else
{
dataDirectory = "/prj/csb/Staging/TNM";
}
long totalFailures = 0;
String sFilePath = dataDirectory + "schema_selection\\tnm_schema_identification.txt.gz";
totalFailures = PerformTNMSchemaSelection(staging, sFilePath);
totalFailures = PerformTNMStaging(staging, dataDirectory, _SCHEMA_FILES);
}
}
public void testVersionInitializationTypes()
{
TNMStagingCSharp.Src.Staging.Staging staging10 = TNMStagingCSharp.Src.Staging.Staging.getInstance(EodDataProvider.getInstance(EodVersion.LATEST));
Assert.AreEqual(EodVersion.LATEST.getVersion(), staging10.getVersion());
TNMStagingCSharp.Src.Staging.Staging stagingLatest = TNMStagingCSharp.Src.Staging.Staging.getInstance(EodDataProvider.getInstance());
Assert.AreEqual(EodVersion.LATEST.getVersion(), stagingLatest.getVersion());
}
public long PerformTNMStaging(TNMStagingCSharp.Src.Staging.Staging staging, String dataDirectory, List <String> _SCHEMA_FILES)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
long totalFiles = 0;
long totalCases = 0;
long totalFailures = 0;
// get the complete list of files
string[] arrfiles = Directory.GetFiles(dataDirectory, "*.*");
if (arrfiles.Length > 0)
{
// sort the files by name
Array.Sort(arrfiles);
foreach (String f in arrfiles)
{
String sFilename = Path.GetFileName(f);
if (File.Exists(f) && sFilename.EndsWith(".gz"))
{
if (_SCHEMA_FILES.Count == 0 || _SCHEMA_FILES.Contains(sFilename))
{
totalFiles += 1;
IntegrationUtils.WritelineToLog("Staging File: " + f);
FileStream fstream = File.Open(f, FileMode.Open, FileAccess.Read, FileShare.Read);
GZipStream decompressionStream = new GZipStream(fstream, CompressionMode.Decompress);
IntegrationUtils.IntegrationResult result =
TnmIntegrationSchemaStage.processTNMSchema(staging, sFilename, decompressionStream, false);
totalCases += result.getNumCases();
totalFailures += result.getNumFailures();
}
}
}
}
stopwatch.Stop();
String perMs = String.Format("{0,12:F4}", ((float)stopwatch.Elapsed.TotalMilliseconds / totalCases)).Trim();
IntegrationUtils.WritelineToLog("");
IntegrationUtils.WritelineToLog("Completed " + totalCases + " cases (" + totalFiles + " files) in " + IntegrationUtils.GenerateTotalTimeString(stopwatch) + " (" + perMs + " ms/case).");
if (totalFailures > 0)
{
IntegrationUtils.WritelineToLog("There were " + totalFailures + " failing cases.");
}
return(totalFailures);
}
public long PerformTNMSchemaSelection(TNMStagingCSharp.Src.Staging.Staging staging, String sFilePath)
{
FileStream fstream = File.Open(sFilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
GZipStream decompressionStream = new GZipStream(fstream, CompressionMode.Decompress);
IntegrationUtils.IntegrationResult result =
TnmIntegrationSchemaSelection.processSchemaSelection(
staging, "tnm_schema_identification.txt.gz", decompressionStream, null);
IntegrationUtils.WritelineToLog("-----------------------------------------------");
return(result.getNumFailures());
}
public static void ClassInit(TestContext context)
{
String basedir = System.IO.Directory.GetCurrentDirectory() + "\\..\\..\\..\\";
if (System.IO.Directory.GetCurrentDirectory().IndexOf("x64") >= 0)
{
basedir += "\\..\\";
}
String sFilePath = basedir + "Resources\\Test\\Misc\\external_algorithm.zip";
FileStream SourceStream = File.Open(sFilePath, FileMode.Open);
ExternalStagingFileDataProvider provider = new ExternalStagingFileDataProvider(SourceStream);
_STAGING = TNMStagingCSharp.Src.Staging.Staging.getInstance(provider);
}
public void testNumericRangeTableMatch()
{
InMemoryDataProvider provider = new InMemoryDataProvider("test", "1.0");
StagingTable table = new StagingTable();
table.setId("psa");
StagingColumnDefinition def1 = new StagingColumnDefinition();
def1.setKey("psa");
def1.setName("PSA Value");
def1.setType(ColumnType.INPUT);
StagingColumnDefinition def2 = new StagingColumnDefinition();
def2.setKey("description");
def2.setName("PSA Description");
def2.setType(ColumnType.DESCRIPTION);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1, def2
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"0.1", "0.1 or less nanograms/milliliter (ng/ml)"
});
table.getRawRows().Add(new List <String>()
{
"0.2-999.9", "0.2 – 999.9 ng/ml"
});
provider.addTable(table);
TNMStagingCSharp.Src.Staging.Staging staging = TNMStagingCSharp.Src.Staging.Staging.getInstance(provider);
Assert.AreEqual(0, staging.findMatchingTableRow("psa", "psa", "0.1"));
Assert.AreEqual(1, staging.findMatchingTableRow("psa", "psa", "0.2"));
Assert.AreEqual(1, staging.findMatchingTableRow("psa", "psa", "500"));
Assert.AreEqual(1, staging.findMatchingTableRow("psa", "psa", "500.99"));
Assert.AreEqual(1, staging.findMatchingTableRow("psa", "psa", "500.0001"));
Assert.AreEqual(1, staging.findMatchingTableRow("psa", "psa", "999.9"));
Assert.AreEqual(-1, staging.findMatchingTableRow("psa", "psa", "1000"));
Assert.AreEqual(-1, staging.findMatchingTableRow("psa", "psa", "-1"));
Assert.AreEqual(-1, staging.findMatchingTableRow("psa", "psa", "0.01"));
}
public void testTNMBigIntegrationTestGood()
{
if (DebugSettings.RUN_HUGE_GOOD_TNM_TESTS)
{
List <String> _SCHEMA_FILES = new List <String>();
IntegrationUtils.WritelineToLog("Starting TNMBigIntegrationTest::testBigIntegrationTestGood...");
TNMStagingCSharp.Src.Staging.Staging staging = null;
if (DebugSettings.USE_LOCAL_ZIP_FILE_FOR_TNM_TESTS)
{
String filename = "TNM_15.zip";
FileStream SourceStream = File.Open(filename, FileMode.Open);
ExternalStagingFileDataProvider provider = new ExternalStagingFileDataProvider(SourceStream);
staging = TNMStagingCSharp.Src.Staging.Staging.getInstance(provider);
}
else
{
staging = TNMStagingCSharp.Src.Staging.Staging.getInstance(TnmDataProvider.getInstance(TnmVersion.LATEST));
}
// hard-code data directory based on Windows vs Linux
String dataDirectory;
if (Environment.OSVersion.Platform != PlatformID.Unix)
{
dataDirectory = "\\\\omni\\btp\\csb\\Staging\\TNM\\";
}
else
{
dataDirectory = "/prj/csb/Staging/TNM";
}
long totalFailures = 0;
String sFilePath = dataDirectory + "schema_selection\\tnm_schema_identification.txt.gz";
totalFailures = PerformTNMSchemaSelection(staging, sFilePath);
totalFailures = PerformTNMStaging(staging, dataDirectory, _SCHEMA_FILES);
}
}
public void testExpectedOutput()
{
if (DebugSettings.RUN_LARGE_TNM_TESTS)
{
String basedir = System.IO.Directory.GetCurrentDirectory() + "\\..\\..\\..\\";
if (System.IO.Directory.GetCurrentDirectory().IndexOf("x64") >= 0)
{
basedir += "\\..\\";
}
String sFilePath = basedir + "Resources\\Test\\TNM\\TNM_13.zip";
FileStream SourceStream = File.Open(sFilePath, FileMode.Open);
ExternalStagingFileDataProvider provider = new ExternalStagingFileDataProvider(SourceStream);
TNMStagingCSharp.Src.Staging.Staging TNM13_STAGING = TNMStagingCSharp.Src.Staging.Staging.getInstance(provider);
sFilePath = basedir + "Resources\\Test\\TNM\\TNM_V13_StagingTestLarge.txt.gz";
FileStream fstream = File.Open(sFilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
GZipStream decompressionStream = new GZipStream(fstream, CompressionMode.Decompress);
TNMStagingCSharp.Src.Staging.Staging OLD_STAGING = _STAGING;
_STAGING = TNM13_STAGING;
TNMStaging_UnitTestApp.Src.Staging.IntegrationUtils.IntegrationResult TNMResult =
TNMStaging_UnitTestApp.Src.Staging.TNM.TnmIntegrationSchemaStage.processTNMSchema(_STAGING, sFilePath, decompressionStream, true);
fstream.Close();
decompressionStream.Close();
_STAGING = OLD_STAGING;
// make sure there were no errors returned
Assert.AreEqual(0, TNMResult.getNumFailures(), "There were failures in the TNMResult Staging tests.");
}
}
public void testGetInputsWithContext()
{
InMemoryDataProvider provider = new InMemoryDataProvider("test", "1.0");
StagingTable table = new StagingTable();
table.setId("table_input1");
StagingColumnDefinition def1 = new StagingColumnDefinition();
def1.setKey("input1");
def1.setName("Input 1");
def1.setType(ColumnType.INPUT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"1"
});
table.getRawRows().Add(new List <String>()
{
"2"
});
provider.addTable(table);
table = new StagingTable();
table.setId("table_input2");
def1 = new StagingColumnDefinition();
def1.setKey("input1");
def1.setName("Input 2");
def1.setType(ColumnType.INPUT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
""
});
table.getRawRows().Add(new List <String>()
{
"A"
});
table.getRawRows().Add(new List <String>()
{
"B"
});
provider.addTable(table);
table = new StagingTable();
table.setId("table_selection");
def1 = new StagingColumnDefinition();
def1.setKey("input1");
def1.setName("Input 1");
def1.setType(ColumnType.INPUT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"*"
});
provider.addTable(table);
table = new StagingTable();
table.setId("table_mapping");
def1 = new StagingColumnDefinition();
def1.setKey("input1");
def1.setName("Input 1");
def1.setType(ColumnType.INPUT);
StagingColumnDefinition def2 = new StagingColumnDefinition();
def2.setKey("input2");
def2.setName("Input 2");
def2.setType(ColumnType.INPUT);
StagingColumnDefinition def3 = new StagingColumnDefinition();
def3.setKey("mapped_field");
def3.setName("Temp value");
def3.setType(ColumnType.INPUT);
StagingColumnDefinition def4 = new StagingColumnDefinition();
def4.setKey("final_output");
def4.setName("Output");
def4.setType(ColumnType.ENDPOINT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1, def2, def3, def4
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"*", "*", "*", "VALUE:ABC"
});
provider.addTable(table);
StagingSchema schema = new StagingSchema();
schema.setId("schema_test");
schema.setSchemaSelectionTable("table_selection");
List <StagingSchemaInput> inputs = new List <StagingSchemaInput>();
inputs.Add(new StagingSchemaInput("input1", "Input 1", "table_input1"));
inputs.Add(new StagingSchemaInput("input2", "Input 2", "table_input2"));
schema.setInputs(inputs);
List <StagingSchemaOutput> outputs = new List <StagingSchemaOutput>();
outputs.Add(new StagingSchemaOutput("final_output", "Final Output"));
schema.setOutputs(outputs);
StagingMapping mapping = new StagingMapping();
mapping.setId("m1");
List <IKeyValue> mapInitialContext = new List <IKeyValue>();
mapInitialContext.Add(new StagingKeyValue("tmp_field", null));
mapping.setInitialContext(mapInitialContext);
StagingTablePath path = new StagingTablePath();
path.setId("table_mapping");
HashSet <IKeyMapping> pathInputMap = new HashSet <IKeyMapping>();
pathInputMap.Add(new StagingKeyMapping("tmp_field", "mapped_field"));
path.setInputMapping(pathInputMap);
HashSet <String> pathInputs = new HashSet <String>();
pathInputs.Add("input1");
pathInputs.Add("input2");
pathInputs.Add("tmp_field");
path.setInputs(pathInputs);
HashSet <String> pathOutputs = new HashSet <String>();
pathOutputs.Add("final_output");
path.setOutputs(pathOutputs);
List <ITablePath> mapTablePaths = new List <ITablePath>();
mapTablePaths.Add(path);
mapping.setTablePaths(mapTablePaths);
List <IMapping> schemaMappings = new List <IMapping>();
schemaMappings.Add(mapping);
schema.setMappings(schemaMappings);
provider.addSchema(schema);
TNMStagingCSharp.Src.Staging.Staging staging = TNMStagingCSharp.Src.Staging.Staging.getInstance(provider);
HashSet <String> testSet1 = staging.getInputs(staging.getSchema("schema_test"));
HashSet <String> testSet2 = new HashSet <String>();
testSet2.Add("input1");
testSet2.Add("input2");
// should only return the "real" inputs and not the temp field set in initial context
Assert.IsTrue(testSet1.SetEquals(testSet2));
}
public void testBlankInputs()
{
InMemoryDataProvider provider = new InMemoryDataProvider("test", "1.0");
StagingTable table = new StagingTable();
table.setId("table_input1");
StagingColumnDefinition def1 = new StagingColumnDefinition();
def1.setKey("input1");
def1.setName("Input 1");
def1.setType(ColumnType.INPUT);
StagingColumnDefinition def2 = new StagingColumnDefinition();
def2.setKey("result1");
def2.setName("Result1");
def2.setType(ColumnType.DESCRIPTION);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1, def2
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"1", "ONE"
});
table.getRawRows().Add(new List <String>()
{
"2", "TWO"
});
provider.addTable(table);
table = new StagingTable();
table.setId("table_input2");
def1 = new StagingColumnDefinition();
def1.setKey("input2");
def1.setName("Input 2");
def1.setType(ColumnType.INPUT);
def2 = new StagingColumnDefinition();
def2.setKey("result2");
def2.setName("Result2");
def2.setType(ColumnType.DESCRIPTION);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1, def2
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"", "Blank"
});
table.getRawRows().Add(new List <String>()
{
"A", "Letter A"
});
table.getRawRows().Add(new List <String>()
{
"B", "Letter B"
});
provider.addTable(table);
table = new StagingTable();
table.setId("table_selection");
def1 = new StagingColumnDefinition();
def1.setKey("input1");
def1.setName("Input 1");
def1.setType(ColumnType.INPUT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"*"
});
provider.addTable(table);
table = new StagingTable();
table.setId("primary_site");
def1 = new StagingColumnDefinition();
def1.setKey("site");
def1.setName("Site");
def1.setType(ColumnType.INPUT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"C509"
});
provider.addTable(table);
table = new StagingTable();
table.setId("histology");
def1 = new StagingColumnDefinition();
def1.setKey("hist");
def1.setName("Histology");
def1.setType(ColumnType.INPUT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"8000"
});
provider.addTable(table);
table = new StagingTable();
table.setId("table_year_dx");
def1 = new StagingColumnDefinition();
def1.setKey("year_dx");
def1.setName("Year DX");
def1.setType(ColumnType.INPUT);
table.setColumnDefinitions(new List <IColumnDefinition>()
{
def1
});
table.setRawRows(new List <List <String> >());
table.getRawRows().Add(new List <String>()
{
"1900-2100"
});
provider.addTable(table);
StagingSchema schema = new StagingSchema();
schema.setId("schema_test");
schema.setSchemaSelectionTable("table_selection");
List <StagingSchemaInput> inputs = new List <StagingSchemaInput>();
inputs.Add(new StagingSchemaInput("site", "Primary Site", "primary_site"));
inputs.Add(new StagingSchemaInput("hist", "Hist", "histology"));
inputs.Add(new StagingSchemaInput("year_dx", "Year DX", "table_year_dx"));
inputs.Add(new StagingSchemaInput("input1", "Input 1", "table_input1"));
inputs.Add(new StagingSchemaInput("input2", "Input 2", "table_input2"));
schema.setInputs(inputs);
provider.addSchema(schema);
TNMStagingCSharp.Src.Staging.Staging staging = TNMStagingCSharp.Src.Staging.Staging.getInstance(provider);
Assert.AreEqual("schema_test", staging.getSchema("schema_test").getId());
// check case where required input field not supplied (i.e. no default); since there are is no workflow defined, this should
// not cause an error
StagingData data = new StagingData("C509", "8000");
data.setInput("year_dx", "2018");
data.setInput("input1", "1");
staging.stage(data);
Assert.AreEqual(StagingData.Result.STAGED, data.getResult());
// pass in blank for "input2"
data = new StagingData("C509", "8000");
data.setInput("year_dx", "2018");
data.setInput("input1", "1");
data.setInput("input2", "");
staging.stage(data);
Assert.AreEqual(StagingData.Result.STAGED, data.getResult());
// pass in null for "input2"
data = new StagingData("C509", "8000");
data.setInput("year_dx", "2018");
data.setInput("input1", "1");
data.setInput("input2", null);
staging.stage(data);
Assert.AreEqual(StagingData.Result.STAGED, data.getResult());
}
// * Process all schemas in file
// * @param staging Staging object
// * @param fileName name of file
// * @param is InputStream
// * @param singleLineNumber if not null, only process this line number
// * @return IntegrationResult
public static IntegrationUtils.IntegrationResult processSchema_MultiTask(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream, int singleLineNumber) //throws IOException, InterruptedException
{
// set up a mapping of output field positions in the CSV file
Dictionary <CsOutput, int> mappings = new Dictionary <CsOutput, int>(100);
mappings[CsOutput.AJCC6_T] = 42;
mappings[CsOutput.AJCC6_TDESCRIPTOR] = 43;
mappings[CsOutput.AJCC6_N] = 44;
mappings[CsOutput.AJCC6_NDESCRIPTOR] = 45;
mappings[CsOutput.AJCC6_M] = 46;
mappings[CsOutput.AJCC6_MDESCRIPTOR] = 47;
mappings[CsOutput.AJCC6_STAGE] = 48;
mappings[CsOutput.AJCC7_T] = 49;
mappings[CsOutput.AJCC7_TDESCRIPTOR] = 50;
mappings[CsOutput.AJCC7_N] = 51;
mappings[CsOutput.AJCC7_NDESCRIPTOR] = 52;
mappings[CsOutput.AJCC7_M] = 53;
mappings[CsOutput.AJCC7_MDESCRIPTOR] = 54;
mappings[CsOutput.AJCC7_STAGE] = 55;
mappings[CsOutput.SS1977_T] = 56;
mappings[CsOutput.SS1977_N] = 57;
mappings[CsOutput.SS1977_M] = 58;
mappings[CsOutput.SS1977_STAGE] = 59;
mappings[CsOutput.SS2000_T] = 60;
mappings[CsOutput.SS2000_N] = 61;
mappings[CsOutput.SS2000_M] = 62;
mappings[CsOutput.SS2000_STAGE] = 63;
mappings[CsOutput.STOR_AJCC6_T] = 64;
mappings[CsOutput.STOR_AJCC6_TDESCRIPTOR] = 65;
mappings[CsOutput.STOR_AJCC6_N] = 66;
mappings[CsOutput.STOR_AJCC6_NDESCRIPTOR] = 67;
mappings[CsOutput.STOR_AJCC6_M] = 68;
mappings[CsOutput.STOR_AJCC6_MDESCRIPTOR] = 69;
mappings[CsOutput.STOR_AJCC6_STAGE] = 70;
mappings[CsOutput.STOR_AJCC7_T] = 71;
mappings[CsOutput.STOR_AJCC7_TDESCRIPTOR] = 72;
mappings[CsOutput.STOR_AJCC7_N] = 73;
mappings[CsOutput.STOR_AJCC7_NDESCRIPTOR] = 74;
mappings[CsOutput.STOR_AJCC7_M] = 75;
mappings[CsOutput.STOR_AJCC7_MDESCRIPTOR] = 76;
mappings[CsOutput.STOR_AJCC7_STAGE] = 77;
mappings[CsOutput.STOR_SS1977_STAGE] = 78;
mappings[CsOutput.STOR_SS2000_STAGE] = 79;
// go over each file
int processedCases = 0;
int iLineNumber = 0;
//int iLineCounter = 0;
MultiTasksExecutor thisMultiTasksExecutor = new MultiTasksExecutor();
thisMultiTasksExecutor.AddAction(new MultiTasksExecutor.ActionCallBack(MultiTask_TaskCompute));
int iThreads = thisMultiTasksExecutor.GetNumThreads();
mMultiTask_Staging = staging;
miMultiTask_FailedCases = 0;
miMultiTask_ThreadProcessedCases = 0;
thisMultiTasksExecutor.StartTasks();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
if (singleLineNumber >= 0)
{
IntegrationUtils.WritelineToLog("Starting " + fileName + ", line # " + singleLineNumber + "...");
}
else
{
IntegrationUtils.WritelineToLog("Starting " + fileName + " [" + (iThreads + 1) + " threads]");
}
// loop over each line in the file
TextReader reader = new StreamReader(inputStream);
String line = reader.ReadLine();
while (line != null)
{
iLineNumber++;
if (iLineNumber >= 0)
{
processedCases++;
// split the CSV record
String[] values = line.Split(",".ToCharArray());
// if a single line was requested, skip all other lines
if (singleLineNumber >= 0 && singleLineNumber != iLineNumber)
{
continue;
}
if (values.Length != 80)
{
IntegrationUtils.WritelineToLog("Line " + iLineNumber + " has " + values.Length + " cells; it should be 80.");
}
else
{
MultiTask_DataObj obj = new MultiTask_DataObj();
obj.mMappings = mappings;
obj.mValues = values;
obj.msFileName = fileName;
obj.miLineNum = iLineNumber;
thisMultiTasksExecutor.AddDataItem(obj);
/*
* iLineCounter++;
* if (iLineCounter > 50000)
* {
* IntegrationUtils.WritelineToLog("Time: " + stopwatch.Elapsed.TotalMilliseconds + " ms.");
* iLineCounter = 0;
* }
*/
}
}
line = reader.ReadLine();
}
thisMultiTasksExecutor.WaitForCompletion();
stopwatch.Stop();
String perMs = String.Format("{0,12:F4}", ((float)stopwatch.Elapsed.TotalMilliseconds / processedCases)).Trim();
IntegrationUtils.WritelineToLog("Completed " + processedCases + " cases for " + fileName + " in " + TNMStaging_UnitTestApp.Src.Staging.IntegrationUtils.GenerateTotalTimeString(stopwatch) + " (" + perMs + " ms/case).");
IntegrationUtils.WritelineToLog("Threads Completed " + miMultiTask_ThreadProcessedCases + " cases.");
if (miMultiTask_FailedCases > 0)
{
IntegrationUtils.WritelineToLog("There were " + miMultiTask_FailedCases + " failures.");
}
else
{
IntegrationUtils.WritelineToLog("");
}
IntegrationUtils.WritelineToLog("-----------------------------------------------");
inputStream.Close();
return(new IntegrationUtils.IntegrationResult(processedCases, miMultiTask_FailedCases));
}
// * Process all schemas in file
// * @param staging Staging object
// * @param fileName name of file
// * @param is InputStream
// * @return IntegrationResult
public static IntegrationUtils.IntegrationResult processSchema(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream) //throws IOException, InterruptedException
{
return(processSchema_MultiTask(staging, fileName, inputStream, -1));
}
// * Process all schemas in TNM file
// * @param staging Staging object
// * @param fileName name of file
// * @param is InputStream
// * @param singleLineNumber if not null, only process this line number
// * @return IntegrationResult
public static IntegrationUtils.IntegrationResult processTNMSchema(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream, int singleLineNumber, bool bJSONFormat) //throws IOException, InterruptedException
{
Dictionary <TnmOutput, String> output_values = null;
Dictionary <TnmInput, String> input_values = null;
// initialize the threads pool
int n = Math.Min(9, Environment.ProcessorCount + 1);
ThreadPool.SetMaxThreads(n, n);
// go over each file
int processedCases = 0;
int iLineNumber = 0;
//int iLineCounter = 0;
MultiTasksExecutor thisMultiTasksExecutor = new MultiTasksExecutor();
thisMultiTasksExecutor.AddAction(new MultiTasksExecutor.ActionCallBack(MultiTask_TaskCompute));
int iThreads = thisMultiTasksExecutor.GetNumThreads();
mMultiTask_Staging = staging;
miMultiTask_FailedCases = 0;
miMultiTask_ThreadProcessedCases = 0;
thisMultiTasksExecutor.StartTasks();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
if (singleLineNumber >= 0)
{
System.Diagnostics.Trace.WriteLine("Starting " + fileName + ", line # " + singleLineNumber + " [" + n + " threads]");
}
else
{
System.Diagnostics.Trace.WriteLine("Starting " + fileName + " [" + (iThreads + 1) + " threads]");
}
// loop over each line in the file
TextReader reader = new StreamReader(inputStream);
String line = reader.ReadLine();
String input_line = "";
String output_line = "";
String[] input_strs;
String[] output_strs;
String[] entries;
String sExpectedResult = "";
bool bStageThis = true;
while (line != null)
{
iLineNumber++;
if (iLineNumber >= 0)
{
bStageThis = true;
if (bJSONFormat)
{
bStageThis = false;
if (line.IndexOf("input=") >= 0)
{
input_line = line.Trim();
}
if (line.IndexOf("expectedOutput=") >= 0)
{
output_line = line.Trim();
}
if (output_line.Length > 0)
{
bStageThis = true;
input_values = new Dictionary <TnmInput, String>();
output_values = new Dictionary <TnmOutput, String>();
input_line = input_line.Substring(7, input_line.Length - 8).Trim();
output_line = output_line.Substring(16, output_line.Length - 17).Trim();
input_strs = input_line.Split(",".ToCharArray());
output_strs = output_line.Split(",".ToCharArray());
// set up a mapping of output field positions in the CSV file
foreach (String s in input_strs)
{
entries = s.Split("=".ToCharArray());
if (entries.Length == 2)
{
entries[0] = entries[0].Trim();
entries[1] = entries[1].Trim();
foreach (TnmInput inp in TnmInput.Values)
{
if (inp.toString() == entries[0])
{
input_values.Add(inp, entries[1]);
}
}
}
else
{
System.Diagnostics.Trace.WriteLine("Line " + iLineNumber + " has " + entries.Length + " cells; it should be 2. (" + input_line + ")");
}
}
foreach (String s in output_strs)
{
entries = s.Split("=".ToCharArray());
if (entries.Length == 2)
{
entries[0] = entries[0].Trim();
entries[1] = entries[1].Trim();
foreach (TnmOutput outp in TnmOutput.Values)
{
if (outp.toString() == entries[0])
{
output_values.Add(outp, entries[1]);
}
}
}
else
{
System.Diagnostics.Trace.WriteLine("Line " + iLineNumber + " has " + entries.Length + " cells; it should be 2. (" + output_line + ")");
}
}
}
}
else
{
input_values = new Dictionary <TnmInput, String>();
output_values = new Dictionary <TnmOutput, String>();
// Each line is a comma delimited string.
input_strs = line.Split(",".ToCharArray());
if (input_strs.Length == 54)
{
String sVal = "";
for (int i = 0; i < 54; i++)
{
sVal = input_strs[i].Trim();
if (sVal.Length > 0)
{
TnmInput in_key = TnmInput.PRIMARY_SITE;
TnmOutput out_key = TnmOutput.DERIVED_VERSION;
switch (i)
{
case 0: in_key = TnmInput.PRIMARY_SITE; break;
case 1: in_key = TnmInput.HISTOLOGY; break;
case 2: in_key = TnmInput.DX_YEAR; break;
case 3: in_key = TnmInput.BEHAVIOR; break;
case 4: in_key = TnmInput.GRADE; break;
case 5: in_key = TnmInput.SEX; break;
case 6: in_key = TnmInput.AGE_AT_DX; break;
case 7: in_key = TnmInput.RX_SUMM_SURGERY; break;
case 8: in_key = TnmInput.RX_SUMM_RADIATION; break;
case 9: in_key = TnmInput.REGIONAL_NODES_POSITIVE; break;
case 10: in_key = TnmInput.CLIN_T; break;
case 11: in_key = TnmInput.CLIN_N; break;
case 12: in_key = TnmInput.CLIN_M; break;
case 13: in_key = TnmInput.CLIN_STAGE_GROUP_DIRECT; break;
case 14: in_key = TnmInput.PATH_T; break;
case 15: in_key = TnmInput.PATH_N; break;
case 16: in_key = TnmInput.PATH_M; break;
case 17: in_key = TnmInput.PATH_STAGE_GROUP_DIRECT; break;
case 18: in_key = TnmInput.SSF1; break;
case 19: in_key = TnmInput.SSF2; break;
case 20: in_key = TnmInput.SSF3; break;
case 21: in_key = TnmInput.SSF4; break;
case 22: in_key = TnmInput.SSF5; break;
case 23: in_key = TnmInput.SSF6; break;
case 24: in_key = TnmInput.SSF7; break;
case 25: in_key = TnmInput.SSF8; break;
case 26: in_key = TnmInput.SSF9; break;
case 27: in_key = TnmInput.SSF10; break;
case 28: in_key = TnmInput.SSF11; break;
case 29: in_key = TnmInput.SSF12; break;
case 30: in_key = TnmInput.SSF13; break;
case 31: in_key = TnmInput.SSF14; break;
case 32: in_key = TnmInput.SSF15; break;
case 33: in_key = TnmInput.SSF16; break;
case 34: in_key = TnmInput.SSF17; break;
case 35: in_key = TnmInput.SSF18; break;
case 36: in_key = TnmInput.SSF19; break;
case 37: in_key = TnmInput.SSF20; break;
case 38: in_key = TnmInput.SSF21; break;
case 39: in_key = TnmInput.SSF22; break;
case 40: in_key = TnmInput.SSF23; break;
case 41: in_key = TnmInput.SSF24; break;
case 42: in_key = TnmInput.SSF25; break;
case 43: out_key = TnmOutput.DERIVED_VERSION; break;
case 44: out_key = TnmOutput.CLIN_STAGE_GROUP; break;
case 45: out_key = TnmOutput.PATH_STAGE_GROUP; break;
case 46: out_key = TnmOutput.COMBINED_STAGE_GROUP; break;
case 47: out_key = TnmOutput.COMBINED_T; break;
case 48: out_key = TnmOutput.COMBINED_N; break;
case 49: out_key = TnmOutput.COMBINED_M; break;
case 50: out_key = TnmOutput.SOURCE_T; break;
case 51: out_key = TnmOutput.SOURCE_N; break;
case 52: out_key = TnmOutput.SOURCE_M; break;
}
if (i <= 42)
{
input_values.Add(in_key, sVal);
}
else if (i <= 52)
{
output_values.Add(out_key, sVal);
}
}
}
sExpectedResult = input_strs[53];
}
else
{
System.Diagnostics.Trace.WriteLine("Error: Line " + iLineNumber + " has " + input_strs.Length + " entries.");
}
}
if (bStageThis)
{
processedCases++;
MultiTask_DataObj obj = new MultiTask_DataObj();
obj.mInputValues = input_values;
obj.mOutputValues = output_values;
obj.mbJSONFormat = bJSONFormat;
obj.msExpectedResult = sExpectedResult;
obj.msFileName = fileName;
obj.miLineNum = iLineNumber;
thisMultiTasksExecutor.AddDataItem(obj);
// DEBUG
/*
* iLineCounter++;
* if (iLineCounter >= 50000)
* {
* IntegrationUtils.WritelineToLog("Line: " + iLineNumber + " Time: " + stopwatch.Elapsed.TotalMilliseconds + " ms.");
* iLineCounter = 0;
* }
*/
input_line = "";
output_line = "";
}
}
line = reader.ReadLine();
}
thisMultiTasksExecutor.WaitForCompletion();
stopwatch.Stop();
String perMs = String.Format("{0,12:F4}", ((float)stopwatch.Elapsed.TotalMilliseconds / processedCases)).Trim();
IntegrationUtils.WritelineToLog("Completed " + processedCases + " cases for " + fileName + " in " + TNMStaging_UnitTestApp.Src.Staging.IntegrationUtils.GenerateTotalTimeString(stopwatch) + " (" + perMs + " ms/case).");
IntegrationUtils.WritelineToLog("Threads Completed " + miMultiTask_ThreadProcessedCases + " cases.");
if (miMultiTask_FailedCases > 0)
{
System.Diagnostics.Trace.WriteLine("There were " + miMultiTask_FailedCases + " failures.");
}
else
{
System.Diagnostics.Trace.WriteLine("");
}
System.Diagnostics.Trace.WriteLine("-----------------------------------------------");
inputStream.Close();
return(new IntegrationUtils.IntegrationResult(processedCases, miMultiTask_FailedCases));
}
// * Process all schemas in the TNM file
// * @param staging Staging object
// * @param fileName name of file
// * @param is InputStream
// * @return IntegrationResult
public static IntegrationUtils.IntegrationResult processTNMSchema(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream, bool bJSONFormat) //throws IOException, InterruptedException
{
return(processTNMSchema(staging, fileName, inputStream, -1, bJSONFormat));
}
// * Process all schemas in file
// * @param staging Staging object
// * @param fileName name of file
// * @param is InputStream
// * @param singleLineNumber if not null, only process this line number
// * @return IntegrationResult
public static IntegrationUtils.IntegrationResult processSchemaSelection_MultiTask(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream, TestContext testLog) //throws IOException, InterruptedException
{
IntegrationUtils.TestReportLog = testLog;
// go over each file
int processedCases = 0;
int iLineNumber = 0;
//int iLineCounter = 0;
MultiTasksExecutor thisMultiTasksExecutor = new MultiTasksExecutor();
thisMultiTasksExecutor.AddAction(new MultiTasksExecutor.ActionCallBack(MultiTask_TaskCompute));
thisMultiTasksExecutor.StartTasks();
mMultiTask_Staging = staging;
miMultiTask_FailedCases = 0;
miMultiTask_ThreadProcessedCases = 0;
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// loop over each line in the file
TextReader reader = new StreamReader(inputStream);
String line = reader.ReadLine();
while (line != null)
{
iLineNumber++;
if (iLineNumber >= 0)
{
processedCases++;
// split the string; important to keep empty trailing values in the resulting array
String[] parts = line.Split(",".ToCharArray(), 1000);
if (parts.Length != 4)
{
throw new System.InvalidOperationException("Bad record in schema_selection.txt on line number: " + iLineNumber);
}
else
{
MultiTask_DataObj obj = new MultiTask_DataObj();
obj.mParts = parts;
obj.msFullLine = line;
obj.miLineNum = iLineNumber;
thisMultiTasksExecutor.AddDataItem(obj);
// DEBUG
/*
* iLineCounter++;
* if (iLineCounter >= 500000)
* {
* IntegrationUtils.WritelineToLog("Processed Cases: " + processedCases);
* iLineCounter = 0;
* }
*/
}
}
line = reader.ReadLine();
}
thisMultiTasksExecutor.WaitForCompletion();
stopwatch.Stop();
String perMs = String.Format("{0,12:F4}", ((float)stopwatch.Elapsed.TotalMilliseconds / processedCases)).Trim();
IntegrationUtils.WritelineToLog("Completed " + processedCases + " cases for " + fileName + " in " + TNMStaging_UnitTestApp.Src.Staging.IntegrationUtils.GenerateTotalTimeString(stopwatch) + " (" + perMs + " ms/case).");
IntegrationUtils.WritelineToLog("Threads Completed " + miMultiTask_ThreadProcessedCases + " cases.");
if (miMultiTask_FailedCases > 0)
{
IntegrationUtils.WritelineToLog("There were " + miMultiTask_FailedCases + " failures.");
}
else
{
IntegrationUtils.WritelineToLog("");
}
IntegrationUtils.WritelineToLog("-----------------------------------------------");
inputStream.Close();
return(new IntegrationUtils.IntegrationResult(processedCases, miMultiTask_FailedCases));
}
public static IntegrationUtils.IntegrationResult processSchemaSelection(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream, TestContext testLog) //throws IOException, InterruptedException
{
return(processSchemaSelection_MultiTask(staging, fileName, inputStream, testLog));
}