// * Process all schemas in file
        // * @param staging Staging object
        // * @param fileName name of file
        // * @param is InputStream
        // * @param singleLineNumber if not null, only process this line number
        // * @return IntegrationResult
        public static IntegrationUtils.IntegrationResult processSchema_MultiTask(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream, int singleLineNumber) //throws IOException, InterruptedException
        {
            // set up a mapping of output field positions in the CSV file
            Dictionary <CsOutput, int> mappings = new Dictionary <CsOutput, int>(100);

            mappings[CsOutput.AJCC6_T]                = 42;
            mappings[CsOutput.AJCC6_TDESCRIPTOR]      = 43;
            mappings[CsOutput.AJCC6_N]                = 44;
            mappings[CsOutput.AJCC6_NDESCRIPTOR]      = 45;
            mappings[CsOutput.AJCC6_M]                = 46;
            mappings[CsOutput.AJCC6_MDESCRIPTOR]      = 47;
            mappings[CsOutput.AJCC6_STAGE]            = 48;
            mappings[CsOutput.AJCC7_T]                = 49;
            mappings[CsOutput.AJCC7_TDESCRIPTOR]      = 50;
            mappings[CsOutput.AJCC7_N]                = 51;
            mappings[CsOutput.AJCC7_NDESCRIPTOR]      = 52;
            mappings[CsOutput.AJCC7_M]                = 53;
            mappings[CsOutput.AJCC7_MDESCRIPTOR]      = 54;
            mappings[CsOutput.AJCC7_STAGE]            = 55;
            mappings[CsOutput.SS1977_T]               = 56;
            mappings[CsOutput.SS1977_N]               = 57;
            mappings[CsOutput.SS1977_M]               = 58;
            mappings[CsOutput.SS1977_STAGE]           = 59;
            mappings[CsOutput.SS2000_T]               = 60;
            mappings[CsOutput.SS2000_N]               = 61;
            mappings[CsOutput.SS2000_M]               = 62;
            mappings[CsOutput.SS2000_STAGE]           = 63;
            mappings[CsOutput.STOR_AJCC6_T]           = 64;
            mappings[CsOutput.STOR_AJCC6_TDESCRIPTOR] = 65;
            mappings[CsOutput.STOR_AJCC6_N]           = 66;
            mappings[CsOutput.STOR_AJCC6_NDESCRIPTOR] = 67;
            mappings[CsOutput.STOR_AJCC6_M]           = 68;
            mappings[CsOutput.STOR_AJCC6_MDESCRIPTOR] = 69;
            mappings[CsOutput.STOR_AJCC6_STAGE]       = 70;
            mappings[CsOutput.STOR_AJCC7_T]           = 71;
            mappings[CsOutput.STOR_AJCC7_TDESCRIPTOR] = 72;
            mappings[CsOutput.STOR_AJCC7_N]           = 73;
            mappings[CsOutput.STOR_AJCC7_NDESCRIPTOR] = 74;
            mappings[CsOutput.STOR_AJCC7_M]           = 75;
            mappings[CsOutput.STOR_AJCC7_MDESCRIPTOR] = 76;
            mappings[CsOutput.STOR_AJCC7_STAGE]       = 77;
            mappings[CsOutput.STOR_SS1977_STAGE]      = 78;
            mappings[CsOutput.STOR_SS2000_STAGE]      = 79;

            // go over each file
            int processedCases = 0;
            int iLineNumber    = 0;
            //int iLineCounter = 0;

            MultiTasksExecutor thisMultiTasksExecutor = new MultiTasksExecutor();

            thisMultiTasksExecutor.AddAction(new MultiTasksExecutor.ActionCallBack(MultiTask_TaskCompute));

            int iThreads = thisMultiTasksExecutor.GetNumThreads();

            mMultiTask_Staging               = staging;
            miMultiTask_FailedCases          = 0;
            miMultiTask_ThreadProcessedCases = 0;
            thisMultiTasksExecutor.StartTasks();

            Stopwatch stopwatch = new Stopwatch();

            stopwatch.Start();

            if (singleLineNumber >= 0)
            {
                IntegrationUtils.WritelineToLog("Starting " + fileName + ", line # " + singleLineNumber + "...");
            }
            else
            {
                IntegrationUtils.WritelineToLog("Starting " + fileName + " [" + (iThreads + 1) + " threads]");
            }

            // loop over each line in the file
            TextReader reader = new StreamReader(inputStream);
            String     line   = reader.ReadLine();

            while (line != null)
            {
                iLineNumber++;

                if (iLineNumber >= 0)
                {
                    processedCases++;

                    // split the CSV record
                    String[] values = line.Split(",".ToCharArray());

                    // if a single line was requested, skip all other lines
                    if (singleLineNumber >= 0 && singleLineNumber != iLineNumber)
                    {
                        continue;
                    }

                    if (values.Length != 80)
                    {
                        IntegrationUtils.WritelineToLog("Line " + iLineNumber + " has " + values.Length + " cells; it should be 80.");
                    }
                    else
                    {
                        MultiTask_DataObj obj = new MultiTask_DataObj();
                        obj.mMappings  = mappings;
                        obj.mValues    = values;
                        obj.msFileName = fileName;
                        obj.miLineNum  = iLineNumber;

                        thisMultiTasksExecutor.AddDataItem(obj);

                        /*
                         * iLineCounter++;
                         * if (iLineCounter > 50000)
                         * {
                         *  IntegrationUtils.WritelineToLog("Time: " + stopwatch.Elapsed.TotalMilliseconds + " ms.");
                         *  iLineCounter = 0;
                         * }
                         */
                    }
                }

                line = reader.ReadLine();
            }

            thisMultiTasksExecutor.WaitForCompletion();


            stopwatch.Stop();
            String perMs = String.Format("{0,12:F4}", ((float)stopwatch.Elapsed.TotalMilliseconds / processedCases)).Trim();

            IntegrationUtils.WritelineToLog("Completed " + processedCases + " cases for " + fileName + " in " + TNMStaging_UnitTestApp.Src.Staging.IntegrationUtils.GenerateTotalTimeString(stopwatch) + " (" + perMs + " ms/case).");

            IntegrationUtils.WritelineToLog("Threads Completed " + miMultiTask_ThreadProcessedCases + " cases.");


            if (miMultiTask_FailedCases > 0)
            {
                IntegrationUtils.WritelineToLog("There were " + miMultiTask_FailedCases + " failures.");
            }
            else
            {
                IntegrationUtils.WritelineToLog("");
            }


            IntegrationUtils.WritelineToLog("-----------------------------------------------");

            inputStream.Close();

            return(new IntegrationUtils.IntegrationResult(processedCases, miMultiTask_FailedCases));
        }
Exemple #2
0
        // * Process all schemas in TNM file
        // * @param staging Staging object
        // * @param fileName name of file
        // * @param is InputStream
        // * @param singleLineNumber if not null, only process this line number
        // * @return IntegrationResult
        public static IntegrationUtils.IntegrationResult processTNMSchema(TNMStagingCSharp.Src.Staging.Staging staging, String fileName, Stream inputStream, int singleLineNumber, bool bJSONFormat) //throws IOException, InterruptedException
        {
            Dictionary <TnmOutput, String> output_values = null;
            Dictionary <TnmInput, String>  input_values  = null;

            // initialize the threads pool
            int n = Math.Min(9, Environment.ProcessorCount + 1);

            ThreadPool.SetMaxThreads(n, n);

            // go over each file
            int processedCases = 0;
            int iLineNumber    = 0;
            //int iLineCounter = 0;

            MultiTasksExecutor thisMultiTasksExecutor = new MultiTasksExecutor();

            thisMultiTasksExecutor.AddAction(new MultiTasksExecutor.ActionCallBack(MultiTask_TaskCompute));

            int iThreads = thisMultiTasksExecutor.GetNumThreads();

            mMultiTask_Staging               = staging;
            miMultiTask_FailedCases          = 0;
            miMultiTask_ThreadProcessedCases = 0;
            thisMultiTasksExecutor.StartTasks();

            Stopwatch stopwatch = new Stopwatch();

            stopwatch.Start();

            if (singleLineNumber >= 0)
            {
                System.Diagnostics.Trace.WriteLine("Starting " + fileName + ", line # " + singleLineNumber + " [" + n + " threads]");
            }
            else
            {
                System.Diagnostics.Trace.WriteLine("Starting " + fileName + " [" + (iThreads + 1) + " threads]");
            }

            // loop over each line in the file
            TextReader reader      = new StreamReader(inputStream);
            String     line        = reader.ReadLine();
            String     input_line  = "";
            String     output_line = "";

            String[] input_strs;
            String[] output_strs;
            String[] entries;
            String   sExpectedResult = "";
            bool     bStageThis      = true;

            while (line != null)
            {
                iLineNumber++;

                if (iLineNumber >= 0)
                {
                    bStageThis = true;
                    if (bJSONFormat)
                    {
                        bStageThis = false;
                        if (line.IndexOf("input=") >= 0)
                        {
                            input_line = line.Trim();
                        }
                        if (line.IndexOf("expectedOutput=") >= 0)
                        {
                            output_line = line.Trim();
                        }

                        if (output_line.Length > 0)
                        {
                            bStageThis    = true;
                            input_values  = new Dictionary <TnmInput, String>();
                            output_values = new Dictionary <TnmOutput, String>();

                            input_line  = input_line.Substring(7, input_line.Length - 8).Trim();
                            output_line = output_line.Substring(16, output_line.Length - 17).Trim();

                            input_strs  = input_line.Split(",".ToCharArray());
                            output_strs = output_line.Split(",".ToCharArray());

                            // set up a mapping of output field positions in the CSV file

                            foreach (String s in input_strs)
                            {
                                entries = s.Split("=".ToCharArray());
                                if (entries.Length == 2)
                                {
                                    entries[0] = entries[0].Trim();
                                    entries[1] = entries[1].Trim();
                                    foreach (TnmInput inp in TnmInput.Values)
                                    {
                                        if (inp.toString() == entries[0])
                                        {
                                            input_values.Add(inp, entries[1]);
                                        }
                                    }
                                }
                                else
                                {
                                    System.Diagnostics.Trace.WriteLine("Line " + iLineNumber + " has " + entries.Length + " cells; it should be 2. (" + input_line + ")");
                                }
                            }

                            foreach (String s in output_strs)
                            {
                                entries = s.Split("=".ToCharArray());
                                if (entries.Length == 2)
                                {
                                    entries[0] = entries[0].Trim();
                                    entries[1] = entries[1].Trim();
                                    foreach (TnmOutput outp in TnmOutput.Values)
                                    {
                                        if (outp.toString() == entries[0])
                                        {
                                            output_values.Add(outp, entries[1]);
                                        }
                                    }
                                }
                                else
                                {
                                    System.Diagnostics.Trace.WriteLine("Line " + iLineNumber + " has " + entries.Length + " cells; it should be 2. (" + output_line + ")");
                                }
                            }
                        }
                    }
                    else
                    {
                        input_values  = new Dictionary <TnmInput, String>();
                        output_values = new Dictionary <TnmOutput, String>();

                        // Each line is a comma delimited string.
                        input_strs = line.Split(",".ToCharArray());

                        if (input_strs.Length == 54)
                        {
                            String sVal = "";
                            for (int i = 0; i < 54; i++)
                            {
                                sVal = input_strs[i].Trim();
                                if (sVal.Length > 0)
                                {
                                    TnmInput  in_key  = TnmInput.PRIMARY_SITE;
                                    TnmOutput out_key = TnmOutput.DERIVED_VERSION;
                                    switch (i)
                                    {
                                    case 0: in_key = TnmInput.PRIMARY_SITE; break;

                                    case 1: in_key = TnmInput.HISTOLOGY; break;

                                    case 2: in_key = TnmInput.DX_YEAR; break;

                                    case 3: in_key = TnmInput.BEHAVIOR; break;

                                    case 4: in_key = TnmInput.GRADE; break;

                                    case 5: in_key = TnmInput.SEX; break;

                                    case 6: in_key = TnmInput.AGE_AT_DX; break;

                                    case 7: in_key = TnmInput.RX_SUMM_SURGERY; break;

                                    case 8: in_key = TnmInput.RX_SUMM_RADIATION; break;

                                    case 9: in_key = TnmInput.REGIONAL_NODES_POSITIVE; break;

                                    case 10: in_key = TnmInput.CLIN_T; break;

                                    case 11: in_key = TnmInput.CLIN_N; break;

                                    case 12: in_key = TnmInput.CLIN_M; break;

                                    case 13: in_key = TnmInput.CLIN_STAGE_GROUP_DIRECT; break;

                                    case 14: in_key = TnmInput.PATH_T; break;

                                    case 15: in_key = TnmInput.PATH_N; break;

                                    case 16: in_key = TnmInput.PATH_M; break;

                                    case 17: in_key = TnmInput.PATH_STAGE_GROUP_DIRECT; break;

                                    case 18: in_key = TnmInput.SSF1; break;

                                    case 19: in_key = TnmInput.SSF2; break;

                                    case 20: in_key = TnmInput.SSF3; break;

                                    case 21: in_key = TnmInput.SSF4; break;

                                    case 22: in_key = TnmInput.SSF5; break;

                                    case 23: in_key = TnmInput.SSF6; break;

                                    case 24: in_key = TnmInput.SSF7; break;

                                    case 25: in_key = TnmInput.SSF8; break;

                                    case 26: in_key = TnmInput.SSF9; break;

                                    case 27: in_key = TnmInput.SSF10; break;

                                    case 28: in_key = TnmInput.SSF11; break;

                                    case 29: in_key = TnmInput.SSF12; break;

                                    case 30: in_key = TnmInput.SSF13; break;

                                    case 31: in_key = TnmInput.SSF14; break;

                                    case 32: in_key = TnmInput.SSF15; break;

                                    case 33: in_key = TnmInput.SSF16; break;

                                    case 34: in_key = TnmInput.SSF17; break;

                                    case 35: in_key = TnmInput.SSF18; break;

                                    case 36: in_key = TnmInput.SSF19; break;

                                    case 37: in_key = TnmInput.SSF20; break;

                                    case 38: in_key = TnmInput.SSF21; break;

                                    case 39: in_key = TnmInput.SSF22; break;

                                    case 40: in_key = TnmInput.SSF23; break;

                                    case 41: in_key = TnmInput.SSF24; break;

                                    case 42: in_key = TnmInput.SSF25; break;

                                    case 43: out_key = TnmOutput.DERIVED_VERSION; break;

                                    case 44: out_key = TnmOutput.CLIN_STAGE_GROUP; break;

                                    case 45: out_key = TnmOutput.PATH_STAGE_GROUP; break;

                                    case 46: out_key = TnmOutput.COMBINED_STAGE_GROUP; break;

                                    case 47: out_key = TnmOutput.COMBINED_T; break;

                                    case 48: out_key = TnmOutput.COMBINED_N; break;

                                    case 49: out_key = TnmOutput.COMBINED_M; break;

                                    case 50: out_key = TnmOutput.SOURCE_T; break;

                                    case 51: out_key = TnmOutput.SOURCE_N; break;

                                    case 52: out_key = TnmOutput.SOURCE_M; break;
                                    }
                                    if (i <= 42)
                                    {
                                        input_values.Add(in_key, sVal);
                                    }
                                    else if (i <= 52)
                                    {
                                        output_values.Add(out_key, sVal);
                                    }
                                }
                            }
                            sExpectedResult = input_strs[53];
                        }
                        else
                        {
                            System.Diagnostics.Trace.WriteLine("Error: Line " + iLineNumber + " has " + input_strs.Length + " entries.");
                        }
                    }

                    if (bStageThis)
                    {
                        processedCases++;

                        MultiTask_DataObj obj = new MultiTask_DataObj();
                        obj.mInputValues     = input_values;
                        obj.mOutputValues    = output_values;
                        obj.mbJSONFormat     = bJSONFormat;
                        obj.msExpectedResult = sExpectedResult;
                        obj.msFileName       = fileName;
                        obj.miLineNum        = iLineNumber;

                        thisMultiTasksExecutor.AddDataItem(obj);

                        // DEBUG

                        /*
                         * iLineCounter++;
                         * if (iLineCounter >= 50000)
                         * {
                         *  IntegrationUtils.WritelineToLog("Line: " + iLineNumber + "   Time: " + stopwatch.Elapsed.TotalMilliseconds + " ms.");
                         *  iLineCounter = 0;
                         * }
                         */

                        input_line  = "";
                        output_line = "";
                    }
                }

                line = reader.ReadLine();
            }

            thisMultiTasksExecutor.WaitForCompletion();

            stopwatch.Stop();
            String perMs = String.Format("{0,12:F4}", ((float)stopwatch.Elapsed.TotalMilliseconds / processedCases)).Trim();

            IntegrationUtils.WritelineToLog("Completed " + processedCases + " cases for " + fileName + " in " + TNMStaging_UnitTestApp.Src.Staging.IntegrationUtils.GenerateTotalTimeString(stopwatch) + " (" + perMs + " ms/case).");
            IntegrationUtils.WritelineToLog("Threads Completed " + miMultiTask_ThreadProcessedCases + " cases.");
            if (miMultiTask_FailedCases > 0)
            {
                System.Diagnostics.Trace.WriteLine("There were " + miMultiTask_FailedCases + " failures.");
            }
            else
            {
                System.Diagnostics.Trace.WriteLine("");
            }


            System.Diagnostics.Trace.WriteLine("-----------------------------------------------");

            inputStream.Close();

            return(new IntegrationUtils.IntegrationResult(processedCases, miMultiTask_FailedCases));
        }