コード例 #1
0
        //the final normalized and concatenated file doesnt need to be vectorized + normalized again.
        // if first line "instances data = " is 4 and the rest of the lines are 3, then the trick is used.
        // if all lines are 3 then only tr3 is used
        // if all lines except the saveArff is tr4 then it will use tr4 data and save to a filename with "3" in it, at the moment this is done for compatibility when loading this file in real-time
        public static Instances ConcatenationPipeLine(string filenameTR3, string filenameTR4)
        {
            //filter tr3 based on top 1000 from tr4 (the trick)
            //load TR4 !!! NOTE: trick changed from tr4 to 3 because i didnt see any increase in % in real time.. + this line can be removed to speed up things.
            Instances data;

            if (GuiPreferences.Instance.CbPeekHigherTRsIGChecked)
            {
                GuiPreferences.Instance.setLog("using final dataset: " + filenameTR4);
                data = WekaTrainingMethods.loadDataSetFile(filenameTR4); // peeking at a higher TR's IG values (trick)
            }
            else
            {
                GuiPreferences.Instance.setLog("using final dataset: " + filenameTR3);
                data = WekaTrainingMethods.loadDataSetFile(filenameTR3); // no peeking (no trick)
            }


            //select 1000 IG values, serialize to file
            WekaTrainingMethods.selectIGSerialize(ref data);

            //load tr3
            data = WekaTrainingMethods.loadDataSetFile(filenameTR3);



            //filter top IG
            data = WekaTrainingMethods.useRemoveFilter(data, Preferences.Instance.attsel.selectedAttributes(), true);

            //save filtered tr3 to a file
            WekaCommonFileOperation.SaveArff(data, GuiPreferences.Instance.WorkDirectory + filenameTR3 + "_filteredIG.arff");
            WekaCommonFileOperation.SaveCSV(data, GuiPreferences.Instance.WorkDirectory + filenameTR3 + "_filteredIG_CSV.arff");

            return(data);
        }
コード例 #2
0
        public static void WekaTrainingPipelineForMultiRuns()
        {
            //todo check max index in file,
            //todo check if needs to remove 204801 from it so it doesnt effect the class.

            Instances finalData = Training_MultiRunProcessing.ConcatenationPipeLine("TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff",
                                                                                    "TrainSet_" + (GuiPreferences.Instance.NudClassifyUsingTR + 1).ToString() + "th_vectors_scaledCS.libsvm.arff");

            WekaTrainingMethods.TrainSMO(finalData);
        }
コード例 #3
0
        /// <summary>
        /// selects top IG features above 0 weight, then saves serialized data to a file.
        /// </summary>
        /// <param name="topIGFeatures"></param>
        /// <param name="data"></param>
        public static void selectIGSerialize(ref Instances data)
        {
            //run ig and get top 1000 or up to 1000 bigger than zero, from tr4
            WekaTrainingMethods.useLowLevelInformationGainFeatureSelection(data);
            GuiPreferences.Instance.setLog(Preferences.Instance.attsel.selectedAttributes().Length.ToString() + " features above zero value selected (including the Class feature)");

            //serialize (save) ALL 204k indices to file.

            //serialize (save) TOP ig indices to file.
            XMLSerializer.serializeArrayToFile <int[]>(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml", Preferences.Instance.attsel.selectedAttributes());
            GuiPreferences.Instance.setLog("saved top + All IG indices to TWO XML files (in the same order as IG gave it)");
            //int [] _trainTopIGFeatures_loaded = DeserializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml");
        }
コード例 #4
0
        public static Instances WekaPipeline_Unprocessed(libSVM_ExtendedProblem _trialProblem)
        {
            //export to libsvm file
            if (_trialProblem.samples == null)
            {
                GuiPreferences.Instance.setLog("Export Failed: Problem has no samples!");
                return(null);
            }

            string trainFileName = GuiPreferences.Instance.WorkDirectory /*+ GuiPreferences.Instance.FileName*/ + "TrainSet";


            //todo add proper named to saved files, check if null is logical at all.
            if ((_trialProblem.samples != null))
            {
                _trialProblem.Save(trainFileName + ".libsvm");
                GuiPreferences.Instance.setLog("saved Original Problem LibSVM file: " + trainFileName + ".libsvm");
            }

            //separate DS to 3rd and 4th TR
            ////example: ExecuteSelectKthVectorScript(@"TrainSet", @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\");
            KthExtractionManager.ExecuteSelectKthVectorScript(/*GuiPreferences.Instance.FileName +*/ "TrainSet", GuiPreferences.Instance.WorkDirectory);
            GuiPreferences.Instance.setLog("Created TR3 & TR4 files");

            //normalize 3rd and 4th TR files.
            NormalizationManager.ScaleTrFiles(GuiPreferences.Instance.WorkDirectory);
            GuiPreferences.Instance.setLog("Normalized TR3 & TR4 files");

            //convert tr4 and tr3 to arff + REMOVE 204801 FAKE FEATURE, THAT WAS PLACES TO MAKE SURE WE GET 204800 FEATURES IN THE ARFF FILE.
            if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS.libsvm", 204800))
            {
                GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm");
            }
            if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm", 204800))
            {
                GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm");
            }

            //---------------------------------- filter tr3 based on top 1000 from tr4 (the trick) -----------------------------
            //load TR4
            ConverterUtils.DataSource source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff");
            Instances data = source.getDataSet();

            //assign last as index.
            if (data.classIndex() == -1)
            {
                data.setClassIndex(data.numAttributes() - 1);
            }

            //if class not nominal, convert to
            if (!data.classAttribute().isNominal())
            {
                var filter = new weka.filters.unsupervised.attribute.NumericToNominal();

                filter.setOptions(weka.core.Utils.splitOptions("-R last"));
                //filter.setAttributeIndices("last");
                filter.setInputFormat(data);
                data = Filter.useFilter(data, filter);
            }

            //run ig and get top 1000 or up to 1000 bigger than zero, from tr4
            WekaTrainingMethods.useLowLevelInformationGainFeatureSelection(data);

            TrainingTesting_SharedVariables._trainTopIGFeatures = Preferences.Instance.attsel.selectedAttributes();

            //this should be done ONCE
            Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length);
            GuiPreferences.Instance.setLog("created fast vector of length " + TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString());

            //serialize (save) topIG indices to file.
            XMLSerializer.serializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml", TrainingTesting_SharedVariables._trainTopIGFeatures);
            GuiPreferences.Instance.setLog("saved IG indices to a file (in the same order as IG gave it)");
            //int [] _trainTopIGFeatures_loaded = DeserializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG_indices.xml");

            GuiPreferences.Instance.setLog(TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString() + " features above zero value selected (including the Class feature)");

            //load tr3
            source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff");
            data   = source.getDataSet();

            //filter top IG
            data = WekaTrainingMethods.useRemoveFilter(data, TrainingTesting_SharedVariables._trainTopIGFeatures, true);

            //after filtering last feature needs to be the class
            if (data.classIndex() == -1)
            {
                data.setClassIndex(data.numAttributes() - 1);
            }

            //save filtered to a file
            WekaCommonFileOperation.SaveLIBSVM(data, GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG");

            return(data);
        }
コード例 #5
0
        /// <summary>
        /// quick tests udp classification via weka's SMO
        /// </summary>
        /// <returns></returns>
        public bool testUdpWekaSMO()
        {
            if (UDPListenActive)
            {
                TrainingTesting_SharedVariables.binary.shouldStop = true;
                GuiPreferences.Instance.setLog("Stopping UDP");
            }
            else
            {
                GuiPreferences.Instance.setLog("Starting UDP");
                TrainingTesting_SharedVariables.binary.shouldStop = false;


                //GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought\6\rtp\";
                //GuiPreferences.Instance.FileName = "tirosh-";
                //GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
                //GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";

                // tirosh null movement processed for 204800 features + 1 class = 204801.

                /*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\1\rtp\";
                 * GuiPreferences.Instance.FileName = "tirosh-";
                 * GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
                 * GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120705.NullClass1_zbaseline.prt";
                 */

                // magali classification

                /*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Kozin_Magali\20121231.movement.3.imagery.1\18-classification.movement\rtp\";
                 * GuiPreferences.Instance.FileName = "tirosh-";
                 * GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
                 * GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";*/


                /// moshe sherf classification, 4 aggregated to test on 1.
                GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\15_classification\rtp\";
                GuiPreferences.Instance.FileName      = "tirosh-";
                GuiPreferences.Instance.FileType      = OriBrainLearnerCore.DataType.rawValue;
                GuiPreferences.Instance.ProtocolFile  = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";

                //read prot file
                Preferences.Instance.prot = new ProtocolManager();

                //get all files in the path with this extention
                GuiManager.getFilePaths("*.vdat");

                //update certain info
                GuiManager.updateFilePaths();

                //assigned after we know what to assign from the protocol
                //PublicMethods.setClassesLabels();
                GuiPreferences.Instance.CmbClass1Selected = 1; //left
                GuiPreferences.Instance.CmbClass2Selected = 2; //right

                //NEED TO ADD A VARIABLE FOR EVERY OPTION IN THE GUI. RAW VALUES. UNPROCESSED. MULTI CLASS. CROSS VALD, GRID, FOLDS, ETC...
                //and for every button a function!

                /*PublicMethods.clearProblem() ;
                 * PublicMethods.clearSVM();
                 * PublicMethods.clearJob();
                 * GC.Collect();*/

                //load a model for testing
                GuiPreferences.Instance.TrainType = TrainingType.Weka;
                GuiPreferences.Instance.setLog("Deserializing Model");
                WekaTrainingMethods.loadModel();

                double[][] rankedArray = XMLSerializer.DeserializeFile <double[][]>(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml");
                for (int a = 0; a < rankedArray.Length; a++)
                {
                    TrainingTesting_SharedVariables._trainTopIGFeatures[a] = Convert.ToInt32(rankedArray[a][0]);
                }

                //this should be done ONCE - move elsewhere.
                Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length);

                //GuiPreferences.Instance.FromTR = from;// 264;
                //GuiPreferences.Instance.ToTR = 100;// 264;
                Preferences.Instance.currentUDPVector = 0;
                Preferences.Instance.udp.RegisterCallBack(TrainingTesting_SharedVariables.binary.loadRawDataUsing_UDP);

                //finally load
                //PublicMethods.binary.loadRawData();

                /*
                 *  //register this function that deals with loading of the filenames
                 *  //BUT ONLY ONCE!
                 *  Preferences.Instance.pipeServer.registerEvent(PublicMethods.binary.loadRawDataUsingPipes_ReceiveData);
                 *
                 *  //PublicMethods.binary.loadRawDataUsingPipes_SendData();
                 *
                 *  //* to automatically send data ever 2s as a simulation and to test the classes.
                 *  PublicMethods.binary.loadRawDataUsingPipes_SendDataTimer();
                 */
            }

            UDPListenActive = !UDPListenActive;
            return(true);
        }
コード例 #6
0
        /// <summary>
        /// quick  the udp classification, model loading etc.
        /// </summary>
        /// <returns></returns>
        public bool testUDPSVMWRAPPER()
        {
            if (UDPListenActive)
            {
                TrainingTesting_SharedVariables.binary.shouldStop = true;
                GuiPreferences.Instance.setLog("Stopping UDP");
            }
            else
            {
                GuiPreferences.Instance.setLog("Starting UDP");
                TrainingTesting_SharedVariables.binary.shouldStop = false;
                GuiPreferences.Instance.WorkDirectory             = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought\6\rtp\";
                GuiPreferences.Instance.FileName     = "tirosh-";
                GuiPreferences.Instance.FileType     = OriBrainLearnerCore.DataType.rawValue;
                GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF_fastreact.prt";

                //read prot file
                Preferences.Instance.prot = new ProtocolManager();

                //get all files in the path with this extention
                GuiManager.getFilePaths("*.vdat");

                //update certain info
                GuiManager.updateFilePaths();

                //assigned after we know what to assign from the protocol
                //PublicMethods.setClassesLabels();
                GuiPreferences.Instance.CmbClass1Selected = 1; //left
                GuiPreferences.Instance.CmbClass2Selected = 2; //right

                //NEED TO ADD A VARIABLE FOR EVERY OPTION IN THE GUI. RAW VALUES. UNPROCESSED. MULTI CLASS. CROSS VALD, GRID, FOLDS, ETC...
                //and for every button a function!

                /*PublicMethods.clearProblem() ;
                 * PublicMethods.clearSVM();
                 * PublicMethods.clearJob();
                 * GC.Collect();*/

                //load a model for testing
                GuiPreferences.Instance.TrainType = TrainingType.TrainTestSplit;
                GuiPreferences.Instance.setLog("Loading model ori.svm - 62mb this will take a while....");
                WekaTrainingMethods.loadModel();

                //GuiPreferences.Instance.FromTR = from;// 264;
                //GuiPreferences.Instance.ToTR = 100;// 264;
                Preferences.Instance.currentUDPVector = 0;
                Preferences.Instance.udp.RegisterCallBack(TrainingTesting_SharedVariables.binary.loadRawDataUsing_UDP);

                //finally load
                //PublicMethods.binary.loadRawData();

                /*
                 *  //register this function that deals with loading of the filenames
                 *  //BUT ONLY ONCE!
                 *  Preferences.Instance.pipeServer.registerEvent(PublicMethods.binary.loadRawDataUsingPipes_ReceiveData);
                 *
                 *  //PublicMethods.binary.loadRawDataUsingPipes_SendData();
                 *
                 *  //* to automatically send data ever 2s as a simulation and to test the classes.
                 *  PublicMethods.binary.loadRawDataUsingPipes_SendDataTimer();
                 */
            }

            UDPListenActive = !UDPListenActive;
            return(true);
        }
コード例 #7
0
        /// <summary>
        /// tests if iron python works.
        /// </summary>
        public void TestIronPython()
        {
            /*IronPythonCLS ir = new IronPythonCLS();
             * var res = ir.ExecuteBusinessRules();
             * GuiPreferences.Instance.setLog(res.ToString());*/
            //ExecuteSelectKthVectorScript();



            string CsharpFileName  = @"TrainSet";
            string CsharpDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\";

            /*ExecuteSelectKthVectorScript(CsharpFileName, CsharpDirectory);
             * svm_scale_java svmscale = new svm_scale_java();
             *
             * string commandLine = "-l 0 " +
             *                   "-s " + CsharpDirectory + "TrainSet_3th_vectors_scale_paramcs.libsvm " +
             *                   "-o " + CsharpDirectory + "TrainSet_3th_vectors_scaledcs.libsvm " +
             *                           CsharpDirectory + "TrainSet_3th_vectors.libsvm";
             *
             * string[] commandArray = commandLine.Split(' ');
             * svmscale.run(commandArray);
             *
             * commandLine = "-l 0 " +
             *                   "-s " + CsharpDirectory + "TrainSet_4th_vectors_scale_paramcs.libsvm " +
             *                   "-o " + CsharpDirectory + "TrainSet_4th_vectors_scaledcs.libsvm " +
             *                           CsharpDirectory + "TrainSet_4th_vectors.libsvm";
             * commandArray = commandLine.Split(' ');
             * svmscale.run(commandArray);*/

            ////////////////////////WekaCommon.Main(null);
            ////////////////////////var source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm");

            //convert tr4 and tr3 to arff

            /*if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm"))
             *  GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm");
             * if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm"))
             *  GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm");*/

            //infogain on tr4 and get 1000 top features.

            ConverterUtils.DataSource source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff");
            Instances data = source.getDataSet();

            if (data.classIndex() == -1)
            {
                data.setClassIndex(data.numAttributes() - 1);
            }

            if (!data.classAttribute().isNominal())
            {
                var filter = new weka.filters.unsupervised.attribute.NumericToNominal();

                filter.setOptions(weka.core.Utils.splitOptions("-R last"));
                //filter.setAttributeIndices("last");
                filter.setInputFormat(data);
                data = Filter.useFilter(data, filter);
            }

            int[] topIGFeatures = Preferences.Instance.attsel.selectedAttributes();

            //load tr3
            source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff");
            data   = source.getDataSet();

            int[] invertedTopIGFeatures = new int[data.numAttributes() - topIGFeatures.Length];

            //alternative use of the filter,
            var dict     = topIGFeatures.ToDictionary(key => key, value => value);
            int position = 0;

            for (int feat = 0; feat < data.numAttributes(); feat++)
            {
                if (!dict.ContainsKey(feat))
                {
                    invertedTopIGFeatures[position] = feat;
                    position++;
                }
            }

            //filter top IG
            //data = WekaCommonMethods.useRemoveFilter(data, topIGFeatures, true);
            data = WekaTrainingMethods.useRemoveFilter(data, invertedTopIGFeatures, false);
            WekaCommonFileOperation.SaveArff(data, CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG2.libsvm1.arff");

            //train

            /*weka.classifiers.functions.SMO smo = new SMO();
             * smo.setOptions(weka.core.Utils.splitOptions(" -C 1.0 -L 0.001 -P 1.0E-12 -N 0 -V -1 -W 1 -K \"weka.classifiers.functions.supportVector.PolyKernel -C 250007 -E 1.0\""));
             * if (data.classIndex() == -1)
             *  data.setClassIndex(data.numAttributes() - 1);
             *
             *
             *
             * smo.buildClassifier(data);
             *
             *
             * //test on self should get 100%
             * weka.classifiers.Evaluation eval = new weka.classifiers.Evaluation(data);
             * eval.evaluateModel(smo, data);
             * GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false));
             *
             * //save model serialize model
             * weka.core.SerializationHelper.write(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model", smo);
             *
             * //load model deserialize model
             * smo = (weka.classifiers.functions.SMO)weka.core.SerializationHelper.read(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model");
             *
             * //test loaded model
             * eval = new weka.classifiers.Evaluation(data);
             * eval.evaluateModel(smo, data);
             * GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false));*/

            //display top IG.
            //PublicMethods.plotBrainDicomViewer();

            if (Preferences.Instance.attsel == null)
            {
                GuiPreferences.Instance.setLog("there are no ranked IG attributes or selected attr, continuing but please fix this possible bug.");
            }

            string dicomDir = CsharpDirectory;

            dicomDir = dicomDir.Substring(0, dicomDir.Length - 4) + @"master\";
            string[] files     = System.IO.Directory.GetFiles(dicomDir, "*.dcm");
            string   firstFile = files[0].Substring(files[0].LastIndexOf(@"\") + 1);

            bool thresholdOrVoxelAmount;

            if (GuiPreferences.Instance.IgSelectionType == IGType.Threshold)
            {
                thresholdOrVoxelAmount = true;
            }
            else
            {
                thresholdOrVoxelAmount = false;
            }

            Form plotForm = new DicomImageViewer.MainForm(dicomDir + firstFile, firstFile, Preferences.Instance.attsel.rankedAttributes(),
                                                          Convert.ToDouble(GuiPreferences.Instance.NudIGThreshold),
                                                          Convert.ToInt32(GuiPreferences.Instance.NudIGVoxelAmount),
                                                          thresholdOrVoxelAmount, GuiPreferences.Instance.WorkDirectory + "brain");

            plotForm.StartPosition = FormStartPosition.CenterParent;
            plotForm.ShowDialog();
            plotForm.Close();
        }
コード例 #8
0
        /// <summary>
        /// WAS A BIG BUTTON: quickloads a range of commands to test the weka pipeline
        /// loads protocol and data, processes for SMO, trick, IG, etc.. has java/python intergration
        /// 1. Trick: QuickLoad, Export to Libsvm, separate to TRs files in libsvm,  convert TR-3 and TR-4 to arff, use TR4 + IG to get 1000 features, filter TR-3 based on features from TR-4, save result to libsvm format, train using LibSvm (grid?), save model, test on training data - must get 100%, display 1000 on viewport
        /// 2. No Trick: QuickLoad, Export to Libsvm, separate to TRs files in libsvm,  convert TR-3 to arff, filter TR-3 based on 1000 top IG, save result to libsvm format, train using LibSvm (grid?), save model, test on training data - must get 100%, display 1000 on viewport
        /// </summary>
        /// <param name="from"></param>
        /// <returns></returns>
        public bool QuickProcessWekaPipeline(int from)
        {
            // --- from this point the loading data phaze begins --- //

            // tirosh null movement

            /*
             * GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\";
             * GuiPreferences.Instance.FileName = "tirosh-";
             * GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
             * GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120705.NullClass1_zbaseline.prt";
             */

            // magali classification

            /*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Kozin_Magali\20121231.movement.3.imagery.1\18-classification.movement\rtp\";
             * GuiPreferences.Instance.FileName = "tirosh-";
             * GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
             * GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";
             * */

            /// moshe sherf classification, 4 aggregated to test on 1.
            //GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\1234-5\";
            //GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\1234-5\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";

            string[] directoryList =
            {
                @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\05_classification\rtp\",
                @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\07_classification\rtp\",
                @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\09_classification\rtp\",
                @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\11_classification\rtp\"
            };
            GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";
            //GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\15_classification\rtp\";

            GuiPreferences.Instance.FileName = "tirosh-";
            GuiPreferences.Instance.FileType = OriBrainLearnerCore.DataType.rawValue;



            //read prot file
            Preferences.Instance.prot = new ProtocolManager();

            double[][] topIGFeatures = {};
            foreach (string directory in directoryList)
            {
                GuiPreferences.Instance.WorkDirectory = directory;
                //delete all files that are going to be created, in order to prevent anomaly vectors.
                string[] deleteFiles =
                {
                    "TrainSet.libsvm",
                    "TrainSet_3th_vectors.libsvm",
                    "TrainSet_3th_vectors_scale_paramCS.libsvm",
                    "TrainSet_3th_vectors_scaledCS.libsvm",
                    "TrainSet_3th_vectors_scaledCS.libsvm.arff",
                    "TrainSet_3th_vectors_scaledCS_filteredIG.arff",
                    "TrainSet_3th_vectors_scaledCS_filteredIG.model",
                    "TrainSet_3th_vectors_scaledCS_filteredIG_indices.xml",
                    "TrainSet_4th_vectors.libsvm",
                    "TrainSet_4th_vectors_scale_paramCS.libsvm",
                    "TrainSet_4th_vectors_scaledCS.libsvm",
                    "TrainSet_4th_vectors_scaledCS.libsvm.arff"
                };

                foreach (string fileName in deleteFiles)
                {
                    FileDirectoryOperations.DeleteFile(GuiPreferences.Instance.WorkDirectory + fileName);
                }

                //get all files in the path with this extention
                GuiManager.getFilePaths("*.vdat");

                //update certain info
                GuiManager.updateFilePaths();

                //assigned after we know what to assign from the protocol
                //PublicMethods.setClassesLabels();
                GuiPreferences.Instance.CmbClass1Selected = 1; //left
                GuiPreferences.Instance.CmbClass2Selected = 2; //right

                //NEED TO ADD A VARIABLE FOR EVERY OPTION IN THE GUI. RAW VALUES. UNPROCESSED. MULTI CLASS. CROSS VALD, GRID, FOLDS, ETC...
                //and for every button a function!


                //for the training set
                GuiPreferences.Instance.FromTR = from; // 264;

                //for the test set
                //GuiPreferences.Instance.FromTR = 46;

                //GuiPreferences.Instance.ToTR = 100;// 264;


                //finally load
                TrainingTesting_SharedVariables.binary.loadRawData();

                topIGFeatures = new double[][] {};
                Instances data;
                //files are loaded,thresholded,vectorized,normalized. false means that IG and training are not done here.
                if (!Training_MultiRunProcessing.ProcessSingleRunOffline(ref topIGFeatures, Preferences.Instance.ProblemOriginal))
                {
                    GuiPreferences.Instance.setLog("Samples are empty");
                }
                //++grab findl vectors and concat them
                // grab min max values for saving the median.
            }

            //create a dir that holds the final DS in C:\
            GuiPreferences.Instance.WorkDirectory = @"C:\FinalData_" + DateTime.Now.ToLongTimeString().Replace(':', '-');
            GuiPreferences.Instance.setLog(@"Creating Final Directory in: " + GuiPreferences.Instance.WorkDirectory);
            FileDirectoryOperations.CreateDirectory(GuiPreferences.Instance.WorkDirectory);
            GuiPreferences.Instance.WorkDirectory += @"\";



            //concatenate libsvm normalized and vectorized files
            FileStream fileStream;
            FileStream outputFileStream = new FileStream(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm", FileMode.CreateNew, FileAccess.Write);

            foreach (string directory in directoryList)
            {
                fileStream = new FileStream(directory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm", FileMode.Open, FileAccess.Read);
                Training_MultiRunProcessing.CopyStream(outputFileStream, fileStream);
                fileStream.Close();
            }
            outputFileStream.Close();

            //save concatenated tr3 to a file
            if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm", 204800))
            {
                GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.arff");
            }



            double[][] feature_max = new double[directoryList.Length][];
            double[][] feature_min = new double[directoryList.Length][];
            int        i           = 0;
            int        max_index   = -1;

            foreach (string directory in directoryList)
            {
                TrainingTesting_SharedVariables._svmscaleTraining.getConfigFileMinMaxValues(
                    directory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scale_paramCS.libsvm",
                    ref feature_max[i], ref feature_min[i], ref max_index);
                i++;
            }


            //calculate Mean + save new min/max param to C:\
            double[] finalFeature_max = new double[feature_max[0].Length];
            double[] finalFeature_min = new double[feature_max[0].Length];

            //create a list with enough values for the runs, in order to calculate the median
            var values_max = new List <double>(feature_max.Length);
            var values_min = new List <double>(feature_max.Length);

            for (int k = 0; k < feature_max.Length; k++)
            {
                //init zeros
                values_max.Add(0);
                values_min.Add(0);
            }

            for (int j = 0; j < feature_max[0].Length; j++)
            {
                for (int k = 0; k < feature_max.Length; k++)
                {
                    values_max[k] = feature_max[k][j];
                    values_min[k] = feature_min[k][j];
                }
                //finalFeature_max[j] = GetMedian(values_max);
                //finalFeature_min[j] = GetMedian(values_min);
                finalFeature_max[j] = values_max.Max();
                finalFeature_min[j] = values_min.Min();
            }

            TrainingTesting_SharedVariables._svmscaleTraining.saveConfigMinMax_CSharp(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scale_paramCS.libsvm", finalFeature_min, finalFeature_max, 204801, 0.0f, 1.0f);

            //todo check max index in file,
            //todo check if needs to remove 204801 from it so it doesnt effect the class.

            double[][] FinaltopIGFeatures = { };
            Instances  finalData          = Training_MultiRunProcessing.ConcatenationPipeLine("TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff", "TrainSet_4th_vectors_scaledCS.libsvm.arff");

            WekaTrainingMethods.TrainSMO(finalData);
            //save median param file

            //display top IG on dicom view
            string dicomDir = directoryList[0];

            dicomDir = dicomDir.Substring(0, dicomDir.Length - 4) + @"master\";
            string[] files     = System.IO.Directory.GetFiles(dicomDir, "*.dcm");
            string   firstFile = files[0].Substring(files[0].LastIndexOf(@"\") + 1);


            bool thresholdOrVoxelAmount;

            if (GuiPreferences.Instance.IgSelectionType == IGType.Threshold)
            {
                thresholdOrVoxelAmount = true;
            }
            else
            {
                thresholdOrVoxelAmount = false;
            }

            //NOTE final top may be empty, please remember that the IG are not at preferences.instance.attsel.selectedattributes or rankedattributes.
            Form plotForm = new DicomImageViewer.MainForm(dicomDir + firstFile, firstFile, FinaltopIGFeatures,
                                                          Convert.ToDouble(GuiPreferences.Instance.NudIGThreshold),
                                                          Convert.ToInt32(GuiPreferences.Instance.NudIGVoxelAmount),
                                                          thresholdOrVoxelAmount,
                                                          GuiPreferences.Instance.WorkDirectory + "brain");

            plotForm.StartPosition = FormStartPosition.CenterParent;
            plotForm.ShowDialog();
            plotForm.Close();

            return(true);
        }