public static Instances WekaPipeline_Unprocessed(libSVM_ExtendedProblem _trialProblem) { //export to libsvm file if (_trialProblem.samples == null) { GuiPreferences.Instance.setLog("Export Failed: Problem has no samples!"); return(null); } string trainFileName = GuiPreferences.Instance.WorkDirectory /*+ GuiPreferences.Instance.FileName*/ + "TrainSet"; //todo add proper named to saved files, check if null is logical at all. if ((_trialProblem.samples != null)) { _trialProblem.Save(trainFileName + ".libsvm"); GuiPreferences.Instance.setLog("saved Original Problem LibSVM file: " + trainFileName + ".libsvm"); } //separate DS to 3rd and 4th TR ////example: ExecuteSelectKthVectorScript(@"TrainSet", @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\"); KthExtractionManager.ExecuteSelectKthVectorScript(/*GuiPreferences.Instance.FileName +*/ "TrainSet", GuiPreferences.Instance.WorkDirectory); GuiPreferences.Instance.setLog("Created TR3 & TR4 files"); //normalize 3rd and 4th TR files. NormalizationManager.ScaleTrFiles(GuiPreferences.Instance.WorkDirectory); GuiPreferences.Instance.setLog("Normalized TR3 & TR4 files"); //convert tr4 and tr3 to arff + REMOVE 204801 FAKE FEATURE, THAT WAS PLACES TO MAKE SURE WE GET 204800 FEATURES IN THE ARFF FILE. if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS.libsvm", 204800)) { GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm"); } if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm", 204800)) { GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm"); } //---------------------------------- filter tr3 based on top 1000 from tr4 (the trick) ----------------------------- //load TR4 ConverterUtils.DataSource source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff"); Instances data = source.getDataSet(); //assign last as index. if (data.classIndex() == -1) { data.setClassIndex(data.numAttributes() - 1); } //if class not nominal, convert to if (!data.classAttribute().isNominal()) { var filter = new weka.filters.unsupervised.attribute.NumericToNominal(); filter.setOptions(weka.core.Utils.splitOptions("-R last")); //filter.setAttributeIndices("last"); filter.setInputFormat(data); data = Filter.useFilter(data, filter); } //run ig and get top 1000 or up to 1000 bigger than zero, from tr4 WekaTrainingMethods.useLowLevelInformationGainFeatureSelection(data); TrainingTesting_SharedVariables._trainTopIGFeatures = Preferences.Instance.attsel.selectedAttributes(); //this should be done ONCE Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length); GuiPreferences.Instance.setLog("created fast vector of length " + TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString()); //serialize (save) topIG indices to file. XMLSerializer.serializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml", TrainingTesting_SharedVariables._trainTopIGFeatures); GuiPreferences.Instance.setLog("saved IG indices to a file (in the same order as IG gave it)"); //int [] _trainTopIGFeatures_loaded = DeserializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG_indices.xml"); GuiPreferences.Instance.setLog(TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString() + " features above zero value selected (including the Class feature)"); //load tr3 source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff"); data = source.getDataSet(); //filter top IG data = WekaTrainingMethods.useRemoveFilter(data, TrainingTesting_SharedVariables._trainTopIGFeatures, true); //after filtering last feature needs to be the class if (data.classIndex() == -1) { data.setClassIndex(data.numAttributes() - 1); } //save filtered to a file WekaCommonFileOperation.SaveLIBSVM(data, GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG"); return(data); }
/// <summary> /// quick tests udp classification via weka's SMO /// </summary> /// <returns></returns> public bool testUdpWekaSMO() { if (UDPListenActive) { TrainingTesting_SharedVariables.binary.shouldStop = true; GuiPreferences.Instance.setLog("Stopping UDP"); } else { GuiPreferences.Instance.setLog("Starting UDP"); TrainingTesting_SharedVariables.binary.shouldStop = false; //GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought\6\rtp\"; //GuiPreferences.Instance.FileName = "tirosh-"; //GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue; //GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt"; // tirosh null movement processed for 204800 features + 1 class = 204801. /*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\1\rtp\"; * GuiPreferences.Instance.FileName = "tirosh-"; * GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue; * GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120705.NullClass1_zbaseline.prt"; */ // magali classification /*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Kozin_Magali\20121231.movement.3.imagery.1\18-classification.movement\rtp\"; * GuiPreferences.Instance.FileName = "tirosh-"; * GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue; * GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";*/ /// moshe sherf classification, 4 aggregated to test on 1. GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\15_classification\rtp\"; GuiPreferences.Instance.FileName = "tirosh-"; GuiPreferences.Instance.FileType = OriBrainLearnerCore.DataType.rawValue; GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt"; //read prot file Preferences.Instance.prot = new ProtocolManager(); //get all files in the path with this extention GuiManager.getFilePaths("*.vdat"); //update certain info GuiManager.updateFilePaths(); //assigned after we know what to assign from the protocol //PublicMethods.setClassesLabels(); GuiPreferences.Instance.CmbClass1Selected = 1; //left GuiPreferences.Instance.CmbClass2Selected = 2; //right //NEED TO ADD A VARIABLE FOR EVERY OPTION IN THE GUI. RAW VALUES. UNPROCESSED. MULTI CLASS. CROSS VALD, GRID, FOLDS, ETC... //and for every button a function! /*PublicMethods.clearProblem() ; * PublicMethods.clearSVM(); * PublicMethods.clearJob(); * GC.Collect();*/ //load a model for testing GuiPreferences.Instance.TrainType = TrainingType.Weka; GuiPreferences.Instance.setLog("Deserializing Model"); WekaTrainingMethods.loadModel(); double[][] rankedArray = XMLSerializer.DeserializeFile <double[][]>(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml"); for (int a = 0; a < rankedArray.Length; a++) { TrainingTesting_SharedVariables._trainTopIGFeatures[a] = Convert.ToInt32(rankedArray[a][0]); } //this should be done ONCE - move elsewhere. Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length); //GuiPreferences.Instance.FromTR = from;// 264; //GuiPreferences.Instance.ToTR = 100;// 264; Preferences.Instance.currentUDPVector = 0; Preferences.Instance.udp.RegisterCallBack(TrainingTesting_SharedVariables.binary.loadRawDataUsing_UDP); //finally load //PublicMethods.binary.loadRawData(); /* * //register this function that deals with loading of the filenames * //BUT ONLY ONCE! * Preferences.Instance.pipeServer.registerEvent(PublicMethods.binary.loadRawDataUsingPipes_ReceiveData); * * //PublicMethods.binary.loadRawDataUsingPipes_SendData(); * * //* to automatically send data ever 2s as a simulation and to test the classes. * PublicMethods.binary.loadRawDataUsingPipes_SendDataTimer(); */ } UDPListenActive = !UDPListenActive; return(true); }