//the final normalized and concatenated file doesnt need to be vectorized + normalized again. // if first line "instances data = " is 4 and the rest of the lines are 3, then the trick is used. // if all lines are 3 then only tr3 is used // if all lines except the saveArff is tr4 then it will use tr4 data and save to a filename with "3" in it, at the moment this is done for compatibility when loading this file in real-time public static Instances ConcatenationPipeLine(string filenameTR3, string filenameTR4) { //filter tr3 based on top 1000 from tr4 (the trick) //load TR4 !!! NOTE: trick changed from tr4 to 3 because i didnt see any increase in % in real time.. + this line can be removed to speed up things. Instances data; if (GuiPreferences.Instance.CbPeekHigherTRsIGChecked) { GuiPreferences.Instance.setLog("using final dataset: " + filenameTR4); data = WekaTrainingMethods.loadDataSetFile(filenameTR4); // peeking at a higher TR's IG values (trick) } else { GuiPreferences.Instance.setLog("using final dataset: " + filenameTR3); data = WekaTrainingMethods.loadDataSetFile(filenameTR3); // no peeking (no trick) } //select 1000 IG values, serialize to file WekaTrainingMethods.selectIGSerialize(ref data); //load tr3 data = WekaTrainingMethods.loadDataSetFile(filenameTR3); //filter top IG data = WekaTrainingMethods.useRemoveFilter(data, Preferences.Instance.attsel.selectedAttributes(), true); //save filtered tr3 to a file WekaCommonFileOperation.SaveArff(data, GuiPreferences.Instance.WorkDirectory + filenameTR3 + "_filteredIG.arff"); WekaCommonFileOperation.SaveCSV(data, GuiPreferences.Instance.WorkDirectory + filenameTR3 + "_filteredIG_CSV.arff"); return(data); }
public static Instances WekaPipeline_Unprocessed(libSVM_ExtendedProblem _trialProblem) { //export to libsvm file if (_trialProblem.samples == null) { GuiPreferences.Instance.setLog("Export Failed: Problem has no samples!"); return(null); } string trainFileName = GuiPreferences.Instance.WorkDirectory /*+ GuiPreferences.Instance.FileName*/ + "TrainSet"; //todo add proper named to saved files, check if null is logical at all. if ((_trialProblem.samples != null)) { _trialProblem.Save(trainFileName + ".libsvm"); GuiPreferences.Instance.setLog("saved Original Problem LibSVM file: " + trainFileName + ".libsvm"); } //separate DS to 3rd and 4th TR ////example: ExecuteSelectKthVectorScript(@"TrainSet", @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\"); KthExtractionManager.ExecuteSelectKthVectorScript(/*GuiPreferences.Instance.FileName +*/ "TrainSet", GuiPreferences.Instance.WorkDirectory); GuiPreferences.Instance.setLog("Created TR3 & TR4 files"); //normalize 3rd and 4th TR files. NormalizationManager.ScaleTrFiles(GuiPreferences.Instance.WorkDirectory); GuiPreferences.Instance.setLog("Normalized TR3 & TR4 files"); //convert tr4 and tr3 to arff + REMOVE 204801 FAKE FEATURE, THAT WAS PLACES TO MAKE SURE WE GET 204800 FEATURES IN THE ARFF FILE. if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS.libsvm", 204800)) { GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm"); } if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm", 204800)) { GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm"); } //---------------------------------- filter tr3 based on top 1000 from tr4 (the trick) ----------------------------- //load TR4 ConverterUtils.DataSource source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff"); Instances data = source.getDataSet(); //assign last as index. if (data.classIndex() == -1) { data.setClassIndex(data.numAttributes() - 1); } //if class not nominal, convert to if (!data.classAttribute().isNominal()) { var filter = new weka.filters.unsupervised.attribute.NumericToNominal(); filter.setOptions(weka.core.Utils.splitOptions("-R last")); //filter.setAttributeIndices("last"); filter.setInputFormat(data); data = Filter.useFilter(data, filter); } //run ig and get top 1000 or up to 1000 bigger than zero, from tr4 WekaTrainingMethods.useLowLevelInformationGainFeatureSelection(data); TrainingTesting_SharedVariables._trainTopIGFeatures = Preferences.Instance.attsel.selectedAttributes(); //this should be done ONCE Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length); GuiPreferences.Instance.setLog("created fast vector of length " + TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString()); //serialize (save) topIG indices to file. XMLSerializer.serializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml", TrainingTesting_SharedVariables._trainTopIGFeatures); GuiPreferences.Instance.setLog("saved IG indices to a file (in the same order as IG gave it)"); //int [] _trainTopIGFeatures_loaded = DeserializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG_indices.xml"); GuiPreferences.Instance.setLog(TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString() + " features above zero value selected (including the Class feature)"); //load tr3 source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff"); data = source.getDataSet(); //filter top IG data = WekaTrainingMethods.useRemoveFilter(data, TrainingTesting_SharedVariables._trainTopIGFeatures, true); //after filtering last feature needs to be the class if (data.classIndex() == -1) { data.setClassIndex(data.numAttributes() - 1); } //save filtered to a file WekaCommonFileOperation.SaveLIBSVM(data, GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG"); return(data); }
/// <summary> /// tests if iron python works. /// </summary> public void TestIronPython() { /*IronPythonCLS ir = new IronPythonCLS(); * var res = ir.ExecuteBusinessRules(); * GuiPreferences.Instance.setLog(res.ToString());*/ //ExecuteSelectKthVectorScript(); string CsharpFileName = @"TrainSet"; string CsharpDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\"; /*ExecuteSelectKthVectorScript(CsharpFileName, CsharpDirectory); * svm_scale_java svmscale = new svm_scale_java(); * * string commandLine = "-l 0 " + * "-s " + CsharpDirectory + "TrainSet_3th_vectors_scale_paramcs.libsvm " + * "-o " + CsharpDirectory + "TrainSet_3th_vectors_scaledcs.libsvm " + * CsharpDirectory + "TrainSet_3th_vectors.libsvm"; * * string[] commandArray = commandLine.Split(' '); * svmscale.run(commandArray); * * commandLine = "-l 0 " + * "-s " + CsharpDirectory + "TrainSet_4th_vectors_scale_paramcs.libsvm " + * "-o " + CsharpDirectory + "TrainSet_4th_vectors_scaledcs.libsvm " + * CsharpDirectory + "TrainSet_4th_vectors.libsvm"; * commandArray = commandLine.Split(' '); * svmscale.run(commandArray);*/ ////////////////////////WekaCommon.Main(null); ////////////////////////var source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm"); //convert tr4 and tr3 to arff /*if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm")) * GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm"); * if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm")) * GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm");*/ //infogain on tr4 and get 1000 top features. ConverterUtils.DataSource source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff"); Instances data = source.getDataSet(); if (data.classIndex() == -1) { data.setClassIndex(data.numAttributes() - 1); } if (!data.classAttribute().isNominal()) { var filter = new weka.filters.unsupervised.attribute.NumericToNominal(); filter.setOptions(weka.core.Utils.splitOptions("-R last")); //filter.setAttributeIndices("last"); filter.setInputFormat(data); data = Filter.useFilter(data, filter); } int[] topIGFeatures = Preferences.Instance.attsel.selectedAttributes(); //load tr3 source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff"); data = source.getDataSet(); int[] invertedTopIGFeatures = new int[data.numAttributes() - topIGFeatures.Length]; //alternative use of the filter, var dict = topIGFeatures.ToDictionary(key => key, value => value); int position = 0; for (int feat = 0; feat < data.numAttributes(); feat++) { if (!dict.ContainsKey(feat)) { invertedTopIGFeatures[position] = feat; position++; } } //filter top IG //data = WekaCommonMethods.useRemoveFilter(data, topIGFeatures, true); data = WekaTrainingMethods.useRemoveFilter(data, invertedTopIGFeatures, false); WekaCommonFileOperation.SaveArff(data, CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG2.libsvm1.arff"); //train /*weka.classifiers.functions.SMO smo = new SMO(); * smo.setOptions(weka.core.Utils.splitOptions(" -C 1.0 -L 0.001 -P 1.0E-12 -N 0 -V -1 -W 1 -K \"weka.classifiers.functions.supportVector.PolyKernel -C 250007 -E 1.0\"")); * if (data.classIndex() == -1) * data.setClassIndex(data.numAttributes() - 1); * * * * smo.buildClassifier(data); * * * //test on self should get 100% * weka.classifiers.Evaluation eval = new weka.classifiers.Evaluation(data); * eval.evaluateModel(smo, data); * GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false)); * * //save model serialize model * weka.core.SerializationHelper.write(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model", smo); * * //load model deserialize model * smo = (weka.classifiers.functions.SMO)weka.core.SerializationHelper.read(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model"); * * //test loaded model * eval = new weka.classifiers.Evaluation(data); * eval.evaluateModel(smo, data); * GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false));*/ //display top IG. //PublicMethods.plotBrainDicomViewer(); if (Preferences.Instance.attsel == null) { GuiPreferences.Instance.setLog("there are no ranked IG attributes or selected attr, continuing but please fix this possible bug."); } string dicomDir = CsharpDirectory; dicomDir = dicomDir.Substring(0, dicomDir.Length - 4) + @"master\"; string[] files = System.IO.Directory.GetFiles(dicomDir, "*.dcm"); string firstFile = files[0].Substring(files[0].LastIndexOf(@"\") + 1); bool thresholdOrVoxelAmount; if (GuiPreferences.Instance.IgSelectionType == IGType.Threshold) { thresholdOrVoxelAmount = true; } else { thresholdOrVoxelAmount = false; } Form plotForm = new DicomImageViewer.MainForm(dicomDir + firstFile, firstFile, Preferences.Instance.attsel.rankedAttributes(), Convert.ToDouble(GuiPreferences.Instance.NudIGThreshold), Convert.ToInt32(GuiPreferences.Instance.NudIGVoxelAmount), thresholdOrVoxelAmount, GuiPreferences.Instance.WorkDirectory + "brain"); plotForm.StartPosition = FormStartPosition.CenterParent; plotForm.ShowDialog(); plotForm.Close(); }