public static void WekaTrainingPipelineForMultiRuns()
{
//todo check max index in file,
//todo check if needs to remove 204801 from it so it doesnt effect the class.
Instances finalData = Training_MultiRunProcessing.ConcatenationPipeLine("TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff",
"TrainSet_" + (GuiPreferences.Instance.NudClassifyUsingTR + 1).ToString() + "th_vectors_scaledCS.libsvm.arff");
WekaTrainingMethods.TrainSMO(finalData);
}
/// <summary>
/// WAS A BIG BUTTON: quickloads a range of commands to test the weka pipeline
/// loads protocol and data, processes for SMO, trick, IG, etc.. has java/python intergration
/// 1. Trick: QuickLoad, Export to Libsvm, separate to TRs files in libsvm, convert TR-3 and TR-4 to arff, use TR4 + IG to get 1000 features, filter TR-3 based on features from TR-4, save result to libsvm format, train using LibSvm (grid?), save model, test on training data - must get 100%, display 1000 on viewport
/// 2. No Trick: QuickLoad, Export to Libsvm, separate to TRs files in libsvm, convert TR-3 to arff, filter TR-3 based on 1000 top IG, save result to libsvm format, train using LibSvm (grid?), save model, test on training data - must get 100%, display 1000 on viewport
/// </summary>
/// <param name="from"></param>
/// <returns></returns>
public bool QuickProcessWekaPipeline(int from)
{
// --- from this point the loading data phaze begins --- //
// tirosh null movement
/*
* GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\";
* GuiPreferences.Instance.FileName = "tirosh-";
* GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
* GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120705.NullClass1_zbaseline.prt";
*/
// magali classification
/*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Kozin_Magali\20121231.movement.3.imagery.1\18-classification.movement\rtp\";
* GuiPreferences.Instance.FileName = "tirosh-";
* GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
* GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";
* */
/// moshe sherf classification, 4 aggregated to test on 1.
//GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\1234-5\";
//GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\1234-5\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";
string[] directoryList =
{
@"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\05_classification\rtp\",
@"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\07_classification\rtp\",
@"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\09_classification\rtp\",
@"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\11_classification\rtp\"
};
GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";
//GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\15_classification\rtp\";
GuiPreferences.Instance.FileName = "tirosh-";
GuiPreferences.Instance.FileType = OriBrainLearnerCore.DataType.rawValue;
//read prot file
Preferences.Instance.prot = new ProtocolManager();
double[][] topIGFeatures = {};
foreach (string directory in directoryList)
{
GuiPreferences.Instance.WorkDirectory = directory;
//delete all files that are going to be created, in order to prevent anomaly vectors.
string[] deleteFiles =
{
"TrainSet.libsvm",
"TrainSet_3th_vectors.libsvm",
"TrainSet_3th_vectors_scale_paramCS.libsvm",
"TrainSet_3th_vectors_scaledCS.libsvm",
"TrainSet_3th_vectors_scaledCS.libsvm.arff",
"TrainSet_3th_vectors_scaledCS_filteredIG.arff",
"TrainSet_3th_vectors_scaledCS_filteredIG.model",
"TrainSet_3th_vectors_scaledCS_filteredIG_indices.xml",
"TrainSet_4th_vectors.libsvm",
"TrainSet_4th_vectors_scale_paramCS.libsvm",
"TrainSet_4th_vectors_scaledCS.libsvm",
"TrainSet_4th_vectors_scaledCS.libsvm.arff"
};
foreach (string fileName in deleteFiles)
{
FileDirectoryOperations.DeleteFile(GuiPreferences.Instance.WorkDirectory + fileName);
}
//get all files in the path with this extention
GuiManager.getFilePaths("*.vdat");
//update certain info
GuiManager.updateFilePaths();
//assigned after we know what to assign from the protocol
//PublicMethods.setClassesLabels();
GuiPreferences.Instance.CmbClass1Selected = 1; //left
GuiPreferences.Instance.CmbClass2Selected = 2; //right
//NEED TO ADD A VARIABLE FOR EVERY OPTION IN THE GUI. RAW VALUES. UNPROCESSED. MULTI CLASS. CROSS VALD, GRID, FOLDS, ETC...
//and for every button a function!
//for the training set
GuiPreferences.Instance.FromTR = from; // 264;
//for the test set
//GuiPreferences.Instance.FromTR = 46;
//GuiPreferences.Instance.ToTR = 100;// 264;
//finally load
TrainingTesting_SharedVariables.binary.loadRawData();
topIGFeatures = new double[][] {};
Instances data;
//files are loaded,thresholded,vectorized,normalized. false means that IG and training are not done here.
if (!Training_MultiRunProcessing.ProcessSingleRunOffline(ref topIGFeatures, Preferences.Instance.ProblemOriginal))
{
GuiPreferences.Instance.setLog("Samples are empty");
}
//++grab findl vectors and concat them
// grab min max values for saving the median.
}
//create a dir that holds the final DS in C:\
GuiPreferences.Instance.WorkDirectory = @"C:\FinalData_" + DateTime.Now.ToLongTimeString().Replace(':', '-');
GuiPreferences.Instance.setLog(@"Creating Final Directory in: " + GuiPreferences.Instance.WorkDirectory);
FileDirectoryOperations.CreateDirectory(GuiPreferences.Instance.WorkDirectory);
GuiPreferences.Instance.WorkDirectory += @"\";
//concatenate libsvm normalized and vectorized files
FileStream fileStream;
FileStream outputFileStream = new FileStream(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm", FileMode.CreateNew, FileAccess.Write);
foreach (string directory in directoryList)
{
fileStream = new FileStream(directory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm", FileMode.Open, FileAccess.Read);
Training_MultiRunProcessing.CopyStream(outputFileStream, fileStream);
fileStream.Close();
}
outputFileStream.Close();
//save concatenated tr3 to a file
if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm", 204800))
{
GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.arff");
}
double[][] feature_max = new double[directoryList.Length][];
double[][] feature_min = new double[directoryList.Length][];
int i = 0;
int max_index = -1;
foreach (string directory in directoryList)
{
TrainingTesting_SharedVariables._svmscaleTraining.getConfigFileMinMaxValues(
directory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scale_paramCS.libsvm",
ref feature_max[i], ref feature_min[i], ref max_index);
i++;
}
//calculate Mean + save new min/max param to C:\
double[] finalFeature_max = new double[feature_max[0].Length];
double[] finalFeature_min = new double[feature_max[0].Length];
//create a list with enough values for the runs, in order to calculate the median
var values_max = new List <double>(feature_max.Length);
var values_min = new List <double>(feature_max.Length);
for (int k = 0; k < feature_max.Length; k++)
{
//init zeros
values_max.Add(0);
values_min.Add(0);
}
for (int j = 0; j < feature_max[0].Length; j++)
{
for (int k = 0; k < feature_max.Length; k++)
{
values_max[k] = feature_max[k][j];
values_min[k] = feature_min[k][j];
}
//finalFeature_max[j] = GetMedian(values_max);
//finalFeature_min[j] = GetMedian(values_min);
finalFeature_max[j] = values_max.Max();
finalFeature_min[j] = values_min.Min();
}
TrainingTesting_SharedVariables._svmscaleTraining.saveConfigMinMax_CSharp(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scale_paramCS.libsvm", finalFeature_min, finalFeature_max, 204801, 0.0f, 1.0f);
//todo check max index in file,
//todo check if needs to remove 204801 from it so it doesnt effect the class.
double[][] FinaltopIGFeatures = { };
Instances finalData = Training_MultiRunProcessing.ConcatenationPipeLine("TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff", "TrainSet_4th_vectors_scaledCS.libsvm.arff");
WekaTrainingMethods.TrainSMO(finalData);
//save median param file
//display top IG on dicom view
string dicomDir = directoryList[0];
dicomDir = dicomDir.Substring(0, dicomDir.Length - 4) + @"master\";
string[] files = System.IO.Directory.GetFiles(dicomDir, "*.dcm");
string firstFile = files[0].Substring(files[0].LastIndexOf(@"\") + 1);
bool thresholdOrVoxelAmount;
if (GuiPreferences.Instance.IgSelectionType == IGType.Threshold)
{
thresholdOrVoxelAmount = true;
}
else
{
thresholdOrVoxelAmount = false;
}
//NOTE final top may be empty, please remember that the IG are not at preferences.instance.attsel.selectedattributes or rankedattributes.
Form plotForm = new DicomImageViewer.MainForm(dicomDir + firstFile, firstFile, FinaltopIGFeatures,
Convert.ToDouble(GuiPreferences.Instance.NudIGThreshold),
Convert.ToInt32(GuiPreferences.Instance.NudIGVoxelAmount),
thresholdOrVoxelAmount,
GuiPreferences.Instance.WorkDirectory + "brain");
plotForm.StartPosition = FormStartPosition.CenterParent;
plotForm.ShowDialog();
plotForm.Close();
return(true);
}