public static Instances WekaPipeline_Unprocessed(libSVM_ExtendedProblem _trialProblem)
{
//export to libsvm file
if (_trialProblem.samples == null)
{
GuiPreferences.Instance.setLog("Export Failed: Problem has no samples!");
return(null);
}
string trainFileName = GuiPreferences.Instance.WorkDirectory /*+ GuiPreferences.Instance.FileName*/ + "TrainSet";
//todo add proper named to saved files, check if null is logical at all.
if ((_trialProblem.samples != null))
{
_trialProblem.Save(trainFileName + ".libsvm");
GuiPreferences.Instance.setLog("saved Original Problem LibSVM file: " + trainFileName + ".libsvm");
}
//separate DS to 3rd and 4th TR
////example: ExecuteSelectKthVectorScript(@"TrainSet", @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\");
KthExtractionManager.ExecuteSelectKthVectorScript(/*GuiPreferences.Instance.FileName +*/ "TrainSet", GuiPreferences.Instance.WorkDirectory);
GuiPreferences.Instance.setLog("Created TR3 & TR4 files");
//normalize 3rd and 4th TR files.
NormalizationManager.ScaleTrFiles(GuiPreferences.Instance.WorkDirectory);
GuiPreferences.Instance.setLog("Normalized TR3 & TR4 files");
//convert tr4 and tr3 to arff + REMOVE 204801 FAKE FEATURE, THAT WAS PLACES TO MAKE SURE WE GET 204800 FEATURES IN THE ARFF FILE.
if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS.libsvm", 204800))
{
GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm");
}
if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm", 204800))
{
GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm");
}
//---------------------------------- filter tr3 based on top 1000 from tr4 (the trick) -----------------------------
//load TR4
ConverterUtils.DataSource source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff");
Instances data = source.getDataSet();
//assign last as index.
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
//if class not nominal, convert to
if (!data.classAttribute().isNominal())
{
var filter = new weka.filters.unsupervised.attribute.NumericToNominal();
filter.setOptions(weka.core.Utils.splitOptions("-R last"));
//filter.setAttributeIndices("last");
filter.setInputFormat(data);
data = Filter.useFilter(data, filter);
}
//run ig and get top 1000 or up to 1000 bigger than zero, from tr4
WekaTrainingMethods.useLowLevelInformationGainFeatureSelection(data);
TrainingTesting_SharedVariables._trainTopIGFeatures = Preferences.Instance.attsel.selectedAttributes();
//this should be done ONCE
Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length);
GuiPreferences.Instance.setLog("created fast vector of length " + TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString());
//serialize (save) topIG indices to file.
XMLSerializer.serializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml", TrainingTesting_SharedVariables._trainTopIGFeatures);
GuiPreferences.Instance.setLog("saved IG indices to a file (in the same order as IG gave it)");
//int [] _trainTopIGFeatures_loaded = DeserializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG_indices.xml");
GuiPreferences.Instance.setLog(TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString() + " features above zero value selected (including the Class feature)");
//load tr3
source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff");
data = source.getDataSet();
//filter top IG
data = WekaTrainingMethods.useRemoveFilter(data, TrainingTesting_SharedVariables._trainTopIGFeatures, true);
//after filtering last feature needs to be the class
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
//save filtered to a file
WekaCommonFileOperation.SaveLIBSVM(data, GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG");
return(data);
}
/// <summary>
/// quick tests udp classification via weka's SMO
/// </summary>
/// <returns></returns>
public bool testUdpWekaSMO()
{
if (UDPListenActive)
{
TrainingTesting_SharedVariables.binary.shouldStop = true;
GuiPreferences.Instance.setLog("Stopping UDP");
}
else
{
GuiPreferences.Instance.setLog("Starting UDP");
TrainingTesting_SharedVariables.binary.shouldStop = false;
//GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought\6\rtp\";
//GuiPreferences.Instance.FileName = "tirosh-";
//GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
//GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";
// tirosh null movement processed for 204800 features + 1 class = 204801.
/*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\1\rtp\";
* GuiPreferences.Instance.FileName = "tirosh-";
* GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
* GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120705.NullClass1_zbaseline.prt";
*/
// magali classification
/*GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Kozin_Magali\20121231.movement.3.imagery.1\18-classification.movement\rtp\";
* GuiPreferences.Instance.FileName = "tirosh-";
* GuiPreferences.Instance.FileType = OriBrainLearnerCore.dataType.rawValue;
* GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";*/
/// moshe sherf classification, 4 aggregated to test on 1.
GuiPreferences.Instance.WorkDirectory = @"H:\My_Dropbox\VERE\Experiment1\Sherf_Moshe\20121010.movement.1\15_classification\rtp\";
GuiPreferences.Instance.FileName = "tirosh-";
GuiPreferences.Instance.FileType = OriBrainLearnerCore.DataType.rawValue;
GuiPreferences.Instance.ProtocolFile = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20113110.short.5th.exp.hands.legs.zscore.thought_LRF.prt";
//read prot file
Preferences.Instance.prot = new ProtocolManager();
//get all files in the path with this extention
GuiManager.getFilePaths("*.vdat");
//update certain info
GuiManager.updateFilePaths();
//assigned after we know what to assign from the protocol
//PublicMethods.setClassesLabels();
GuiPreferences.Instance.CmbClass1Selected = 1; //left
GuiPreferences.Instance.CmbClass2Selected = 2; //right
//NEED TO ADD A VARIABLE FOR EVERY OPTION IN THE GUI. RAW VALUES. UNPROCESSED. MULTI CLASS. CROSS VALD, GRID, FOLDS, ETC...
//and for every button a function!
/*PublicMethods.clearProblem() ;
* PublicMethods.clearSVM();
* PublicMethods.clearJob();
* GC.Collect();*/
//load a model for testing
GuiPreferences.Instance.TrainType = TrainingType.Weka;
GuiPreferences.Instance.setLog("Deserializing Model");
WekaTrainingMethods.loadModel();
double[][] rankedArray = XMLSerializer.DeserializeFile <double[][]>(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml");
for (int a = 0; a < rankedArray.Length; a++)
{
TrainingTesting_SharedVariables._trainTopIGFeatures[a] = Convert.ToInt32(rankedArray[a][0]);
}
//this should be done ONCE - move elsewhere.
Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length);
//GuiPreferences.Instance.FromTR = from;// 264;
//GuiPreferences.Instance.ToTR = 100;// 264;
Preferences.Instance.currentUDPVector = 0;
Preferences.Instance.udp.RegisterCallBack(TrainingTesting_SharedVariables.binary.loadRawDataUsing_UDP);
//finally load
//PublicMethods.binary.loadRawData();
/*
* //register this function that deals with loading of the filenames
* //BUT ONLY ONCE!
* Preferences.Instance.pipeServer.registerEvent(PublicMethods.binary.loadRawDataUsingPipes_ReceiveData);
*
* //PublicMethods.binary.loadRawDataUsingPipes_SendData();
*
* //* to automatically send data ever 2s as a simulation and to test the classes.
* PublicMethods.binary.loadRawDataUsingPipes_SendDataTimer();
*/
}
UDPListenActive = !UDPListenActive;
return(true);
}