//converts a libsvm file to arff
public static bool ConvertLIBSVM2ARFF(string filename, int featureToRemove)
{
try
{
var source = new ConverterUtils.DataSource(filename);
Instances insts = source.getDataSet();
if (featureToRemove > -1)
{
insts.deleteAttributeAt(featureToRemove);
}
if (insts.classIndex() == -1)
{
insts.setClassIndex(insts.numAttributes());
}
if (!insts.classAttribute().isNominal())
{
var filter = new weka.filters.unsupervised.attribute.NumericToNominal();
filter.setOptions(weka.core.Utils.splitOptions("-R last"));
filter.setInputFormat(insts);
insts = Filter.useFilter(insts, filter);
}
SaveArff(insts, filename);
GuiPreferences.Instance.setLog("Converted to arff.");
}
catch (Exception e)
{
//optional save method
/*var writer = new java.io.BufferedWriter(new java.io.FileWriter(filename+"1.arff"));
* writer.write(insts.toString());
* writer.flush();
* writer.close();*/
GuiPreferences.Instance.setLog(e.ToString() + " ");
GuiPreferences.Instance.setLog(filename);
return(false);
}
return(true);
}
/// <summary>
/// loads arff file using weka, assigns index as last feature and converts to nominal if needed
/// </summary>
/// <returns></returns>
public static Instances loadDataSetFile(string filename)
{
ConverterUtils.DataSource source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + filename);
Instances data = source.getDataSet();
//assign last as index.
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
//if class not nominal, convert to
if (!data.classAttribute().isNominal())
{
var filter = new weka.filters.unsupervised.attribute.NumericToNominal();
filter.setOptions(weka.core.Utils.splitOptions("-R last"));
//filter.setAttributeIndices("last");
filter.setInputFormat(data);
data = Filter.useFilter(data, filter);
}
return(data);
}
public static Instances WekaPipeline_Unprocessed(libSVM_ExtendedProblem _trialProblem)
{
//export to libsvm file
if (_trialProblem.samples == null)
{
GuiPreferences.Instance.setLog("Export Failed: Problem has no samples!");
return(null);
}
string trainFileName = GuiPreferences.Instance.WorkDirectory /*+ GuiPreferences.Instance.FileName*/ + "TrainSet";
//todo add proper named to saved files, check if null is logical at all.
if ((_trialProblem.samples != null))
{
_trialProblem.Save(trainFileName + ".libsvm");
GuiPreferences.Instance.setLog("saved Original Problem LibSVM file: " + trainFileName + ".libsvm");
}
//separate DS to 3rd and 4th TR
////example: ExecuteSelectKthVectorScript(@"TrainSet", @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\");
KthExtractionManager.ExecuteSelectKthVectorScript(/*GuiPreferences.Instance.FileName +*/ "TrainSet", GuiPreferences.Instance.WorkDirectory);
GuiPreferences.Instance.setLog("Created TR3 & TR4 files");
//normalize 3rd and 4th TR files.
NormalizationManager.ScaleTrFiles(GuiPreferences.Instance.WorkDirectory);
GuiPreferences.Instance.setLog("Normalized TR3 & TR4 files");
//convert tr4 and tr3 to arff + REMOVE 204801 FAKE FEATURE, THAT WAS PLACES TO MAKE SURE WE GET 204800 FEATURES IN THE ARFF FILE.
if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS.libsvm", 204800))
{
GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm");
}
if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm", 204800))
{
GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm");
}
//---------------------------------- filter tr3 based on top 1000 from tr4 (the trick) -----------------------------
//load TR4
ConverterUtils.DataSource source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff");
Instances data = source.getDataSet();
//assign last as index.
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
//if class not nominal, convert to
if (!data.classAttribute().isNominal())
{
var filter = new weka.filters.unsupervised.attribute.NumericToNominal();
filter.setOptions(weka.core.Utils.splitOptions("-R last"));
//filter.setAttributeIndices("last");
filter.setInputFormat(data);
data = Filter.useFilter(data, filter);
}
//run ig and get top 1000 or up to 1000 bigger than zero, from tr4
WekaTrainingMethods.useLowLevelInformationGainFeatureSelection(data);
TrainingTesting_SharedVariables._trainTopIGFeatures = Preferences.Instance.attsel.selectedAttributes();
//this should be done ONCE
Preferences.Instance.fastvector = RealTimeProcessing.CreateFastVector(TrainingTesting_SharedVariables._trainTopIGFeatures.Length);
GuiPreferences.Instance.setLog("created fast vector of length " + TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString());
//serialize (save) topIG indices to file.
XMLSerializer.serializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG_indices.xml", TrainingTesting_SharedVariables._trainTopIGFeatures);
GuiPreferences.Instance.setLog("saved IG indices to a file (in the same order as IG gave it)");
//int [] _trainTopIGFeatures_loaded = DeserializeArrayToFile(GuiPreferences.Instance.WorkDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG_indices.xml");
GuiPreferences.Instance.setLog(TrainingTesting_SharedVariables._trainTopIGFeatures.Length.ToString() + " features above zero value selected (including the Class feature)");
//load tr3
source = new ConverterUtils.DataSource(GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff");
data = source.getDataSet();
//filter top IG
data = WekaTrainingMethods.useRemoveFilter(data, TrainingTesting_SharedVariables._trainTopIGFeatures, true);
//after filtering last feature needs to be the class
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
//save filtered to a file
WekaCommonFileOperation.SaveLIBSVM(data, GuiPreferences.Instance.WorkDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG");
return(data);
}
public static weka.core.Instances ConvertNumericToNominal(weka.core.Instances insts)
{
weka.filters.Filter nomFilter = new weka.filters.unsupervised.attribute.NumericToNominal();
nomFilter.setInputFormat(insts);
return(weka.filters.Filter.useFilter(insts, nomFilter));
}
/// <summary>
/// tests if iron python works.
/// </summary>
public void TestIronPython()
{
/*IronPythonCLS ir = new IronPythonCLS();
* var res = ir.ExecuteBusinessRules();
* GuiPreferences.Instance.setLog(res.ToString());*/
//ExecuteSelectKthVectorScript();
string CsharpFileName = @"TrainSet";
string CsharpDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\";
/*ExecuteSelectKthVectorScript(CsharpFileName, CsharpDirectory);
* svm_scale_java svmscale = new svm_scale_java();
*
* string commandLine = "-l 0 " +
* "-s " + CsharpDirectory + "TrainSet_3th_vectors_scale_paramcs.libsvm " +
* "-o " + CsharpDirectory + "TrainSet_3th_vectors_scaledcs.libsvm " +
* CsharpDirectory + "TrainSet_3th_vectors.libsvm";
*
* string[] commandArray = commandLine.Split(' ');
* svmscale.run(commandArray);
*
* commandLine = "-l 0 " +
* "-s " + CsharpDirectory + "TrainSet_4th_vectors_scale_paramcs.libsvm " +
* "-o " + CsharpDirectory + "TrainSet_4th_vectors_scaledcs.libsvm " +
* CsharpDirectory + "TrainSet_4th_vectors.libsvm";
* commandArray = commandLine.Split(' ');
* svmscale.run(commandArray);*/
////////////////////////WekaCommon.Main(null);
////////////////////////var source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm");
//convert tr4 and tr3 to arff
/*if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm"))
* GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm");
* if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm"))
* GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm");*/
//infogain on tr4 and get 1000 top features.
ConverterUtils.DataSource source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff");
Instances data = source.getDataSet();
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
if (!data.classAttribute().isNominal())
{
var filter = new weka.filters.unsupervised.attribute.NumericToNominal();
filter.setOptions(weka.core.Utils.splitOptions("-R last"));
//filter.setAttributeIndices("last");
filter.setInputFormat(data);
data = Filter.useFilter(data, filter);
}
int[] topIGFeatures = Preferences.Instance.attsel.selectedAttributes();
//load tr3
source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff");
data = source.getDataSet();
int[] invertedTopIGFeatures = new int[data.numAttributes() - topIGFeatures.Length];
//alternative use of the filter,
var dict = topIGFeatures.ToDictionary(key => key, value => value);
int position = 0;
for (int feat = 0; feat < data.numAttributes(); feat++)
{
if (!dict.ContainsKey(feat))
{
invertedTopIGFeatures[position] = feat;
position++;
}
}
//filter top IG
//data = WekaCommonMethods.useRemoveFilter(data, topIGFeatures, true);
data = WekaTrainingMethods.useRemoveFilter(data, invertedTopIGFeatures, false);
WekaCommonFileOperation.SaveArff(data, CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG2.libsvm1.arff");
//train
/*weka.classifiers.functions.SMO smo = new SMO();
* smo.setOptions(weka.core.Utils.splitOptions(" -C 1.0 -L 0.001 -P 1.0E-12 -N 0 -V -1 -W 1 -K \"weka.classifiers.functions.supportVector.PolyKernel -C 250007 -E 1.0\""));
* if (data.classIndex() == -1)
* data.setClassIndex(data.numAttributes() - 1);
*
*
*
* smo.buildClassifier(data);
*
*
* //test on self should get 100%
* weka.classifiers.Evaluation eval = new weka.classifiers.Evaluation(data);
* eval.evaluateModel(smo, data);
* GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false));
*
* //save model serialize model
* weka.core.SerializationHelper.write(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model", smo);
*
* //load model deserialize model
* smo = (weka.classifiers.functions.SMO)weka.core.SerializationHelper.read(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model");
*
* //test loaded model
* eval = new weka.classifiers.Evaluation(data);
* eval.evaluateModel(smo, data);
* GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false));*/
//display top IG.
//PublicMethods.plotBrainDicomViewer();
if (Preferences.Instance.attsel == null)
{
GuiPreferences.Instance.setLog("there are no ranked IG attributes or selected attr, continuing but please fix this possible bug.");
}
string dicomDir = CsharpDirectory;
dicomDir = dicomDir.Substring(0, dicomDir.Length - 4) + @"master\";
string[] files = System.IO.Directory.GetFiles(dicomDir, "*.dcm");
string firstFile = files[0].Substring(files[0].LastIndexOf(@"\") + 1);
bool thresholdOrVoxelAmount;
if (GuiPreferences.Instance.IgSelectionType == IGType.Threshold)
{
thresholdOrVoxelAmount = true;
}
else
{
thresholdOrVoxelAmount = false;
}
Form plotForm = new DicomImageViewer.MainForm(dicomDir + firstFile, firstFile, Preferences.Instance.attsel.rankedAttributes(),
Convert.ToDouble(GuiPreferences.Instance.NudIGThreshold),
Convert.ToInt32(GuiPreferences.Instance.NudIGVoxelAmount),
thresholdOrVoxelAmount, GuiPreferences.Instance.WorkDirectory + "brain");
plotForm.StartPosition = FormStartPosition.CenterParent;
plotForm.ShowDialog();
plotForm.Close();
}
