//the final normalized and concatenated file doesnt need to be vectorized + normalized again.
// if first line "instances data = " is 4 and the rest of the lines are 3, then the trick is used.
// if all lines are 3 then only tr3 is used
// if all lines except the saveArff is tr4 then it will use tr4 data and save to a filename with "3" in it, at the moment this is done for compatibility when loading this file in real-time
public static Instances ConcatenationPipeLine(string filenameTR3, string filenameTR4)
{
//filter tr3 based on top 1000 from tr4 (the trick)
//load TR4 !!! NOTE: trick changed from tr4 to 3 because i didnt see any increase in % in real time.. + this line can be removed to speed up things.
Instances data;
if (GuiPreferences.Instance.CbPeekHigherTRsIGChecked)
{
GuiPreferences.Instance.setLog("using final dataset: " + filenameTR4);
data = WekaTrainingMethods.loadDataSetFile(filenameTR4); // peeking at a higher TR's IG values (trick)
}
else
{
GuiPreferences.Instance.setLog("using final dataset: " + filenameTR3);
data = WekaTrainingMethods.loadDataSetFile(filenameTR3); // no peeking (no trick)
}
//select 1000 IG values, serialize to file
WekaTrainingMethods.selectIGSerialize(ref data);
//load tr3
data = WekaTrainingMethods.loadDataSetFile(filenameTR3);
//filter top IG
data = WekaTrainingMethods.useRemoveFilter(data, Preferences.Instance.attsel.selectedAttributes(), true);
//save filtered tr3 to a file
WekaCommonFileOperation.SaveArff(data, GuiPreferences.Instance.WorkDirectory + filenameTR3 + "_filteredIG.arff");
WekaCommonFileOperation.SaveCSV(data, GuiPreferences.Instance.WorkDirectory + filenameTR3 + "_filteredIG_CSV.arff");
return(data);
}
/// <summary>
/// tests if iron python works.
/// </summary>
public void TestIronPython()
{
/*IronPythonCLS ir = new IronPythonCLS();
* var res = ir.ExecuteBusinessRules();
* GuiPreferences.Instance.setLog(res.ToString());*/
//ExecuteSelectKthVectorScript();
string CsharpFileName = @"TrainSet";
string CsharpDirectory = @"H:\My_Dropbox\VERE\MRI_data\Tirosh\20120508.Rapid+NullClass.day2\4\rtp\";
/*ExecuteSelectKthVectorScript(CsharpFileName, CsharpDirectory);
* svm_scale_java svmscale = new svm_scale_java();
*
* string commandLine = "-l 0 " +
* "-s " + CsharpDirectory + "TrainSet_3th_vectors_scale_paramcs.libsvm " +
* "-o " + CsharpDirectory + "TrainSet_3th_vectors_scaledcs.libsvm " +
* CsharpDirectory + "TrainSet_3th_vectors.libsvm";
*
* string[] commandArray = commandLine.Split(' ');
* svmscale.run(commandArray);
*
* commandLine = "-l 0 " +
* "-s " + CsharpDirectory + "TrainSet_4th_vectors_scale_paramcs.libsvm " +
* "-o " + CsharpDirectory + "TrainSet_4th_vectors_scaledcs.libsvm " +
* CsharpDirectory + "TrainSet_4th_vectors.libsvm";
* commandArray = commandLine.Split(' ');
* svmscale.run(commandArray);*/
////////////////////////WekaCommon.Main(null);
////////////////////////var source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm");
//convert tr4 and tr3 to arff
/*if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_3th_vectors_scaledCS.libsvm"))
* GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_3th_vectors_scaledCS.libsvm");
* if (WekaCommonFileOperation.ConvertLIBSVM2ARFF(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm"))
* GuiPreferences.Instance.setLog("Converted to ARFF: TrainSet_4th_vectors_scaledCS.libsvm");*/
//infogain on tr4 and get 1000 top features.
ConverterUtils.DataSource source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_4th_vectors_scaledCS.libsvm.arff");
Instances data = source.getDataSet();
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
if (!data.classAttribute().isNominal())
{
var filter = new weka.filters.unsupervised.attribute.NumericToNominal();
filter.setOptions(weka.core.Utils.splitOptions("-R last"));
//filter.setAttributeIndices("last");
filter.setInputFormat(data);
data = Filter.useFilter(data, filter);
}
int[] topIGFeatures = Preferences.Instance.attsel.selectedAttributes();
//load tr3
source = new ConverterUtils.DataSource(CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS.libsvm.arff");
data = source.getDataSet();
int[] invertedTopIGFeatures = new int[data.numAttributes() - topIGFeatures.Length];
//alternative use of the filter,
var dict = topIGFeatures.ToDictionary(key => key, value => value);
int position = 0;
for (int feat = 0; feat < data.numAttributes(); feat++)
{
if (!dict.ContainsKey(feat))
{
invertedTopIGFeatures[position] = feat;
position++;
}
}
//filter top IG
//data = WekaCommonMethods.useRemoveFilter(data, topIGFeatures, true);
data = WekaTrainingMethods.useRemoveFilter(data, invertedTopIGFeatures, false);
WekaCommonFileOperation.SaveArff(data, CsharpDirectory + "TrainSet_" + GuiPreferences.Instance.NudClassifyUsingTR.ToString() + "th_vectors_scaledCS_filteredIG2.libsvm1.arff");
//train
/*weka.classifiers.functions.SMO smo = new SMO();
* smo.setOptions(weka.core.Utils.splitOptions(" -C 1.0 -L 0.001 -P 1.0E-12 -N 0 -V -1 -W 1 -K \"weka.classifiers.functions.supportVector.PolyKernel -C 250007 -E 1.0\""));
* if (data.classIndex() == -1)
* data.setClassIndex(data.numAttributes() - 1);
*
*
*
* smo.buildClassifier(data);
*
*
* //test on self should get 100%
* weka.classifiers.Evaluation eval = new weka.classifiers.Evaluation(data);
* eval.evaluateModel(smo, data);
* GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false));
*
* //save model serialize model
* weka.core.SerializationHelper.write(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model", smo);
*
* //load model deserialize model
* smo = (weka.classifiers.functions.SMO)weka.core.SerializationHelper.read(CsharpDirectory + "TrainSet_3th_vectors_scaledCS_filteredIG.libsvm.arff.model");
*
* //test loaded model
* eval = new weka.classifiers.Evaluation(data);
* eval.evaluateModel(smo, data);
* GuiPreferences.Instance.setLog(eval.toSummaryString("\nResults\n======\n", false));*/
//display top IG.
//PublicMethods.plotBrainDicomViewer();
if (Preferences.Instance.attsel == null)
{
GuiPreferences.Instance.setLog("there are no ranked IG attributes or selected attr, continuing but please fix this possible bug.");
}
string dicomDir = CsharpDirectory;
dicomDir = dicomDir.Substring(0, dicomDir.Length - 4) + @"master\";
string[] files = System.IO.Directory.GetFiles(dicomDir, "*.dcm");
string firstFile = files[0].Substring(files[0].LastIndexOf(@"\") + 1);
bool thresholdOrVoxelAmount;
if (GuiPreferences.Instance.IgSelectionType == IGType.Threshold)
{
thresholdOrVoxelAmount = true;
}
else
{
thresholdOrVoxelAmount = false;
}
Form plotForm = new DicomImageViewer.MainForm(dicomDir + firstFile, firstFile, Preferences.Instance.attsel.rankedAttributes(),
Convert.ToDouble(GuiPreferences.Instance.NudIGThreshold),
Convert.ToInt32(GuiPreferences.Instance.NudIGVoxelAmount),
thresholdOrVoxelAmount, GuiPreferences.Instance.WorkDirectory + "brain");
plotForm.StartPosition = FormStartPosition.CenterParent;
plotForm.ShowDialog();
plotForm.Close();
}
