public String PrintClassifierTestReport()
{
try {
Debug.Log("Classifier: Number of instances: " + playerData.numInstances());
weka.classifiers.Evaluation eval = new weka.classifiers.Evaluation(playerData);
eval.crossValidateModel(classifier, playerData, 10, new java.util.Random(1));
Debug.Log(eval.toSummaryString("\nClassifier: Cross Validate Results: \n======\n", false));
return(eval.toSummaryString("\nResults\n======\n", false));
}
catch (java.lang.Exception ex)
{
Debug.LogError(ex.getMessage());
}
return(null);
}
//**************************************************************************************
/// <summary>
/// Build cllasifier model and save it to a file.
/// </summary>
public override void Build(CandlestickCollection iCandlestick)
{
List <int> trainingPoints = null;
// Calculate average profit and std dev
if (J48Info.ProfitAverage is null || J48Info.ProfitStdDev is null)
{
trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime);
float[] profits = FullToTraining(new List <float>(CalculateFutureProfits(iCandlestick[kTrainingPeriod], ProfitTime)), trainingPoints).ToArray();
J48Info.ProfitStdDev = Statistics.StandardDeviation(profits);
J48Info.ProfitAverage = J48Info.ParentID is null ? 0.0f : Statistics.ArithmeticMean(profits);
WekaJ48Info.UpdateDB(J48Info);
}
// Build model
if (!File.Exists(ModelFilename))
{
OutputMessage("Building model");
if (trainingPoints is null)
{
trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime);
}
Model = new weka.classifiers.trees.J48();
Model.buildClassifier(CreateInstances(iCandlestick, trainingPoints, Attributes, Parameters, Period, ProfitTime));
weka.core.SerializationHelper.write(ModelFilename, Model);
}
// Perfrom crossfold test
if (J48Info.Precision is null)
{
if (Model is null)
{
LoadModel();
}
OutputMessage("Perfroming crossfold");
if (trainingPoints is null)
{
trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime);
}
var instances = CreateInstances(iCandlestick, trainingPoints, Attributes, Parameters, Period, ProfitTime);
var evaluation = new weka.classifiers.Evaluation(instances);
evaluation.crossValidateModel(Model, instances, 10, new java.util.Random(0));
J48Info.Precision = (float)evaluation.pctCorrect();
WekaJ48Info.UpdateDB(J48Info);
}
// Perfrom singular test
if (J48Info.IsSingular == null)
{
if (Model is null)
{
LoadModel();
}
OutputMessage("Perfroming singular test");
var results = new SortedList <Prediction, List <int> >();
foreach (Prediction p in (Prediction[])Enum.GetValues(typeof(Prediction)))
{
results.Add(p, new List <int>());
}
if (trainingPoints is null)
{
trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime);
}
var parameters = CalculateParameters(Parameters, iCandlestick, trainingPoints, Period);
for (int k = 0; k < parameters.Count; k++)
{
var pred = Predict(parameters[k]);
results[pred].Add(trainingPoints[k]);
}
J48Info.IsSingular = results.Count(x => x.Value.Count > 0) <= 1;
WekaJ48Info.UpdateDB(J48Info);
}
// Calculating prediction profits
if (J48Info.PredictionProfits.Count(x => x != null) == 0)
{
if (Model is null)
{
LoadModel();
}
OutputMessage("Calculating prediction profits");
if (trainingPoints is null)
{
trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime);
}
var predictionPoints = GetHistoricalPredictionPoints(iCandlestick, trainingPoints);
foreach (Prediction p in (Prediction[])Enum.GetValues(typeof(Prediction)))
{
float[] profits = FullToTraining(new List <float>(CalculateFutureProfits(iCandlestick[kTrainingPeriod], ProfitTime)), predictionPoints[p]).ToArray();
if (profits.Length < 10)
{
J48Info.PredictionProfits[(int)p] = DecisionToFutureProfit(p, (float)J48Info.ProfitStdDev, (float)J48Info.ProfitAverage);
}
else
{
J48Info.PredictionProfits[(int)p] = Statistics.ArithmeticMean(profits);
}
}
WekaJ48Info.UpdateDB(J48Info);
}
// Create children
if (!J48Info.ReproductionComplete.GetValueOrDefault(false))
{
lock (this)
{
if (J48Info.Precision > 50.0f && !J48Info.IsSingular.GetValueOrDefault(false))
{
OutputMessage("Creating children");
if (trainingPoints is null)
{
trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime);
}
var predictionPoints = GetHistoricalPredictionPoints(iCandlestick, trainingPoints);
foreach (Prediction p in (Prediction[])Enum.GetValues(typeof(Prediction)))
{
if (predictionPoints[p] != null && predictionPoints[p].Count >= 1000 && J48Info.ChildrenID[(int)p] == null)
{
var child = CreateNew(ParametersID, Parameters, Period, ProfitTime, predictionPoints[p]);
// Set parent
child.J48Info.ParentID = ID;
WekaJ48Info.UpdateDB(child.J48Info);
// Update parent info
J48Info.ChildrenID[(int)p] = (int)child.ID;
WekaJ48Info.UpdateDB(J48Info);
childs[(int)p] = child;
}
}
}
J48Info.ReproductionComplete = true;
WekaJ48Info.UpdateDB(J48Info);
}
}
}
// Test the classification result of each map that a user played,
// with the data available as if they were playing through it
public static void classifyTest(String dataString, String playerID)
{
String results = "";
try {
java.io.StringReader stringReader = new java.io.StringReader(dataString);
java.io.BufferedReader buffReader = new java.io.BufferedReader(stringReader);
/* NOTE THAT FOR NAIVE BAYES ALL WEIGHTS CAN BE = 1*/
//weka.core.converters.ConverterUtils.DataSource source = new weka.core.converters.ConverterUtils.DataSource("iris.arff");
weka.core.Instances data = new weka.core.Instances(buffReader); //source.getDataSet();
// setting class attribute if the data format does not provide this information
// For example, the XRFF format saves the class attribute information as well
if (data.classIndex() == -1)
{
data.setClassIndex(data.numAttributes() - 1);
}
weka.classifiers.Classifier cl;
for (int i = 3; i < data.numInstances(); i++)
{
cl = new weka.classifiers.bayes.NaiveBayes();
//cl = new weka.classifiers.trees.J48();
//cl = new weka.classifiers.lazy.IB1();
//cl = new weka.classifiers.functions.MultilayerPerceptron();
((weka.classifiers.functions.MultilayerPerceptron)cl).setHiddenLayers("12");
weka.core.Instances subset = new weka.core.Instances(data, 0, i);
cl.buildClassifier(subset);
weka.classifiers.Evaluation eval = new weka.classifiers.Evaluation(subset);
eval.crossValidateModel(cl, subset, 3, new java.util.Random(1));
results = results + eval.pctCorrect(); // For accuracy measurement
/* For Mathews Correlation Coefficient */
//double TP = eval.numTruePositives(1);
//double FP = eval.numFalsePositives(1);
//double TN = eval.numTrueNegatives(1);
//double FN = eval.numFalseNegatives(1);
//double correlationCoeff = ((TP*TN)-(FP*FN))/Math.Sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN));
//results = results + correlationCoeff;
if (i != data.numInstances() - 1)
{
results = results + ", ";
}
if (i == data.numInstances() - 1)
{
Debug.Log("Player: " + playerID + ", Num Maps: " + data.numInstances() + ", AUC: " + eval.areaUnderROC(1));
}
}
} catch (java.lang.Exception ex)
{
Debug.LogError(ex.getMessage());
}
// Write values to file for a matlab read
// For accuracy
StreamWriter writer = new StreamWriter("DataForMatlab/" + playerID + "_CrossFoldValidations_NeuralNet.txt");
//StreamWriter writer = new StreamWriter("DataForMatlab/"+playerID+"_CrossFoldCorrCoeff.txt"); // For mathews cc
writer.WriteLine(results);
writer.Close();
Debug.Log(playerID + " has been written to file");
}