private void BinaryClassification_DataContextChanged(object sender, DependencyPropertyChangedEventArgs e)
{
if (!(this.DataContext is Models.ModelPerformance))
{
return;
}
var modelPerf = (Models.ModelPerformance) this.DataContext;
var perfData = ((Models.ModelPerformance) this.DataContext).PerformanceData;
if (perfData == null)
{
return;
}
//
var observation = perfData["obs_train"].Select(x => (int)(double)x).ToArray();
var prediction = perfData["prd_train"].Select(x => (int)Math.Round((double)x)).ToArray();
var cm = new ConfusionMatrix(observation, prediction, perfData["Classes"].Count);
//
modelPerf.AAcc = (float)Math.Round(ConfusionMatrix.AAC(cm.Matrix), 3);
modelPerf.OAcc = (float)Math.Round(ConfusionMatrix.OAC(cm.Matrix), 3);
modelPerf.MacPrec = (float)Math.Round(ConfusionMatrix.MacroPrecision(cm.Matrix), 3);
modelPerf.MicPrec = (float)Math.Round(ConfusionMatrix.MicroPrecision(cm.Matrix), 3);
modelPerf.MacRcall = (float)Math.Round(ConfusionMatrix.MacroRecall(cm.Matrix), 3);
modelPerf.MicRcall = (float)Math.Round(ConfusionMatrix.MicroRecall(cm.Matrix), 3);
modelPerf.HSS = (float)Math.Round(ConfusionMatrix.HSS(cm.Matrix, observation.Length), 3);
modelPerf.PSS = (float)Math.Round(ConfusionMatrix.PSS(cm.Matrix, observation.Length), 3);
}
private void setConfusionMatrix(ConfusionMatrix cm, bool isTrainingData)
{
int rowCount = isTrainingData ? m_yobs.Length : m_yobst.Length;
////confusion matrix for MCC
txOAccuracy.Text = ConfusionMatrix.OAC(cm.Matrix).ToString("F3");
txAAccuracy.Text = (ConfusionMatrix.AAC(cm.Matrix)).ToString("F3");
txMiPrecision.Text = ConfusionMatrix.MicroPrecision(cm.Matrix).ToString("F3");
txMaPrecision.Text = ConfusionMatrix.MacroPrecision(cm.Matrix).ToString("F3");
txMiRecall.Text = ConfusionMatrix.MicroRecall(cm.Matrix).ToString("F3");
txMaRecall.Text = ConfusionMatrix.MacroRecall(cm.Matrix).ToString("F3");
txHSS.Text = ConfusionMatrix.HSS(cm.Matrix, rowCount).ToString("F3");
txPSS.Text = ConfusionMatrix.PSS(cm.Matrix, rowCount).ToString("F3");
}
public static ModelPerformance CalculatePerformance(EvaluationResult evalResult, string dataSetName = "Training set")
{
var mpt = new ModelPerformance();
mpt.DatSetName = dataSetName;
mpt.Classes = evalResult.OutputClasses.ToArray();
//define collections for the results
var actual = new double[evalResult.Actual.Count()];
var predicted = new double[evalResult.Predicted.Count()];
//extract result value
for (int i = 0; i < evalResult.Actual.Count(); i++)
{
actual[i] = evalResult.Actual[i];
}
for (int i = 0; i < evalResult.Predicted.Count(); i++)
{
predicted[i] = evalResult.Predicted[i];
}
//regression
if (evalResult.OutputClasses.Count == 1)
{
//Training data set
mpt.SE = (float)actual.SE(predicted);
mpt.RMSE = (float)actual.RMSE(predicted);
mpt.NSE = (float)actual.NSE(predicted);
mpt.PB = (float)actual.PBIAS(predicted);
mpt.CORR = (float)actual.R(predicted);
mpt.DETC = (float)actual.R2(predicted);
}
if (evalResult.OutputClasses.Count < 2 && evalResult.ActualEx.First().Count > 1)
{
//Training data set
var actualValues = evalResult.ActualEx.Select(x => x.Select(xx => (double)xx).ToArray()).ToArray();
var predictedValues = evalResult.PredictedEx.Select(x => x.Select(xx => (double)xx).ToArray()).ToArray();
mpt.SE = (float)AdvancedStatistics.SE(actualValues, predictedValues);
mpt.RMSE = (float)AdvancedStatistics.RMSE(actualValues, predictedValues);
mpt.NSE = (float)AdvancedStatistics.NSE(actualValues, predictedValues);
mpt.PB = (float)AdvancedStatistics.PBIAS(actualValues, predictedValues);
mpt.CORR = (float)AdvancedStatistics.R(actualValues, predictedValues);
mpt.DETC = (float)AdvancedStatistics.R2(actualValues, predictedValues);
}
else if (evalResult.OutputClasses.Count > 1)
{
var retVal = TransformResult(evalResult.ActualEx, evalResult.PredictedEx);
retVal.Add("Classes", mpt.Classes.ToList <object>());
mpt.PerformanceData = retVal;
if (evalResult.OutputClasses.Count == 2)
{
//construct confusion matrix
var o = retVal["obs_train"].Select(x => int.Parse(x.ToString())).ToArray();
var p = retVal["prd_train"].Select(x => ((double)x) < 0.5 ? 0 : 1).ToArray();
var cm = new ConfusionMatrix(o, p, 2);
//get result
mpt.ER = (float)ConfusionMatrix.Error(cm.Matrix);
//mpt.AUC =
//confusion matrix for current threshold
mpt.Acc = (float)ConfusionMatrix.OAC(cm.Matrix);
mpt.ER = (float)ConfusionMatrix.Error(cm.Matrix);
mpt.Precision = (float)ConfusionMatrix.Precision(cm.Matrix, 1);
mpt.Recall = (float)ConfusionMatrix.Recall(cm.Matrix, 1);
mpt.F1Score = (float)ConfusionMatrix.Fscore(cm.Matrix, 1);
mpt.HSS = (float)ConfusionMatrix.HSS(cm.Matrix, o.Length);
mpt.PSS = (float)ConfusionMatrix.PSS(cm.Matrix, o.Length);
//lab
//mpt.Label = m_Classes[1];// "TRUE";
//txNegativeLable.Text = m_Classes[0];// "FALSE";
//false and true positive negative
mpt.FN = (float)cm.Matrix[1][0]; //cm.FalseNegatives
mpt.FP = (float)cm.Matrix[0][1]; //cm.FalsePositives
mpt.TP = (float)cm.Matrix[1][1]; //cm.TruePositives
mpt.TN = (float)cm.Matrix[0][0]; //cm.TrueNegatives
}
else
{
//construct confusion matrix
var o = retVal["obs_train"].Select(x => int.Parse(x.ToString())).ToArray();
var p = retVal["prd_train"].Select(x => int.Parse(x.ToString())).ToArray();
var cm = new ConfusionMatrix(o, p, evalResult.OutputClasses.Count);
//confusion matrix for current threshold
mpt.OAcc = (float)ConfusionMatrix.OAC(cm.Matrix);
mpt.AAcc = (float)ConfusionMatrix.AAC(cm.Matrix);
mpt.MacPrec = (float)ConfusionMatrix.MacroPrecision(cm.Matrix);
mpt.MacRcall = (float)ConfusionMatrix.MacroRecall(cm.Matrix);
mpt.MicPrec = (float)ConfusionMatrix.MicroPrecision(cm.Matrix);
mpt.MicRcall = (float)ConfusionMatrix.MicroRecall(cm.Matrix);
//mpt.MacFscore = (float)ConfusionMatrix.MacroFscore(cm.Matrix);
//mpt.MicFscore = (float)ConfusionMatrix.MicroFscore(cm.Matrix);
mpt.HSS = (float)ConfusionMatrix.HSS(cm.Matrix, o.Length);
mpt.PSS = (float)ConfusionMatrix.PSS(cm.Matrix, o.Length);
}
}
return(mpt);
}
public void MCC_Classifier_Test01()
{
int[] y1 = new int[13] {
0, 1, 2, 1, 2, 0, 1, 0, 2, 2, 2, 0, 2
};
int[] y2 = new int[13] {
1, 2, 0, 1, 2, 0, 1, 0, 2, 1, 2, 0, 2
};
//construct binary Confusion matrix
var cm = new ConfusionMatrix(y1, y2, 3);
//
double v1 = ConfusionMatrix.TPi(cm.Matrix, 0);
double v2 = ConfusionMatrix.FPi(cm.Matrix, 0);
double v3 = ConfusionMatrix.FNi(cm.Matrix, 0);
double v4 = ConfusionMatrix.TNi(cm.Matrix, 0);
Assert.True(v1 == 3);
Assert.True(v2 == 1);
Assert.True(v3 == 1);
Assert.True(v4 == 8);
v1 = ConfusionMatrix.TPi(cm.Matrix, 1);
v2 = ConfusionMatrix.FPi(cm.Matrix, 1);
v3 = ConfusionMatrix.FNi(cm.Matrix, 1);
v4 = ConfusionMatrix.TNi(cm.Matrix, 1);
Assert.True(v1 == 2);
Assert.True(v2 == 2);
Assert.True(v3 == 1);
Assert.True(v4 == 8);
v1 = ConfusionMatrix.TPi(cm.Matrix, 2);
v2 = ConfusionMatrix.FPi(cm.Matrix, 2);
v3 = ConfusionMatrix.FNi(cm.Matrix, 2);
v4 = ConfusionMatrix.TNi(cm.Matrix, 2);
Assert.True(v1 == 4);
Assert.True(v2 == 1);
Assert.True(v3 == 2);
Assert.True(v4 == 6);
//Accuracy
v1 = ((3.0 + 8.0) / 13.0 + (2.0 + 8.0) / 13.0 + (4.0 + 6.0) / 13.0) / 3;
v2 = (3.0 + 2.0 + 4.0) / 13.0;
double dval1 = ConfusionMatrix.OAC(cm.Matrix);
double dval2 = ConfusionMatrix.AAC(cm.Matrix);
Assert.True(dval1 == v2);
Assert.True(dval2 == v1);
//Error
//Precision
var pv1 = (3.0 + 2.0 + 4.0) / (3.0 + 2.0 + 4.0 + 1.0 + 2.0 + 1.0);
var pv2 = (3.0 / (3.0 + 1.0) + 2.0 / (2.0 + 2.0) + 4.0 / (4.0 + 1.0)) / (3);
dval1 = ConfusionMatrix.MicroPrecision(cm.Matrix);
dval2 = ConfusionMatrix.MacroPrecision(cm.Matrix);
Assert.True(dval1 == pv1);
Assert.True(dval2 == pv2);
//Recall
var rv1 = (3.0 + 2.0 + 4.0) / (3.0 + 2.0 + 4.0 + 1.0 + 1.0 + 2.0);
var rv2 = (3.0 / (3.0 + 1.0) + 2.0 / (2.0 + 1.0) + 4.0 / (4.0 + 2.0)) / (3);
dval1 = ConfusionMatrix.MicroRecall(cm.Matrix);
dval2 = ConfusionMatrix.MacroRecall(cm.Matrix);
Assert.True(dval1 == rv1);
Assert.True(dval2 == rv2);
//FScore
dval1 = ConfusionMatrix.MicroFscore(cm.Matrix);
dval2 = ConfusionMatrix.MacroFscore(cm.Matrix);
Assert.True(dval1 == 2 * (pv1 * rv1) / (pv1 + rv1));
Assert.True(dval2 == 2 * (pv2 * rv2) / (pv2 + rv2));
}
public void Binary_Classifier_Test01()
{
int[] y1 = new int[6] {
0, 1, 1, 0, 1, 1
};
double[] y2 = new double[6] {
0.103345, 0.816285, 0.36523, 0.164645, 0.988035, 0.963756
};
var y22 = calculateOutput(y2, 0.5);
int[] y11 = new int[] { 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1 };
double[] y21 = new double[] { 0.583333, 0, 1, 0.1026, 0.058482, 0.222538, 0, 0.306647, 0.05, 0.345238, 0, 0.45, 0.137926, 0.291667, 0, 1, 0.137926, 0.055871, 0, 0, 0.46875, 0.479167, 0, 0.125, 0, 0, 0.240183, 0.270833, 0.125, 1, 0.5625, 0.1026, 1, 0.125, 0.375, 1, 0.137926, 0.240183, 0.275984, 0.5, 0.137926, 0.625, 0.5, 0.5, 0.875, 0, 0.875, 0.765625, 0.137926, 0.875, 1, 0.375, 1, 0.1026, 0.45, 0.583333, 0.351637, 1, 0.1026, 0, 0.6875, 0.25, 0, 0.625, 0, 0.1026, 0.649777, 1, 0.202232, 0, 0, 0, 0.479167, 0.375, 0.25, 1, 0.17748, 0.125, 0.5625, 0.1026, 0.75463, 0, 0, 0, 1, 0.1026, 1, 0.458333, 0.1026, 1, 0.875, 1 };
var y222 = calculateOutput(y21, 0.5);
//construct binary Confusion matrix
var cm = new ConfusionMatrix(y1, y22, 2);
//
var val1 = cm.Matrix[1][1];
var val2 = ConfusionMatrix.TPi(cm.Matrix, 1);
Assert.True(val1 == val2);
//
val1 = cm.Matrix[1][0];
val2 = ConfusionMatrix.FNi(cm.Matrix, 1);
Assert.True(val1 == val2);
//
val1 = cm.Matrix[0][1];
val2 = ConfusionMatrix.FPi(cm.Matrix, 1);
Assert.True(val1 == val2);
//
val1 = cm.Matrix[0][0];
val2 = ConfusionMatrix.TNi(cm.Matrix, 1);
Assert.True(val1 == val2);
//Accuracy
double dval1 = (cm.Matrix[0][0] + cm.Matrix[1][1]) / (double)y1.Length;
double dval2 = ConfusionMatrix.OAC(cm.Matrix);
double dval3 = ConfusionMatrix.AAC(cm.Matrix);
Assert.True(dval1 == dval2);
Assert.True(dval2 == dval3);
//Error
//Precision a/a+b
var p1 = (double)cm.Matrix[1][1] / (double)(cm.Matrix[0][1] + cm.Matrix[1][1]);
dval2 = ConfusionMatrix.Precision(cm.Matrix, 1);
Assert.True(p1 == dval2);
//Recall - a/a+c
var r1 = (double)cm.Matrix[1][1] / (double)(cm.Matrix[1][0] + cm.Matrix[1][1]);
dval2 = ConfusionMatrix.Recall(cm.Matrix, 1);
Assert.True(r1 == dval2);
//FScore
dval1 = 2 * (p1 * r1) / (p1 + r1);
dval2 = ConfusionMatrix.Fscore(cm.Matrix, 1);
Assert.True(dval1 == dval2);
//Specificity
//dval1 = ConfusionMatrix.TNR(cm.Matrix);
//dval2 = ConfusionMatrix.Specificity(cm.Matrix, 1);
}