public TestingPageViewModel(DataProvider dataProvider, NeuralNetworks neuralNetwork)
: base(dataProvider, neuralNetwork)
{
UseTrainingSet = false;
_confusionMatrix = new ConfusionMatrix("ConfusionMatrix", dataProvider);
AddCommandButtons();
}
public static ConfusionMatrix CrossValidation(DataTable data, int folds)
{
DataTable t = CollectionExtensions.OrderRandomly(data.AsEnumerable()).CopyToDataTable();
ConfusionMatrix res = new ConfusionMatrix(classValues);
int n = t.Rows.Count;
int c = (int)Math.Floor(n / (double)folds);
int start = 0;
int stop = c - 1;
int fold = 1;
while (start < n) {
if (stop > n) { stop = n; }
//use start and stop to get a subset of data as test
//set everything else as train
IEnumerable<DataRow> testEnum = null, trainEnum = null;
CollectionExtensions.Split(t.AsEnumerable(), start, stop, ref testEnum, ref trainEnum);
DataTable trainingData = trainEnum.CopyToDataTable();
start = stop + 1;
stop = start - 1 + c;
Train(trainingData);
ConfusionMatrix subres = Test(testEnum.CopyToDataTable());
res.Add(subres); //also updates the accuracy
fold++;
}
return res;
}
public ConfusionMatrix ParseMatrix(string file)
{
string operation = file.Substring(0, file.Length - 4);
EditOperation editOperation;
switch (operation)
{
case "deletions":
editOperation = EditOperation.Deletion;
break;
case "insertions":
editOperation = EditOperation.Insertion;
break;
case "substitutions":
editOperation = EditOperation.Substitution;
break;
case "transpositions":
editOperation = EditOperation.Transposition;
break;
default:
editOperation = EditOperation.Unknown;
break;
}
ConfusionMatrix matrix = new ConfusionMatrix(editOperation);
Encoding enc = EncodingDetector.DetectEncoding(file);
string[] lines = System.IO.File.ReadAllLines(file, enc);
char[] delimiter = new char[] { ',' };
string header = lines.First().Replace("#:", "");
string[] keys = header.Split(delimiter);
foreach (string line in lines)
{
if (line.StartsWith("#"))
continue;
string[] parts = line.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
string key = parts.First();
string[] values = parts.Last().Split(delimiter);
for (int i = 0; i < values.Length; i++)
{
matrix.SetValue(key[0], keys[i][0], int.Parse(values[i]));
}
}
return matrix;
}
public void ComputeTest5()
{
double[][] inputs = training.Submatrix(null, 0, 3);
int[] labels = Tools.Scale(0, 1, -1, 1, training.GetColumn(4)).ToInt32();
Gaussian kernel = Gaussian.Estimate(inputs);
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.UseClassProportions = true;
double error = smo.Run();
Assert.AreEqual(1, smo.Complexity);
Assert.AreEqual(0.4, smo.PositiveWeight);
Assert.AreEqual(1.0, smo.NegativeWeight);
Assert.AreEqual(0.4, smo.WeightRatio, 1e-10);
Assert.AreEqual(0.38095238095238093, error);
Assert.AreEqual(265.78327637381551, (machine.Kernel as Gaussian).Sigma);
Assert.AreEqual(32, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(7, matrix.FalseNegatives);
Assert.AreEqual(9, matrix.FalsePositives);
Assert.AreEqual(5, matrix.TruePositives);
Assert.AreEqual(21, matrix.TrueNegatives);
Assert.AreEqual(0.41666666666666669, matrix.Sensitivity);
Assert.AreEqual(0.7, matrix.Specificity);
}
public void ChiSquareTest()
{
ConfusionMatrix target = new ConfusionMatrix(6, 2, 6, 18);
double[,] expected = target.ExpectedValues;
Assert.AreEqual(3, target.ExpectedValues[0, 0]);
Assert.AreEqual(5, target.ExpectedValues[0, 1]);
Assert.AreEqual(9, target.ExpectedValues[1, 0]);
Assert.AreEqual(15, target.ExpectedValues[1, 1]);
Assert.AreEqual(6.4, target.ChiSquare, 1e-5);
ChiSquareTest test = new ChiSquareTest(target);
Assert.AreEqual(target.ChiSquare, test.Statistic);
Assert.AreEqual(1, test.DegreesOfFreedom);
Assert.IsTrue(test.Significant);
}
public void Test()
{
const int ClassCount = 3;
var matrix = new ConfusionMatrix(new List <string> {
"Foo", "Bar", "Baz"
});
var rnd = new Random();
for (int i = 0; i < 1000; i++)
{
int target = rnd.Next(ClassCount);
int predicted = rnd.Next(ClassCount);
matrix.Prediction(target, predicted);
}
matrix.CalculateResult(1000);
string result = matrix.GetReport();
}
public void chunking_dataset_crf()
{
Chunking chunking = new Chunking(path: Path.GetTempPath());
// Learn a mapping between each word to an integer class label:
var wordMap = new Codification().Learn(chunking.Words);
// Learn a mapping between each tag to an integer class labels:
var tagMap = new Codification().Learn(chunking.Tags);
// Convert the training and testing sets into integer labels:
int[][] trainX = wordMap.Transform(chunking.Training.Item1);
int[][] testX = wordMap.Transform(chunking.Testing.Item1);
// Convert the training and testing tags into integer labels:
int[][] trainY = tagMap.Transform(chunking.Training.Item2);
int[][] testY = tagMap.Transform(chunking.Testing.Item2);
int numberOfClasses = chunking.Tags.Length;
int numberOfSymbols = chunking.Words.Length;
// Learn one Markov model using the training data
var teacher = new QuasiNewtonLearning <int>()
{
Function = new MarkovDiscreteFunction(states: numberOfClasses, symbols: numberOfSymbols)
};
// Use the teacher to learn a Conditional Random Field model
ConditionalRandomField <int> crf = teacher.Learn(trainX, trainY);
// Use the crf to predict instances:
int[][] predY = crf.Decide(testX);
// Check the accuracy of the model:
var cm = new ConfusionMatrix(
predicted: predY.Concatenate(),
expected: testY.Concatenate());
double acc = cm.Accuracy;
Assert.AreEqual(0.99983114169322662, acc, 1e-10);
}
private void CompareMatrics(ClassificationMetrics metrics)
{
Assert.Equal(.98, metrics.AccuracyMacro);
Assert.Equal(.98, metrics.AccuracyMicro, 2);
Assert.Equal(.06, metrics.LogLoss, 2);
Assert.InRange(metrics.LogLossReduction, 94, 96);
Assert.Equal(1, metrics.TopKAccuracy);
Assert.Equal(3, metrics.PerClassLogLoss.Length);
Assert.Equal(0, metrics.PerClassLogLoss[0], 1);
Assert.Equal(.1, metrics.PerClassLogLoss[1], 1);
Assert.Equal(.1, metrics.PerClassLogLoss[2], 1);
ConfusionMatrix matrix = metrics.ConfusionMatrix;
Assert.Equal(3, matrix.Order);
Assert.Equal(3, matrix.ClassNames.Count);
Assert.Equal("0", matrix.ClassNames[0]);
Assert.Equal("1", matrix.ClassNames[1]);
Assert.Equal("2", matrix.ClassNames[2]);
Assert.Equal(50, matrix[0, 0]);
Assert.Equal(50, matrix["0", "0"]);
Assert.Equal(0, matrix[0, 1]);
Assert.Equal(0, matrix["0", "1"]);
Assert.Equal(0, matrix[0, 2]);
Assert.Equal(0, matrix["0", "2"]);
Assert.Equal(0, matrix[1, 0]);
Assert.Equal(0, matrix["1", "0"]);
Assert.Equal(48, matrix[1, 1]);
Assert.Equal(48, matrix["1", "1"]);
Assert.Equal(2, matrix[1, 2]);
Assert.Equal(2, matrix["1", "2"]);
Assert.Equal(0, matrix[2, 0]);
Assert.Equal(0, matrix["2", "0"]);
Assert.Equal(1, matrix[2, 1]);
Assert.Equal(1, matrix["2", "1"]);
Assert.Equal(49, matrix[2, 2]);
Assert.Equal(49, matrix["2", "2"]);
}
public void Test_ClassificationWith_BackwardsEliminationKnnModel()
{
// Given
var randomizer = new Random(55);
var data = TestDataBuilder.ReadIrisData();
var trainingDataPercentage = 0.8;
int trainingDataCount = (int)(data.RowCount * trainingDataPercentage);
var shuffledIndices = data.RowIndices.Shuffle();
var trainingIndices = shuffledIndices.Take(trainingDataCount).ToList();
var testIndices =
shuffledIndices.Skip(trainingDataCount).Take(shuffledIndices.Count - trainingDataCount).ToList();
var trainingData = data.GetSubsetByRows(trainingIndices);
var testData = data.GetSubsetByRows(testIndices);
var weightingFunction = new GaussianFunction(0.07);
var predictor = new SimpleKnnClassifier<string>(
new EuclideanDistanceMeasure(),
new MinMaxNormalizer(),
weightingFunction.GetValue);
var modelBuilder = new BackwardsEliminationKnnModelBuilder<string>(
new MinMaxNormalizer(),
predictor,
new ClassificationAccuracyError<string>()
);
var modelParams = new KnnAdditionalParams(3, true);
var errorMeasure = new MeanSquareError();
var subject = new BackwardsEliminationKnnClassifier<string>(
new EuclideanDistanceMeasure(),
new MinMaxNormalizer(),
weightingFunction.GetValue);
// When
var model = modelBuilder.BuildModel(trainingData, "iris_class", modelParams);
var actualResults = subject.Predict(testData, model, "iris_class");
var confusionMatrix = new ConfusionMatrix<string>(testData.GetColumnVector<string>("iris_class"),
actualResults);
// Then
Assert.IsTrue(confusionMatrix.Accuracy >= 0.95);
}
public EvaluationResult Evaluate()
{
var result = new EvaluationResult();
// load evaluation data
result.StartMeasure(EvaluationResult.RecordType.LoadDataset);
var reader = new ExcelReader(Helpers.DatasetPath);
DataTable dataStore = reader.GetWorksheet("Evaluation");
int[] labels = dataStore.ToVector <int>("Label");
string[] learnData = dataStore.ToVector <string>("Sentiment");
result.StopMeasure();
// tokenize
result.StartMeasure(EvaluationResult.RecordType.Tokenization);
string[][] tokenized = learnData.Select(x => _preprocessor.Process(x)).ToArray();
result.StopMeasure();
// benchmark featurization
result.StartMeasure(EvaluationResult.RecordType.Featurization);
int[][] learnTokenized = _bagOfWords.Transform(tokenized).ToInt32();
result.StopMeasure();
// benchmark classification
result.StartMeasure(EvaluationResult.RecordType.Classification);
int[] testResult = _bayes.Decide(learnTokenized);
result.StopMeasure();
// calculate stats
result.StartMeasure(EvaluationResult.RecordType.Statistics);
var mat = new ConfusionMatrix(testResult, labels);
var roc = new ReceiverOperatingCharacteristic(labels, testResult.ToDouble());
roc.Compute(200);
result.StopMeasure();
// save metrics
result.Matrix = mat;
result.Roc = roc;
return(result);
}
static void RunDiabetesTest(string trainingFile, string testFile)
{
Console.WriteLine($"[{GetTimeStamp()}] Diabetes test:");
ReferenceTable table = new ReferenceTable();
Console.WriteLine($"[{GetTimeStamp()}] Loading training file: {trainingFile}.");
List <Record> trainingRecords = DiscretizeDataset(ParseRecords(File.ReadAllLines(trainingFile)), table);
Console.WriteLine($"[{GetTimeStamp()}] Loading test file: {trainingFile}.");
List <Record> testRecords = DiscretizeDataset(ParseRecords(File.ReadAllLines(testFile)), table);
var attributes = table.GetIndex().Select(idx => new DiscreteAttribute(idx, table.GetName(idx), table.GetValues(idx))).ToList();
//DecisionTree.AssignProbabilitiesByClass(attributes, trainingRecords, false);
attributes.ForEach(attribute => DecisionTree.AssignProbabilities(attribute, trainingRecords));
attributes.ForEach(attribute => DecisionTree.AssignProbabilities(attribute, testRecords.Union(trainingRecords).ToList()));
Console.WriteLine($"[{GetTimeStamp()}] Building Ensemble of ID3 decision trees...");
Random sampler = new Random();
var ensemble = new Ensemble();
for (int i = 0; i <= 1000; i++)
{
DecisionTree tree = new DecisionTree(attributes, SampledData(trainingRecords, sampler)).Build();
ensemble.AddVoter(tree.Test);
if (i != 0 && i != 20 && i != 100 && i != 500 && i != 1000 && i != 2000)
{
continue;
}
ConfusionMatrix trainingMatrix = RunPredictions(trainingRecords, rec => rec.IsPositive, ensemble.Test);
ConfusionMatrix testMatrix = RunPredictions(testRecords, rec => rec.IsPositive, ensemble.Test);
Console.WriteLine("----------------------------------------------------------------");
Console.WriteLine($"[{GetTimeStamp()}][Ensemble: {i}] Printing sanity results: ");
PrintResults(trainingMatrix);
Console.WriteLine($"[{GetTimeStamp()}][Ensemble: {i}] Printing prediction results: ");
PrintResults(testMatrix);
}
}
public void Test_ClassificationWith_BackwardsEliminationKnnModel()
{
// Given
var randomizer = new Random(55);
var data = TestDataBuilder.ReadIrisData();
var trainingDataPercentage = 0.8;
int trainingDataCount = (int)(data.RowCount * trainingDataPercentage);
var shuffledIndices = data.RowIndices.Shuffle();
var trainingIndices = shuffledIndices.Take(trainingDataCount).ToList();
var testIndices =
shuffledIndices.Skip(trainingDataCount).Take(shuffledIndices.Count - trainingDataCount).ToList();
var trainingData = data.GetSubsetByRows(trainingIndices);
var testData = data.GetSubsetByRows(testIndices);
var weightingFunction = new GaussianFunction(0.07);
var predictor = new SimpleKnnClassifier <string>(
new EuclideanDistanceMeasure(),
new MinMaxNormalizer(),
weightingFunction.GetValue);
var modelBuilder = new BackwardsEliminationKnnModelBuilder <string>(
new MinMaxNormalizer(),
predictor,
new ClassificationAccuracyError <string>()
);
var modelParams = new KnnAdditionalParams(3, true);
var errorMeasure = new MeanSquareError();
var subject = new BackwardsEliminationKnnClassifier <string>(
new EuclideanDistanceMeasure(),
new MinMaxNormalizer(),
weightingFunction.GetValue);
// When
var model = modelBuilder.BuildModel(trainingData, "iris_class", modelParams);
var actualResults = subject.Predict(testData, model, "iris_class");
var confusionMatrix = new ConfusionMatrix <string>(testData.GetColumnVector <string>("iris_class"),
actualResults);
// Then
Assert.IsTrue(confusionMatrix.Accuracy >= 0.95);
}
/// <summary>
/// Creates a new McNemar test.
/// </summary>
///
/// <param name="matrix">The contingency table to test.</param>
/// <param name="yatesCorrection">True to use Yate's correction of
/// continuity, falser otherwise. Default is false.</param>
///
public McNemarTest(ConfusionMatrix matrix, bool yatesCorrection = false)
{
int a = matrix.TruePositives;
int b = matrix.FalseNegatives;
int c = matrix.FalsePositives;
int d = matrix.TrueNegatives;
double u = b - c;
if (yatesCorrection)
{
u = Math.Abs(u) - 0.5;
}
double chiSquare = (u * u) / (b + c);
int df = 1;
Compute(chiSquare, df);
}
public void McNemarTestConstructorTest()
{
int[,] matrix =
{
{ 101, 121 },
{ 59, 33 },
};
ConfusionMatrix a = new ConfusionMatrix(matrix);
McNemarTest target = new McNemarTest(a, true);
Assert.AreEqual(21.0125, target.Statistic);
Assert.AreEqual(1, target.DegreesOfFreedom);
McNemarTest target2 = new McNemarTest(a, false);
Assert.AreEqual(21.355555, target2.Statistic, 1e-5);
Assert.AreEqual(1, target2.DegreesOfFreedom);
}
public static ConfusionMatrix Test(DataTable t)
{
ConfusionMatrix cmat = new ConfusionMatrix(classValues);
int numDims = t.Columns.Count - 1; //remove 1 for the class labels
foreach (DataRow row in t.Rows) {
//convert the datarow into an expression
Sample sample = new Sample(row, classIndex);
//test the sample and update results
string prediction = TestInstance(sample);
cmat.matrix[prediction][sample.classLabel]++;
if (sample.classLabel.Equals(prediction)) {
cmat.numCorrect++;
} else {
cmat.numWrong++;
}
}
cmat.accuracy = ((double)cmat.numCorrect) / t.Rows.Count;
return cmat;
}
public virtual void TestBasic()
{
string expected = " Guess/Gold C1 C2 C3 Marg. (Guess)\n" + " C1 2 0 0 2\n" + " C2 1 0 0 1\n" + " C3 0 0 1 1\n"
+ " Marg. (Gold) 3 0 1\n\n" + " C1 = a prec=1, recall=0.66667, spec=1, f1=0.8\n" + " C2 = b prec=0, recall=n/a, spec=0.75, f1=n/a\n" + " C3 = c prec=1, recall=1, spec=1, f1=1\n";
ConfusionMatrix <string> conf = new ConfusionMatrix <string>(Locale.Us);
conf.Add("a", "a");
conf.Add("a", "a");
conf.Add("b", "a");
conf.Add("c", "c");
string result = conf.PrintTable();
if (echo)
{
System.Console.Error.WriteLine(result);
}
else
{
NUnit.Framework.Assert.AreEqual(expected, result);
}
}
private static void knn(double[][] inputs, int[] outputs)
{
// Create a new k-NN algorithm:
var knn = new KNearestNeighbors()
{
K = 3, // base a decision on the class labels of the three nearest neighbors of the query point
Distance = new Euclidean() // actually the default
};
// Learn a k-NN classifier
knn = knn.Learn(inputs, outputs);
// Get predictions according to kNN
int[] predicted = knn.Decide(inputs);
// Create a confusion matrix to check the quality of the predictions:
var cm = new ConfusionMatrix(predicted: predicted, expected: outputs);
// Check the accuracy measure:
double accuracy = cm.Accuracy; // (should be 1.0 or 100%)
}
public static double GeometricMean(this ConfusionMatrix confusionMatrix)
{
if (confusionMatrix.NumberOfClasses == 2)
{
var truePositives = confusionMatrix.GetCountForClassPair(0, 0);
var trueNegatives = confusionMatrix.GetCountForClassPair(1, 1);
var falseNegatives = confusionMatrix.GetCountForClassPair(1, 0);
var falsePositives = confusionMatrix.GetCountForClassPair(0, 1);
var sensitivty = truePositives / (truePositives + falseNegatives);
var specificity = trueNegatives / (falsePositives + trueNegatives);
var geometricMean = Math.Sqrt(sensitivty * specificity);
return(geometricMean);
}
else
{
return(0.0);
}
}
public virtual void TestBulkAdd()
{
string expected = " Guess/Gold C1 C2 Marg. (Guess)\n" + " C1 10 5 15\n" + " C2 2 3 5\n" + " Marg. (Gold) 12 8\n\n" + " C1 = 1 prec=0.66667, recall=0.83333, spec=0.375, f1=0.74074\n"
+ " C2 = 2 prec=0.6, recall=0.375, spec=0.83333, f1=0.46154\n";
ConfusionMatrix <int> conf = new ConfusionMatrix <int>(Locale.Us);
conf.Add(1, 1, 10);
conf.Add(1, 2, 5);
conf.Add(2, 1, 2);
conf.Add(2, 2, 3);
string result = conf.PrintTable();
if (echo)
{
System.Console.Error.WriteLine(result);
}
else
{
NUnit.Framework.Assert.AreEqual(expected, result);
}
}
public void Test(Action <int, bool> progressFunc)
{
if (TestSet == null)
{
return;
}
float deltaSetPercent = 1.0f / (TestSetSize) * 100.0f;
float percent = 0.0f;
float prevTestSensitivity = TestingSensitivity;
ConfusionMatrix.ResetMatrix();
for (uint set = 0; set < TestSetSize; set++, percent += deltaSetPercent)
{
progressFunc((int)percent, false);
Propagate(TestSet[set], testing: true);
EvaluateOutput();
}
TestingSensitivity = Sensitivity();
progressFunc((int)percent, false);
}
public override List <ConfusionMatrix> TestModel(TrainningData trainningData)
{
ContinuousDataTableAdapter continuousDataTableAdapter = new ContinuousDataTableAdapter();
DataTable continuousDataTable = continuousDataTableAdapter.GetData();
DataTable dataTable = continuousDataTable.DefaultView.ToTable(false, TableMetaData.TestingAttributes);
string[] columnNames;
double[][] inputs = dataTable.ToArray(out columnNames);
int[] expected = trainningData.ClassificationAttribute;
int[] predicted = ComputeModel(inputs);
int positiveValue = trainningData.PositiveValue;
int negativeValue = trainningData.NegativeValue;
ConfusionMatrix confusionMatrix = new ConfusionMatrix(predicted, expected, positiveValue, negativeValue);
return(new List <ConfusionMatrix> {
confusionMatrix
});
}
public override ConfusionMatrix Execute()
{
//Create a knn classifer with 2 classes
var knn = new KNearestNeighbors(k: k,
classes: 2,
inputs: trainingSet,
outputs: trainingOutput);
//Map the classifier over the test set
//This wil return an array where index i is the classificatioon of the i-th vector
//of the testSet
var predicted = AlgorithmHelpers
.MergeArrays(trainingSet, testSet)
.Select(x => knn.Compute(x))
.ToArray();
//Create a new confusion matrix with the calculated parameters
var cmatrix = new ConfusionMatrix(predicted, AlgorithmHelpers.MergeArrays(trainingOutput, expected), POSITIVE, NEGATIVE);
return(cmatrix);
}
private void btnTestingRun_Click(object sender, EventArgs e)
{
if (svm == null || dgvTestingSource.DataSource == null)
{
MessageBox.Show("Please create a machine first.");
return;
}
// Creates a matrix from the source data table
double[,] table = (dgvTestingSource.DataSource as DataTable).ToMatrix();
// Extract the first and second columns (X and Y)
double[][] inputs = table.GetColumns(0, 1).ToJagged();
// Extract the expected output labels
bool[] expected = Classes.Decide(table.GetColumn(2));
// Compute the actual machine outputs
bool[] output = svm.Decide(inputs);
// Use confusion matrix to compute some performance metrics
ConfusionMatrix cm = new ConfusionMatrix(output, expected);
dgvPerformance.Rows.Add();
dgvPerformance.Rows[0].Cells["TruePositives"].Value = cm.TruePositives;
dgvPerformance.Rows[0].Cells["FalseNegatives"].Value = cm.FalseNegatives;
dgvPerformance.Rows[0].Cells["TrueNegatives"].Value = cm.TrueNegatives;
dgvPerformance.Rows[0].Cells["FalsePositives"].Value = cm.FalsePositives;
dgvPerformance.Rows[0].Cells["Sensitivity"].Value = cm.Sensitivity;
dgvPerformance.Rows[0].Cells["Specificity"].Value = cm.Specificity;
dgvPerformance.Rows[0].Cells["Efficiency"].Value = cm.Efficiency;
dgvPerformance.Rows[0].Cells["Accuracy"].Value = cm.Accuracy;
// Create performance scatter plot
CreateResultScatterplot(zedGraphControl1, inputs, expected.ToMinusOnePlusOne().ToDouble(), output.ToMinusOnePlusOne().ToDouble());
}
/// <summary>
/// Writes performance results for individual classes to a specified stream writer.
/// </summary>
/// <param name="writer">The <see cref="StreamWriter"/> to write to.</param>
/// <param name="confusionMatrix">The confusion matrix.</param>
/// <param name="auc">The per-class AUC.</param>
private void WriteIndividualClassPerformance(
StreamWriter writer,
ConfusionMatrix <string> confusionMatrix,
IDictionary <string, double> auc)
{
writer.WriteLine();
writer.WriteLine(" Performance on individual classes");
writer.WriteLine("===================================");
writer.WriteLine();
writer.WriteLine(
" {0,5} {1,15} {2,10} {3,11} {4,9} {5,10} {6,10} {7,10} {8,10}",
"Index",
"Label",
"#Truth",
"#Predicted",
"#Correct",
"Precision",
"Recall",
"F1",
"AUC");
writer.WriteLine("----------------------------------------------------------------------------------------------------");
foreach (var classLabelIndex in confusionMatrix.ClassLabelSet.Indexes)
{
string classLabel = confusionMatrix.ClassLabelSet.GetElementByIndex(classLabelIndex);
writer.WriteLine(
" {0,5} {1,15} {2,10} {3,11} {4,9} {5,10:0.0000} {6,10:0.0000} {7,10:0.0000} {8,10:0.0000}",
classLabelIndex + 1,
classLabel,
confusionMatrix.TrueLabelCount(classLabel),
confusionMatrix.PredictedLabelCount(classLabel),
confusionMatrix[classLabel, classLabel],
confusionMatrix.Precision(classLabel),
confusionMatrix.Recall(classLabel),
confusionMatrix.F1(classLabel),
auc[classLabel]);
}
}
// printare la confusion
void Awake()
{
DateTime _start = DateTime.Now;
int forest = setNumberOfTrees();
int dataSet = SettingGui.dataSet;
Measure measure = setMeasure();
switch (dataSet)
{
case 1:
performanceIONO(forest, measure);
break;
case 2:
performanceGerman(forest, measure);
break;
case 3:
performanceBreastCancer(forest, measure);
break;
case 4:
performanceVehicle(forest, measure);
break;
case 5:
performanceDiabetes(forest, measure);
break;
}
TimeSpan elapsed = DateTime.Now - _start;
time.text = "Time consumed: " + elapsed.Minutes + "min : " + elapsed.Seconds + "''";
confusion = p.C;
setAccuracyErrorRate();
ConfusionMatrixDropDown();
SetTestingExamples();
string memorya = GetAllocatedMemory();
memoryConsumed.text = "Memory consumed: " + memorya;
}
public void learn_stump_classifier()
{
#region doc_learn
// Let's say we want to classify the following 2-dimensional
// data samples into 2 possible classes, either true or false:
double[][] inputs =
{
new double[] { 10, 42 },
new double[] { 162, 96 },
new double[] { 125, 20 },
new double[] { 96, 6 },
new double[] { 2, 73 },
new double[] { 52, 51 },
new double[] { 71, 49 },
};
// And those are their associated class labels
bool[] outputs =
{
false, false, true, true, false, false, true
};
// We create a learning algorithm as:
var teacher = new ThresholdLearning();
// Now, we can use the Learn method to learn a classifier:
DecisionStump classifier = teacher.Learn(inputs, outputs);
// Now, we can check how good it is using a confusion matrix:
var cm = ConfusionMatrix.Estimate(classifier, inputs, outputs);
double error = cm.Error; // should be ~0.14
// We can also compute the model outputs for new samples using
bool y = classifier.Decide(new double[] { 71, 48 }); // should be false
#endregion
Assert.AreEqual(false, y);
Assert.AreEqual(0.14285714285714285, error, 1e-10);
}
public void ConfusionMatrixConstructorTest3()
{
// System output
int[] predicted = new int[] { 2, 0, 1 };
// Correct output
int[] expected = new int[] { 5, 2, 1 };
// 1 means positive (the others shall be treated as negatives)
int positiveValue = 1;
ConfusionMatrix target = new ConfusionMatrix(predicted, expected, positiveValue);
int falseNegatives = 0;
int falsePositives = 0;
int truePositives = 1;
int trueNegatives = 2;
Assert.AreEqual(predicted.Length, target.Samples);
Assert.AreEqual(2, target.ActualNegatives);
Assert.AreEqual(1, target.ActualPositives);
Assert.AreEqual(2, target.PredictedNegatives);
Assert.AreEqual(1, target.PredictedPositives);
Assert.AreEqual(falseNegatives, target.FalseNegatives);
Assert.AreEqual(falsePositives, target.FalsePositives);
Assert.AreEqual(truePositives, target.TruePositives);
Assert.AreEqual(trueNegatives, target.TrueNegatives);
Assert.AreEqual(1.0, target.Accuracy);
Assert.AreEqual(1.0, target.Sensitivity);
Assert.AreEqual(1.0, target.Specificity);
Assert.AreEqual(1.0, target.Efficiency);
// Perfect prediction
Assert.AreEqual(1.0, target.MatthewsCorrelationCoefficient);
}
/// <summary>
/// Constructs a new Fisher's exact test.
/// </summary>
///
/// <param name="matrix">The matrix to be tested.</param>
/// <param name="alternate">The alternative hypothesis (research hypothesis) to test.</param>
///
public FisherExactTest(ConfusionMatrix matrix,
OneSampleHypothesis alternate = OneSampleHypothesis.ValueIsDifferentFromHypothesis)
{
if (matrix == null)
{
throw new ArgumentNullException("matrix");
}
int[,] mat = matrix.Matrix;
int a = mat[0, 0];
int b = mat[0, 1];
int c = mat[1, 0];
int N = matrix.Samples; // total number in population
int k = a; // number of success in sample
int m = a + b; // number of success in population
int n = a + c; // sample size
Compute(k, N, n, m, alternate);
}
public override ConfusionMatrix Execute()
{
//The Least Squares algorithm
//It uses a PartialLeastSquaresAnalysis library object using a non-linear iterative partial least squares algorithm
//and runs on the mean-centered and standardized data
//Create an analysis
var pls = new PartialLeastSquaresAnalysis(trainingSet,
trainingOutput,
AnalysisMethod.Standardize,
PartialLeastSquaresAlgorithm.NIPALS);
pls.Compute();
//After computing the analysis
//create a linear model to predict new variables
MultivariateLinearRegression regression = pls.CreateRegression();
//This will hold the result of the classifications
var predictedLifted = new int[testSet.GetLength(0)][];
for (int i = 0; i < predictedLifted.Length; ++i)
{
predictedLifted[i] = regression
.Compute(testSet.GetRow(i)) //Retrieve the row vector of the test set
.Select(x => Convert.ToInt32(x)) // Convert the result to int
.ToArray();
}
//Unlift the prediction vector
var predicted = predictedLifted
.SelectMany(x => x)
.ToArray();
//Create a new confusion matrix with the calculated parameters
ConfusionMatrix cmatrix = new ConfusionMatrix(predicted, expected, POSITIVE, NEGATIVE);
return(cmatrix);
}
/// <summary>
/// Tests the previously created machine into a new set of data.
/// </summary>
///
private void btnTestingRun_Click(object sender, EventArgs e)
{
if (svm == null || dgvTestingSource.DataSource == null)
{
MessageBox.Show("Please create a machine first.");
return;
}
// Creates a matrix from the source data table
double[,] table = (dgvTestingSource.DataSource as DataTable).ToMatrix();
// Extract the first and second columns (X and Y)
double[][] inputs = table.GetColumns(0, 1).ToArray();
// Extract the expected output labels
int[] expected = table.GetColumn(2).ToInt32();
int[] output = new int[expected.Length];
// Compute the actual machine outputs
for (int i = 0; i < expected.Length; i++)
{
output[i] = svm.Compute(inputs[i]) > 0.5 ? 1 : -1;
}
// Use confusion matrix to compute some performance metrics
ConfusionMatrix confusionMatrix = new ConfusionMatrix(output, expected, 1, -1);
dgvPerformance.DataSource = new[] { confusionMatrix };
// Create performance scatter plot
CreateResultScatterplot(zedGraphControl1, inputs, expected.ToDouble(), output.ToDouble());
}
public static double MatthewsCorrelationCoefficient(this ConfusionMatrix confusionMatrix)
{
if (confusionMatrix.NumberOfClasses == 2)
{
var truePositives = confusionMatrix.GetCountForClassPair(0, 0);
var trueNegatives = confusionMatrix.GetCountForClassPair(1, 1);
var falseNegatives = confusionMatrix.GetCountForClassPair(1, 0);
var falsePositives = confusionMatrix.GetCountForClassPair(0, 1);
var mccNumerator = truePositives * trueNegatives - falsePositives * falseNegatives;
var mccDenominator = Math.Sqrt(
1.0 * (truePositives + falsePositives) * (truePositives + falseNegatives) * (trueNegatives + falsePositives) * (trueNegatives + falseNegatives)
);
var mcc = mccNumerator / mccDenominator;
return(mcc);
}
else
{
return(0.0);
}
}
private static void crf(int[][] trainX, int[][] trainY, int[][] testX, int[][] testY)
{
int numberOfClasses = 44; // chunking.Tags.Length;
int numberOfSymbols = 21589; // chunking.Words.Length;
// Learn one Markov model using the training data
var teacher = new QuasiNewtonLearning <int>()
{
Function = new MarkovDiscreteFunction(states: numberOfClasses, symbols: numberOfSymbols)
};
// Use the teacher to learn a Conditional Random Field model
ConditionalRandomField <int> crf = teacher.Learn(trainX, trainY);
// Use the crf to predict instances:
int[][] predY = crf.Decide(testX);
// Check the accuracy of the model:
var cm = new ConfusionMatrix(predicted: predY.Concatenate(), expected: testY.Concatenate());
double acc = cm.Accuracy;
}
public virtual void TestRealLabels()
{
string expected = " Guess/Gold a b c Marg. (Guess)\n" + " a 2 0 0 2\n" + " b 1 0 0 1\n" + " c 0 0 1 1\n"
+ " Marg. (Gold) 3 0 1\n\n" + " a prec=1, recall=0.66667, spec=1, f1=0.8\n" + " b prec=0, recall=n/a, spec=0.75, f1=n/a\n" + " c prec=1, recall=1, spec=1, f1=1\n";
ConfusionMatrix <string> conf = new ConfusionMatrix <string>(Locale.Us);
conf.SetUseRealLabels(true);
conf.Add("a", "a");
conf.Add("a", "a");
conf.Add("b", "a");
conf.Add("c", "c");
string result = conf.PrintTable();
if (echo)
{
System.Console.Error.WriteLine(result);
}
else
{
NUnit.Framework.Assert.AreEqual(expected, result);
}
}
/// <summary>
/// Tests the previously created tree into a new set of data.
/// </summary>
///
private void btnTestingRun_Click(object sender, EventArgs e)
{
if (tree == null || dgvTestingSource.DataSource == null)
{
MessageBox.Show("Please create a machine first.");
return;
}
// Creates a matrix from the entire source data table
double[][] table = (dgvLearningSource.DataSource as DataTable).ToJagged(out columnNames);
// Get only the input vector values (first two columns)
double[][] inputs = table.GetColumns(0, 1, 2, 3, 4, 5);
// Get the expected output labels (last column)
int[] expected = table.GetColumn(6).ToInt32();
// Compute the actual tree outputs
int[] actual = tree.Decide(inputs);
// Use confusion matrix to compute some statistics.
ConfusionMatrix confusionMatrix = new ConfusionMatrix(actual, expected, 1, 0);
dgvPerformance.DataSource = new[] { confusionMatrix };
// Create performance scatter plot
CreateResultScatterplot(zedGraphControl1, inputs, expected.ToDouble(), actual.ToDouble());
}
private static void decisionTree(double[][] inputs, int[] outputs)
{
var teacher = new C45Learning()
{
// Note: all the settings below are optional
Join = 5, // Variables can participate 5 times in a decision path
MaxVariables = 0, // No limit on the number of variables to consider
MaxHeight = 10 // The tree can have a maximum height of 10
};
// Use the learning algorithm to induce the tree
DecisionTree tree = teacher.Learn(inputs, outputs);
// Classify the samples using the model
int[] predicted = tree.Decide(inputs);
// Create a confusion matrix to check the quality of the predictions:
var cm = new ConfusionMatrix(predicted: predicted, expected: outputs);
// Check the accuracy measure:
double accuracy = cm.Accuracy; // (should be 1.0 or 100%)
}
