public void ClassificationEnsembleModel_PredictProbability_single()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var learner = new ClassificationEnsembleLearner(learners, new MeanProbabilityClassificationEnsembleStrategy());
var sut = learner.Learn(observations, targets);
var rows = targets.Length;
var predictions = new ProbabilityPrediction[rows];
for (int i = 0; i < rows; i++)
{
predictions[i] = sut.PredictProbability(observations.Row(i));
}
var metric = new LogLossClassificationProbabilityMetric();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.32562112824941963, actual, 0.0000001);
}
public void ClassificationStackingEnsembleModel_PredictProbability_single()
{
var parser = new CsvParser(() => new StringReader(Resources.AptitudeData));
var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix();
var targets = parser.EnumerateRows("Pass").ToF64Vector();
var rows = targets.Length;
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var learner = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), false);
var sut = learner.Learn(observations, targets);
var predictions = new ProbabilityPrediction[rows];
for (int i = 0; i < rows; i++)
{
predictions[i] = sut.PredictProbability(observations.Row(i));
}
var metric = new LogLossClassificationProbabilityMetric();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.6696598716465223, actual, 0.0000001);
}
public void LogLossClassificationMetric_ErrorString_TargetStringMapping()
{
var sut = new LogLossClassificationProbabilityMetric(1e-15);
var predictions = new ProbabilityPrediction[] {
new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 }
}),
new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 }
}),
new ProbabilityPrediction(2, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }
}),
};
var targets = new double[] { 0, 1, 2 };
var targetStringMapping = new Dictionary <double, string> {
{ 0, "One" }, { 1, "Two" }, { 2, "Three" }
};
var actual = sut.ErrorString(targets, predictions, targetStringMapping);
var expected = ";One;Two;Three;One;Two;Three\r\nOne;1.000;0.000;0.000;100.000;0.000;0.000\r\nTwo;0.000;1.000;0.000;0.000;100.000;0.000\r\nThree;0.000;0.000;1.000;0.000;0.000;100.000\r\nError: 36.620\r\n";
Assert.AreEqual(expected, actual);
}
public void ClassificationNeuralNetModel_PredictProbability_Single()
{
var numberOfObservations = 500;
var numberOfFeatures = 5;
var numberOfClasses = 5;
var random = new Random(32);
var observations = new F64Matrix(numberOfObservations, numberOfFeatures);
observations.Map(() => random.NextDouble());
var targets = Enumerable.Range(0, numberOfObservations)
.Select(i => (double)random.Next(0, numberOfClasses)).ToArray();
var sut = ClassificationNeuralNetModel.Load(() => new StringReader(m_classificationNeuralNetModelText));
var predictions = new ProbabilityPrediction[numberOfObservations];
for (int i = 0; i < numberOfObservations; i++)
{
predictions[i] = sut.PredictProbability(observations.Row(i));
}
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(targets, predictions.Select(p => p.Prediction).ToArray());
Assert.AreEqual(0.762, actual);
}
public void ClassificationStackingEnsembleModel_PredictProbability_single()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var learner = new ClassificationStackingEnsembleLearner(learners,
new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), false);
var sut = learner.Learn(observations, targets);
var rows = targets.Length;
var predictions = new ProbabilityPrediction[rows];
for (int i = 0; i < rows; i++)
{
predictions[i] = sut.PredictProbability(observations.Row(i));
}
var metric = new LogLossClassificationProbabilityMetric();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.6696598716465223, actual, 0.0000001);
}
public void RocAucClassificationMetric_Error_Not_Binary()
{
var targets = new double[] { 0, 1, 2 };
var probabilities = new ProbabilityPrediction[0];
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.Error(targets, probabilities);
}
public void RocAucClassificationMetric_Error()
{
var targets = new double[] { 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.052380952 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.993377483 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.111111111 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.193377483 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.793377483 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.012345679 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.885860173 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.714285714 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.993377483 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.993377483 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.954545455 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.Error(targets, probabilities);
Assert.AreEqual(0.0085470085470086277, actual, 0.00001);
}
/// <summary>
/// Predicts the observation subset provided by indices with probabilities
/// </summary>
/// <param name="observations"></param>
/// <param name="indices"></param>
/// <returns></returns>
public ProbabilityPrediction[] PredictProbability(F64Matrix observations, int[] indices)
{
var rows = observations.RowCount;
var predictions = new ProbabilityPrediction[indices.Length];
for (int i = 0; i < indices.Length; i++)
{
predictions[i] = Tree.PredictProbability(observations.Row(indices[i]));
}
return(predictions);
}
/// <summary>
/// Predicts a set of observations with probabilities
/// </summary>
/// <param name="observations"></param>
/// <returns></returns>
public ProbabilityPrediction[] PredictProbability(F64Matrix observations)
{
var rows = observations.RowCount;
var predictions = new ProbabilityPrediction[rows];
for (int i = 0; i < rows; i++)
{
predictions[i] = PredictProbability(observations.Row(i));
}
return(predictions);
}
public void RocAucClassificationMetric_Error_No_Error()
{
var targets = new double[] { 0, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0 }, { 1.0, 0.0 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 1 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.Error(targets, probabilities);
Assert.AreEqual(0.0, actual);
}
/// <summary>
/// Predicts a single observation using the ensembled probabilities
/// </summary>
/// <param name="observation"></param>
/// <returns></returns>
public ProbabilityPrediction PredictProbability(double[] observation)
{
var ensembleCols = m_ensembleModels.Length;
var ensemblePredictions = new ProbabilityPrediction[ensembleCols];
for (int i = 0; i < m_ensembleModels.Length; i++)
{
ensemblePredictions[i] = m_ensembleModels[i].Predict(observation);
}
return(m_ensembleStrategy.Combine(ensemblePredictions));
}
/// <summary>
/// Predicts a set of observations using the ensembled probabilities
/// </summary>
/// <param name="observations"></param>
/// <returns></returns>
public ProbabilityPrediction[] PredictProbability(F64Matrix observations)
{
var predictions = new ProbabilityPrediction[observations.RowCount];
var observation = new double[observations.ColumnCount];
for (int i = 0; i < observations.RowCount; i++)
{
observations.Row(i, observation);
predictions[i] = PredictProbability(observation);
}
return(predictions);
}
/// <summary>
/// Geometric mean probability classification ensemble strategy. Class probabilities are combined using the geometric mean across all models.
/// </summary>
/// <param name="ensemblePredictions"></param>
/// <param name="predictions"></param>
public void Combine(ProbabilityPrediction[][] ensemblePredictions, ProbabilityPrediction[] predictions)
{
var currentObservation = new ProbabilityPrediction[ensemblePredictions.Length];
for (int i = 0; i < predictions.Length; i++)
{
for (int j = 0; j < currentObservation.Length; j++)
{
currentObservation[j] = ensemblePredictions[j][i];
}
predictions[i] = Combine(currentObservation);
}
}
public void RandomClassificationEnsembleSelection_Constructor_Number_Of_Availible_Models_Lower_Than_Number_Of_Models_To_Select()
{
var sut = new RandomClassificationEnsembleSelection(
new LogLossClassificationProbabilityMetric(),
new MeanProbabilityClassificationEnsembleStrategy(), 5, 1, true);
var observations = new ProbabilityPrediction[3][];
observations.Select(t => new ProbabilityPrediction[10]).ToArray();
var targets = new double[10];
sut.Select(observations, targets);
}
public void RocAucClassificationMetric_ErrorString()
{
var targets = new double[] { 0, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0 }, { 1.0, 0.0 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 1 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.ErrorString(targets, probabilities);
var expected = ";0;1;0;1\r\n0;1.000;0.000;100.000;0.000\r\n1;0.000;1.000;0.000;100.000\r\nError: 0.000\r\n";
Assert.AreEqual(expected, actual);
}
/// <summary>
///
/// </summary>
/// <param name="observations"></param>
/// <returns></returns>
public ProbabilityPrediction[] PredictProbability(F64Matrix observations)
{
var rows = observations.RowCount;
var cols = observations.ColumnCount;
var observation = new double[cols];
var predictions = new ProbabilityPrediction[rows];
for (int row = 0; row < rows; row++)
{
observations.Row(row, observation);
predictions[row] = PredictProbability(observation);
}
return(predictions);
}
public void RocAucClassificationMetric_ErrorString_TargetStringMapping()
{
var targets = new double[] { 0, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0 }, { 1.0, 0.0 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 1 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var targetStringMapping = new Dictionary <double, string> {
{ 0, "Negative" }, { 1, "Positive" }
};
var actual = sut.ErrorString(targets, probabilities, targetStringMapping);
var expected = ";Negative;Positive;Negative;Positive\r\nNegative;1.000;0.000;100.000;0.000\r\nPositive;0.000;1.000;0.000;100.000\r\nError: 0.000\r\n";
Assert.AreEqual(expected, actual);
}
double SelectNextModelToAdd(ProbabilityPrediction[][] crossValidatedModelPredictions,
double[] targets,
double currentBestError)
{
var rows = crossValidatedModelPredictions.First().Length;
var candidateModelMatrix = new ProbabilityPrediction[m_selectedModelIndices.Count + 1][];
var candidatePredictions = new ProbabilityPrediction[rows];
var candidateModelIndices = new int[m_selectedModelIndices.Count + 1];
var bestError = currentBestError;
var bestIndex = -1;
foreach (var index in m_remainingModelIndices)
{
m_selectedModelIndices.CopyTo(candidateModelIndices);
candidateModelIndices[candidateModelIndices.Length - 1] = index;
for (int i = 0; i < candidateModelIndices.Length; i++)
{
candidateModelMatrix[i] = crossValidatedModelPredictions[candidateModelIndices[i]];
}
m_ensembleStrategy.Combine(candidateModelMatrix, candidatePredictions);
var error = m_metric.Error(targets, candidatePredictions);
if (error < bestError)
{
bestError = error;
bestIndex = index;
}
}
if(bestIndex != -1)
{
m_selectedModelIndices.Add(bestIndex);
if(!m_selectWithReplacement)
{
m_remainingModelIndices.Remove(bestIndex);
}
}
return bestError;
}
double SelectNextModelToRemove(ProbabilityPrediction[][] crossValidatedModelPredictions,
double[] targets,
double currentBestError)
{
var rows = crossValidatedModelPredictions.First().Length;
var candidateModelMatrix = new ProbabilityPrediction[m_remainingModelIndices.Count - 1][];
var candidatePredictions = new ProbabilityPrediction[rows];
var candidateModelIndices = new int[m_remainingModelIndices.Count - 1];
var bestError = currentBestError;
var bestIndex = -1;
foreach (var index in m_remainingModelIndices)
{
var candidateIndex = 0;
for (int i = 0; i < m_remainingModelIndices.Count; i++)
{
var curIndex = m_remainingModelIndices[i];
if (curIndex != index)
{
candidateModelIndices[candidateIndex++] = m_remainingModelIndices[i];
}
}
for (int i = 0; i < candidateModelIndices.Length; i++)
{
candidateModelMatrix[i] = crossValidatedModelPredictions[candidateModelIndices[i]];
}
m_ensembleStrategy.Combine(candidateModelMatrix, candidatePredictions);
var error = m_metric.Error(targets, candidatePredictions);
if (error < bestError)
{
bestError = error;
bestIndex = index;
}
}
m_remainingModelIndices.Remove(bestIndex);
return(bestError);
}
public void ClassificationDecisionTreeModel_PredictProbability_Multiple_Indexed()
{
var parser = new CsvParser(() => new StringReader(Resources.AptitudeData));
var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix();
var targets = parser.EnumerateRows("Pass").ToF64Vector();
var rows = targets.Length;
var learner = new ClassificationDecisionTreeLearner(100, 5, 2, 0.001, 42);
var sut = learner.Learn(observations, targets);
var indices = new int[] { 0, 3, 4, 5, 6, 7, 8, 9, 20, 21 };
var actual = sut.PredictProbability(observations, indices);
var indexedTargets = targets.GetIndices(indices);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(indexedTargets, actual.Select(p => p.Prediction).ToArray());
Assert.AreEqual(0.1, error, 0.0000001);
var expected = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.571428571428571 }, { 1, 0.428571428571429 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.571428571428571 }, { 1, 0.428571428571429 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.428571428571429 }, { 1, 0.571428571428571 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.75 }, { 1, 0.25 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.285714285714286 }, { 1, 0.714285714285714 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.75 }, { 1, 0.25 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.857142857142857 }, { 1, 0.142857142857143 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.285714285714286 }, { 1, 0.714285714285714 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.857142857142857 }, { 1, 0.142857142857143 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.285714285714286 }, { 1, 0.714285714285714 },
}), };
CollectionAssert.AreEqual(expected, actual);
}
/// <summary>
/// Greedy forward selection of ensemble models.
/// </summary>
/// <param name="crossValidatedModelPredictions">cross validated predictions from multiple models.
/// Each row in the matrix corresponds to predictions from a separate model</param>
/// <param name="targets">Corresponding targets</param>
/// <returns>The indices of the selected model</returns>
public int[] Select(ProbabilityPrediction[][] crossValidatedModelPredictions, double[] targets)
{
if (crossValidatedModelPredictions.Length < m_numberOfModelsToSelect)
{
throw new ArgumentException("Availible models: " + crossValidatedModelPredictions.Length +
" is smaller than number of models to select: " + m_numberOfModelsToSelect);
}
m_allIndices = Enumerable.Range(0, crossValidatedModelPredictions.Length).ToArray();
var rows = crossValidatedModelPredictions.First().Length;
var candidateModelMatrix = new ProbabilityPrediction[m_numberOfModelsToSelect][];
var candidatePredictions = new ProbabilityPrediction[rows];
var candidateModelIndices = new int[m_numberOfModelsToSelect];
var bestModelIndices = new int[m_numberOfModelsToSelect];
var bestError = double.MaxValue;
for (int i = 0; i < m_iterations; i++)
{
SelectNextRandomIndices(candidateModelIndices);
for (int j = 0; j < candidateModelIndices.Length; j++)
{
candidateModelMatrix[j] = crossValidatedModelPredictions[candidateModelIndices[j]];
}
m_ensembleStrategy.Combine(candidateModelMatrix, candidatePredictions);
var error = m_metric.Error(targets, candidatePredictions);
if (error < bestError)
{
bestError = error;
candidateModelIndices.CopyTo(bestModelIndices, 0);
Trace.WriteLine("Models selected: " + bestModelIndices.Length + ": " + error);
}
}
Trace.WriteLine("Selected model indices: " + string.Join(", ", bestModelIndices.ToArray()));
return(bestModelIndices);
}
public void LogLossClassificationMetric_Error_1()
{
var sut = new LogLossClassificationProbabilityMetric(1e-15);
var predictions = new ProbabilityPrediction[] {
new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 1.0 }, { 1, 0.0 }, { 2, 0.0 }
}),
new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 }
}),
new ProbabilityPrediction(2, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }
}),
};
var targets = new double[] { 0, 1, 2 };
var actual = sut.Error(targets, predictions);
Assert.AreEqual(9.9920072216264128e-16, actual, 1e-17);
}
public void LogLossClassificationMetric_Error_2()
{
var sut = new LogLossClassificationProbabilityMetric(1e-15);
var predictions = new ProbabilityPrediction[] {
new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 }
}),
new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 }
}),
new ProbabilityPrediction(2, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }
}),
};
var targets = new double[] { 0, 1, 2 };
var actual = sut.Error(targets, predictions);
Assert.AreEqual(0.36620409622270467, actual, 0.0001);
}
public void ClassificationDecisionTreeModel_PredictProbability_Multiple_Indexed()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learner = new ClassificationDecisionTreeLearner(100, 5, 2, 0.001, 42);
var sut = learner.Learn(observations, targets);
var indices = new int[] { 0, 3, 4, 5, 6, 7, 8, 9, 20, 21 };
var actual = sut.PredictProbability(observations, indices);
var indexedTargets = targets.GetIndices(indices);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(indexedTargets, actual.Select(p => p.Prediction).ToArray());
Assert.AreEqual(0.1, error, 0.0000001);
var expected = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.571428571428571 }, { 1, 0.428571428571429 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.571428571428571 }, { 1, 0.428571428571429 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.428571428571429 }, { 1, 0.571428571428571 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.75 }, { 1, 0.25 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.285714285714286 }, { 1, 0.714285714285714 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.75 }, { 1, 0.25 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.857142857142857 }, { 1, 0.142857142857143 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.285714285714286 }, { 1, 0.714285714285714 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.857142857142857 }, { 1, 0.142857142857143 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.285714285714286 }, { 1, 0.714285714285714 },
}), };
CollectionAssert.AreEqual(expected, actual);
}
public void LogLossClassificationMetric_ErrorString()
{
var sut = new LogLossClassificationProbabilityMetric(1e-15);
var predictions = new ProbabilityPrediction[] {
new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 }
}),
new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 }
}),
new ProbabilityPrediction(2, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }
}),
};
var targets = new double[] { 0, 1, 2 };
var actual = sut.ErrorString(targets, predictions);
var expected = ";0;1;2;0;1;2\r\n0;1.000;0.000;0.000;100.000;0.000;0.000\r\n1;0.000;1.000;0.000;0.000;100.000;0.000\r\n2;0.000;0.000;1.000;0.000;0.000;100.000\r\nError: 36.620\r\n";
Assert.AreEqual(expected, actual);
}
public void MeanProbabilityClassificationEnsembleStrategy_Combine()
{
var values = new ProbabilityPrediction[]
{
new ProbabilityPrediction(1.0, new Dictionary <double, double> {
{ 0.0, 0.3 }, { 1.0, 0.88 }
}),
new ProbabilityPrediction(0.0, new Dictionary <double, double> {
{ 0.0, 0.66 }, { 1.0, 0.33 }
}),
new ProbabilityPrediction(1.0, new Dictionary <double, double> {
{ 0.0, 0.01 }, { 1.0, 0.99 }
}),
};
var sut = new MeanProbabilityClassificationEnsembleStrategy();
var actual = sut.Combine(values);
var expected = new ProbabilityPrediction(1.0, new Dictionary <double, double> {
{ 0.0, 0.323333333333333 }, { 1.0, 0.733333333333333 }
});
Assert.AreEqual(expected, actual);
}
public void GeometricMeanProbabilityClassificationEnsembleStrategy_Combine()
{
var values = new ProbabilityPrediction[]
{
new ProbabilityPrediction(1.0, new Dictionary <double, double> {
{ 0.0, 0.3 }, { 1.0, 0.88 }
}),
new ProbabilityPrediction(0.0, new Dictionary <double, double> {
{ 0.0, 0.66 }, { 1.0, 0.33 }
}),
new ProbabilityPrediction(1.0, new Dictionary <double, double> {
{ 0.0, 0.01 }, { 1.0, 0.99 }
}),
};
var sut = new GeometricMeanProbabilityClassificationEnsembleStrategy();
var actual = sut.Combine(values);
var expected = new ProbabilityPrediction(1.0, new Dictionary <double, double> {
{ 0.0, 0.159846490962181 }, { 1.0, 0.840153509037819 }
});
Assert.AreEqual(expected, actual);
}
public void ClassificationAdaBoostModel_PredictProbability_Single()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learner = new ClassificationAdaBoostLearner(10, 1, 3);
var sut = learner.Learn(observations, targets);
var rows = targets.Length;
var actual = new ProbabilityPrediction[rows];
for (int i = 0; i < rows; i++)
{
actual[i] = sut.PredictProbability(observations.Row(i));
}
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, actual.Select(p => p.Prediction).ToArray());
Assert.AreEqual(0.038461538461538464, error, 0.0000001);
var expected = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.553917222019051 }, { 1, 0.446082777980949 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.455270122123639 }, { 1, 0.544729877876361 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.590671208378385 }, { 1, 0.409328791621616 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.564961572849738 }, { 1, 0.435038427150263 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.455270122123639 }, { 1, 0.544729877876361 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.549970403132686 }, { 1, 0.450029596867314 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.417527839140627 }, { 1, 0.582472160859373 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.409988559960094 }, { 1, 0.590011440039906 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.630894242807786 }, { 1, 0.369105757192214 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.436954866525023 }, { 1, 0.563045133474978 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.461264944069783 }, { 1, 0.538735055930217 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.590671208378385 }, { 1, 0.409328791621616 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.549503146925505 }, { 1, 0.450496853074495 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.537653803214063 }, { 1, 0.462346196785938 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.37650723540928 }, { 1, 0.62349276459072 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.573579890413618 }, { 1, 0.426420109586382 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.549970403132686 }, { 1, 0.450029596867314 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.524371409810479 }, { 1, 0.475628590189522 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.436954866525023 }, { 1, 0.563045133474978 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.471117379964633 }, { 1, 0.528882620035367 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.630894242807786 }, { 1, 0.369105757192214 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.436954866525023 }, { 1, 0.563045133474978 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.404976804073458 }, { 1, 0.595023195926542 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.573579890413618 }, { 1, 0.426420109586382 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.549970403132686 }, { 1, 0.450029596867314 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.630894242807786 }, { 1, 0.369105757192214 },
}), };
CollectionAssert.AreEqual(expected, actual);
}
public void ClassificationGradientBoostModel_PredictProbability_Single()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learner = new ClassificationGradientBoostLearner(100, 0.1, 3, 1, 1e-6, 1, 0, new GradientBoostBinomialLoss(), false);
var sut = learner.Learn(observations, targets);
var rows = targets.Length;
var actual = new ProbabilityPrediction[rows];
for (int i = 0; i < rows; i++)
{
actual[i] = sut.PredictProbability(observations.Row(i));
}
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, actual.Select(p => p.Prediction).ToArray());
Assert.AreEqual(0.038461538461538464, error, 0.0000001);
var expected = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00153419685769873 }, { 0, 0.998465803142301 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.497135615200052 }, { 0, 0.502864384799948 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00674291737944022 }, { 0, 0.99325708262056 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00153419685769873 }, { 0, 0.998465803142301 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.497135615200052 }, { 0, 0.502864384799948 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00428497228545111 }, { 0, 0.995715027714549 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.987907185249206 }, { 0, 0.0120928147507945 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.982783250692275 }, { 0, 0.0172167493077254 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00262490179961228 }, { 0, 0.997375098200388 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.996417847055106 }, { 0, 0.00358215294489364 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.995341658753364 }, { 0, 0.00465834124663571 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00674291737944022 }, { 0, 0.99325708262056 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.0118633115475969 }, { 0, 0.988136688452403 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00048646805791186 }, { 0, 0.999513531942088 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.999891769651047 }, { 0, 0.000108230348952856 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00334655581934884 }, { 0, 0.996653444180651 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00428497228545111 }, { 0, 0.995715027714549 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.0118633115475969 }, { 0, 0.988136688452403 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.996417847055106 }, { 0, 0.00358215294489362 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.993419876193791 }, { 0, 0.00658012380620933 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00262490179961228 }, { 0, 0.997375098200388 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.996417847055106 }, { 0, 0.00358215294489362 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 1, 0.988568859753437 }, { 0, 0.0114311402465632 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00334655581934884 }, { 0, 0.996653444180651 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00428497228545111 }, { 0, 0.995715027714549 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 1, 0.00262490179961228 }, { 0, 0.997375098200388 },
}), };
CollectionAssert.AreEqual(expected, actual);
}
public void ClassificationForestModel_PredictProbability_Single()
{
var parser = new CsvParser(() => new StringReader(Resources.AptitudeData));
var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix();
var targets = parser.EnumerateRows("Pass").ToF64Vector();
var rows = targets.Length;
var learner = new ClassificationRandomForestLearner(100, 1, 100, 1, 0.0001, 1.0, 42, false);
var sut = learner.Learn(observations, targets);
var actual = new ProbabilityPrediction[rows];
for (int i = 0; i < rows; i++)
{
actual[i] = sut.PredictProbability(observations.Row(i));
}
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, actual.Select(p => p.Prediction).ToArray());
Assert.AreEqual(0.076923076923076927, error, m_delta);
var expected = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.650149027443145 }, { 1, 0.349850972556855 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.566943847818848 }, { 1, 0.433056152181152 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.726936489980608 }, { 1, 0.273063510019392 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.752781908451026 }, { 1, 0.247218091548974 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.566943847818848 }, { 1, 0.433056152181152 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.792506836300954 }, { 1, 0.207493163699046 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.491736055611056 }, { 1, 0.508263944388944 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.574583315377433 }, { 1, 0.425416684622567 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.838724674018791 }, { 1, 0.161275325981208 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.241480824730825 }, { 1, 0.758519175269175 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.385258186258186 }, { 1, 0.614741813741813 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.726936489980608 }, { 1, 0.273063510019392 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.706733044733045 }, { 1, 0.293266955266955 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.801266011766012 }, { 1, 0.198733988233988 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.294952297702298 }, { 1, 0.705047702297702 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.821706914001031 }, { 1, 0.178293085998968 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.780062391856509 }, { 1, 0.21993760814349 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.554444388944389 }, { 1, 0.445555611055611 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.261349872349872 }, { 1, 0.738650127650127 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.419758186258186 }, { 1, 0.580241813741813 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.71382231249143 }, { 1, 0.28617768750857 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.241480824730825 }, { 1, 0.758519175269175 },
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.47562148962149 }, { 1, 0.52437851037851 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.821706914001031 }, { 1, 0.178293085998968 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.792506836300954 }, { 1, 0.207493163699046 },
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.666244987039105 }, { 1, 0.333755012960895 },
}) };
CollectionAssert.AreEqual(expected, actual);
}
