public void ClassificationStackingEnsembleModel_PredictProbability_single() { var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet(); var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9) }; var learner = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9), new RandomCrossValidation <ProbabilityPrediction>(5, 23), false); var sut = learner.Learn(observations, targets); var rows = targets.Length; var predictions = new ProbabilityPrediction[rows]; for (int i = 0; i < rows; i++) { predictions[i] = sut.PredictProbability(observations.Row(i)); } var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.6696598716465223, actual, 0.0000001); }
public void ClassificationForwardSearchModelSelectingEnsembleLearner_Learn_Start_With_3_Models() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var sut = new ClassificationForwardSearchModelSelectingEnsembleLearner(learners, 5, new StratifiedCrossValidation <ProbabilityPrediction>(5, 23), ensembleStrategy, metric, 3, true); var parser = new CsvParser(() => new StringReader(Resources.Glass)); var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix(); var targets = parser.EnumerateRows("Target").ToF64Vector(); var model = sut.Learn(observations, targets); var predictions = model.PredictProbability(observations); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.54434276244488244, actual, 0.0001); }
public void ClassificationModelSelectingEnsembleLearner_Learn_Start_With_3_Models() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var ensembleSelection = new ForwardSearchClassificationEnsembleSelection(metric, ensembleStrategy, 5, 3, true); var sut = new ClassificationModelSelectingEnsembleLearner(learners, new RandomCrossValidation <ProbabilityPrediction>(5, 23), ensembleStrategy, ensembleSelection); var(observations, targets) = DataSetUtilities.LoadGlassDataSet(); var model = sut.Learn(observations, targets); var predictions = model.PredictProbability(observations); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.55183985816428427, actual, 0.0001); }
public void ClassificationBackwardEliminationModelSelectingEnsembleLearner_Learn_Indexed() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var sut = new ClassificationBackwardEliminationModelSelectingEnsembleLearner(learners, 5, new RandomCrossValidation <ProbabilityPrediction>(5, 23), ensembleStrategy, metric); var(observations, targets) = DataSetUtilities.LoadGlassDataSet(); var indices = Enumerable.Range(0, 25).ToArray(); var model = sut.Learn(observations, targets, indices); var predictions = model.PredictProbability(observations); var actual = metric.Error(targets, predictions); Assert.AreEqual(2.3682546920482164, actual, 0.0001); }
public void ClassificationEnsembleModel_PredictProbability_Multiple() { var parser = new CsvParser(() => new StringReader(Resources.AptitudeData)); var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix(); var targets = parser.EnumerateRows("Pass").ToF64Vector(); var rows = targets.Length; var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9) }; var learner = new ClassificationEnsembleLearner(learners, new MeanProbabilityClassificationEnsembleStrategy()); var sut = learner.Learn(observations, targets); var predictions = sut.PredictProbability(observations); var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.32562112824941963, actual, 0.0000001); }
public void ClassificationBackwardEliminationModelSelectingEnsembleLearner_CreateMetaFeatures_Then_Learn() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var sut = new ClassificationBackwardEliminationModelSelectingEnsembleLearner(learners, 5); var parser = new CsvParser(() => new StringReader(Resources.Glass)); var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix(); var targets = parser.EnumerateRows("Target").ToF64Vector(); var metaObservations = sut.LearnMetaFeatures(observations, targets); var model = sut.SelectModels(observations, metaObservations, targets); var predictions = model.PredictProbability(observations); var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.52351727716455632, actual, 0.0001); }
public void LogLossClassificationMetric_ErrorString_TargetStringMapping() { var sut = new LogLossClassificationProbabilityMetric(1e-15); var predictions = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> { { 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 } }), new ProbabilityPrediction(1, new Dictionary <double, double> { { 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 } }), new ProbabilityPrediction(2, new Dictionary <double, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 } }), }; var targets = new double[] { 0, 1, 2 }; var targetStringMapping = new Dictionary <double, string> { { 0, "One" }, { 1, "Two" }, { 2, "Three" } }; var actual = sut.ErrorString(targets, predictions, targetStringMapping); var expected = ";One;Two;Three;One;Two;Three\r\nOne;1.000;0.000;0.000;100.000;0.000;0.000\r\nTwo;0.000;1.000;0.000;0.000;100.000;0.000\r\nThree;0.000;0.000;1.000;0.000;0.000;100.000\r\nError: 36.620\r\n"; Assert.AreEqual(expected, actual); }
public void ClassificationStackingEnsembleModel_PredictProbability_single() { var parser = new CsvParser(() => new StringReader(Resources.AptitudeData)); var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix(); var targets = parser.EnumerateRows("Pass").ToF64Vector(); var rows = targets.Length; var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9) }; var learner = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9), new RandomCrossValidation <ProbabilityPrediction>(5, 23), false); var sut = learner.Learn(observations, targets); var predictions = new ProbabilityPrediction[rows]; for (int i = 0; i < rows; i++) { predictions[i] = sut.PredictProbability(observations.Row(i)); } var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.6696598716465223, actual, 0.0000001); }
public void ClassificationBackwardEliminationModelSelectingEnsembleLearner_CreateMetaFeatures_Then_Learn() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var sut = new ClassificationBackwardEliminationModelSelectingEnsembleLearner(learners, 5); var(observations, targets) = DataSetUtilities.LoadGlassDataSet(); var metaObservations = sut.LearnMetaFeatures(observations, targets); var model = sut.SelectModels(observations, metaObservations, targets); var predictions = model.PredictProbability(observations); var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.52351727716455632, actual, 0.0001); }
public void ClassificationForwardSearchModelSelectingEnsembleLearner_Learn() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var sut = new ClassificationForwardSearchModelSelectingEnsembleLearner(learners, 5); var parser = new CsvParser(() => new StringReader(Resources.Glass)); var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix(); var targets = parser.EnumerateRows("Target").ToF64Vector(); var model = sut.Learn(observations, targets); var predictions = model.PredictProbability(observations); var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.51787562976713208, actual, 0.0001); }
public void ClassificationRandomModelSelectingEnsembleLearner_Learn_Without_Replacement() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var sut = new ClassificationRandomModelSelectingEnsembleLearner(learners, 5, new StratifiedCrossValidation <ProbabilityPrediction>(5, 23), ensembleStrategy, metric, 1, false); var(observations, targets) = DataSetUtilities.LoadGlassDataSet(); var model = sut.Learn(observations, targets); var predictions = model.PredictProbability(observations); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.5805783545646459, actual, 0.0001); }
public void ClassificationRandomModelSelectingEnsembleLearner_Learn() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var sut = new ClassificationRandomModelSelectingEnsembleLearner(learners, 5); var(observations, targets) = DataSetUtilities.LoadGlassDataSet(); var model = sut.Learn(observations, targets); var predictions = model.PredictProbability(observations); var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.60969181130388794, actual, 0.0001); }
public void ClassificationEnsembleModel_PredictProbability_single() { var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet(); var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9) }; var learner = new ClassificationEnsembleLearner(learners, new MeanProbabilityClassificationEnsembleStrategy()); var sut = learner.Learn(observations, targets); var rows = targets.Length; var predictions = new ProbabilityPrediction[rows]; for (int i = 0; i < rows; i++) { predictions[i] = sut.PredictProbability(observations.Row(i)); } var metric = new LogLossClassificationProbabilityMetric(); var actual = metric.Error(targets, predictions); Assert.AreEqual(0.32562112824941963, actual, 0.0000001); }
public void ClassificationModelSelectingEnsembleLearner_Constructor_CrossValidation_Null() { var learners = new IIndexedLearner <ProbabilityPrediction> [4]; var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var ensembleSelection = new ForwardSearchClassificationEnsembleSelection(metric, ensembleStrategy, 5, 1, true); var sut = new ClassificationModelSelectingEnsembleLearner(learners, null, ensembleStrategy, ensembleSelection); }
public void ClassificationModelSelectingEnsembleLearner_Constructor_EnsembleSelection_Null() { var learners = new IIndexedLearner <ProbabilityPrediction> [4]; var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var crossValidation = new RandomCrossValidation <ProbabilityPrediction>(5); var sut = new ClassificationModelSelectingEnsembleLearner(learners, crossValidation, ensembleStrategy, null); }
public void ClassificationModelSelectingEnsembleLearner_Constructor_Learners_Null() { var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var ensembleSelection = new ForwardSearchClassificationEnsembleSelection(metric, ensembleStrategy, 5, 1, true); var crossValidation = new RandomCrossValidation <ProbabilityPrediction>(5); var sut = new ClassificationModelSelectingEnsembleLearner(null, crossValidation, ensembleStrategy, ensembleSelection); }
public void ClassificationModel_PredictProbability_Threshold_On_Probability() { #region learner creation // Use StreamReader(filepath) when running from filesystem var parser = new CsvParser(() => new StringReader(Resources.winequality_white)); var targetName = "quality"; // read feature matrix var observations = parser.EnumerateRows(c => c != targetName) .ToF64Matrix(); // read classification targets and convert to binary problem (low quality/high quality). var targets = parser.EnumerateRows(targetName) .ToF64Vector().Select(t => t < 5 ? 0.0 : 1.0).ToArray(); var translation = new Dictionary <double, string> { { 0.0, "Low quality" }, { 1.0, "High quality" } }; // create learner var learner = new ClassificationDecisionTreeLearner(maximumTreeDepth: 5); #endregion // learns a ClassificationDecisionTreeModel var model = learner.Learn(observations, targets); // predict probabilities for all observations var probabilityPredictions = model.PredictProbability(observations); // zip target and probabilities to keep order var zip = targets.Zip(probabilityPredictions, (t, p) => new { Target = t, Prediction = p }); // threhold on the probabilty of the predicted class. // This will remove the obserations that the model is uncertain about. var probabilityThreshold = 0.90; var thresholdedResult = zip.Where(kvp => kvp.Prediction.Probabilities[kvp.Prediction.Prediction] > probabilityThreshold); // evaluate the resulting observations var thresholdedPredictions = thresholdedResult.Select(p => p.Prediction).ToArray(); var thresholdedTargets = thresholdedResult.Select(p => p.Target).ToArray(); // evaluate only on probability thresholded data var metric = new LogLossClassificationProbabilityMetric(); Trace.WriteLine("ProbabilityThresholded Result:"); Trace.WriteLine(metric.ErrorString(thresholdedTargets, thresholdedPredictions, translation)); Trace.WriteLine(""); // evaluate on all data for comparison Trace.WriteLine("All data result:"); Trace.WriteLine(metric.ErrorString(targets, probabilityPredictions, translation)); }
public void LearningCurves_Calculate_ProbabilityPrediction() { #region Read data // Use StreamReader(filepath) when running from filesystem var parser = new CsvParser(() => new StringReader(Resources.winequality_white)); var targetName = "quality"; // read feature matrix var observations = parser.EnumerateRows(c => c != targetName) .ToF64Matrix(); // read classification targets and convert to binary problem (low quality/high quality). var targets = parser.EnumerateRows(targetName) .ToF64Vector().Select(t => t < 5 ? 0.0 : 1.0).ToArray(); #endregion // metric for measuring model error var metric = new LogLossClassificationProbabilityMetric(); // creates cross validator, observations are shuffled randomly var learningCurveCalculator = new RandomShuffleLearningCurvesCalculator <ProbabilityPrediction>(metric, samplePercentages: new double[] { 0.05, 0.1, 0.2, 0.4, 0.8, 1.0 }, trainingPercentage: 0.7, numberOfShufflesPrSample: 5); // create learner var learner = new ClassificationDecisionTreeLearner(maximumTreeDepth: 5); // calculate learning curve var learningCurve = learningCurveCalculator.Calculate(learner, observations, targets); // write to csv var writer = new StringWriter(); learningCurve.Write(() => writer); // trace result // Plotting the learning curves will help determine if the model has high bias or high variance. // This information can be used to determine what to try next in order to improve the model. Trace.WriteLine(writer.ToString()); // alternatively, write to file //learningCurve.Write(() => new StreamWriter(filePath)); }
public void LogLossClassificationMetric_Error_2() { var sut = new LogLossClassificationProbabilityMetric(1e-15); var predictions = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> { { 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 } }), new ProbabilityPrediction(1, new Dictionary <double, double> { { 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 } }), new ProbabilityPrediction(2, new Dictionary <double, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 } }), }; var targets = new double[] { 0, 1, 2 }; var actual = sut.Error(targets, predictions); Assert.AreEqual(0.36620409622270467, actual, 0.0001); }
public void LogLossClassificationMetric_Error_1() { var sut = new LogLossClassificationProbabilityMetric(1e-15); var predictions = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> { { 0, 1.0 }, { 1, 0.0 }, { 2, 0.0 } }), new ProbabilityPrediction(1, new Dictionary <double, double> { { 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 } }), new ProbabilityPrediction(2, new Dictionary <double, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 } }), }; var targets = new double[] { 0, 1, 2 }; var actual = sut.Error(targets, predictions); Assert.AreEqual(9.9920072216264128e-16, actual, 1e-17); }
public void LogLossClassificationMetric_ErrorString() { var sut = new LogLossClassificationProbabilityMetric(1e-15); var predictions = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> { { 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 } }), new ProbabilityPrediction(1, new Dictionary <double, double> { { 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 } }), new ProbabilityPrediction(2, new Dictionary <double, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 } }), }; var targets = new double[] { 0, 1, 2 }; var actual = sut.ErrorString(targets, predictions); var expected = ";0;1;2;0;1;2\r\n0;1.000;0.000;0.000;100.000;0.000;0.000\r\n1;0.000;1.000;0.000;0.000;100.000;0.000\r\n2;0.000;0.000;1.000;0.000;0.000;100.000\r\nError: 36.620\r\n"; Assert.AreEqual(expected, actual); }
public void CrossValidation_CrossValidate_ProbabilityPredictions() { #region Read data // Use StreamReader(filepath) when running from filesystem var parser = new CsvParser(() => new StringReader(Resources.winequality_white)); var targetName = "quality"; // read feature matrix (all columns different from the targetName) var observations = parser.EnumerateRows(c => c != targetName) .ToF64Matrix(); // read targets var targets = parser.EnumerateRows(targetName) .ToF64Vector(); #endregion // creates cross validator, observations are shuffled randomly var cv = new RandomCrossValidation <ProbabilityPrediction>(crossValidationFolds: 5, seed: 42); // create learner var learner = new ClassificationDecisionTreeLearner(); // cross-validated predictions var cvPredictions = cv.CrossValidate(learner, observations, targets); // metric for measuring model error var metric = new LogLossClassificationProbabilityMetric(); // cross-validation provides an estimate on how the model will perform on unseen data Trace.WriteLine("Cross-validation error: " + metric.Error(targets, cvPredictions)); // train and predict training set for comparison var predictions = learner.Learn(observations, targets).PredictProbability(observations); // The training set is NOT a good estimate of how well the model will perfrom on unseen data. Trace.WriteLine("Training error: " + metric.Error(targets, predictions)); }
public void ClassificationModelSelectingEnsembleLearner_Learn_Indexed() { var learners = new IIndexedLearner <ProbabilityPrediction>[] { new ClassificationDecisionTreeLearner(2), new ClassificationDecisionTreeLearner(5), new ClassificationDecisionTreeLearner(7), new ClassificationDecisionTreeLearner(9), new ClassificationDecisionTreeLearner(11), new ClassificationDecisionTreeLearner(21), new ClassificationDecisionTreeLearner(23), new ClassificationDecisionTreeLearner(1), new ClassificationDecisionTreeLearner(14), new ClassificationDecisionTreeLearner(17), new ClassificationDecisionTreeLearner(19), new ClassificationDecisionTreeLearner(33) }; var metric = new LogLossClassificationProbabilityMetric(); var ensembleStrategy = new MeanProbabilityClassificationEnsembleStrategy(); var ensembleSelection = new ForwardSearchClassificationEnsembleSelection(metric, ensembleStrategy, 5, 1, true); var sut = new ClassificationModelSelectingEnsembleLearner(learners, new RandomCrossValidation <ProbabilityPrediction>(5, 23), ensembleStrategy, ensembleSelection); var parser = new CsvParser(() => new StringReader(Resources.Glass)); var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix(); var targets = parser.EnumerateRows("Target").ToF64Vector(); var indices = Enumerable.Range(0, 25).ToArray(); var model = sut.Learn(observations, targets, indices); var predictions = model.PredictProbability(observations); var actual = metric.Error(targets, predictions); Assert.AreEqual(2.3682546920482164, actual, 0.0001); }