public void ClassificationNeuralNetModel_Save()
{
var numberOfObservations = 500;
var numberOfFeatures = 5;
var numberOfClasses = 5;
var random = new Random(32);
var observations = new F64Matrix(numberOfObservations, numberOfFeatures);
observations.Map(() => random.NextDouble());
var targets = Enumerable.Range(0, numberOfObservations).Select(i => (double)random.Next(0, numberOfClasses)).ToArray();
var net = new NeuralNet();
net.Add(new InputLayer(numberOfFeatures));
net.Add(new DenseLayer(10));
net.Add(new SvmLayer(numberOfClasses));
var learner = new ClassificationNeuralNetLearner(net, new AccuracyLoss());
var sut = learner.Learn(observations, targets);
// save model.
var writer = new StringWriter();
sut.Save(() => writer);
// load model and assert prediction results.
sut = ClassificationNeuralNetModel.Load(() => new StringReader(writer.ToString()));
var predictions = sut.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(targets, predictions);
Assert.AreEqual(0.762, actual, 0.0000001);
}
public void ClassificationStackingEnsembleLearner_CreateMetaFeatures_Then_Learn()
{
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var sut = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), false);
var parser = new CsvParser(() => new StringReader(Resources.Glass));
var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix();
var targets = parser.EnumerateRows("Target").ToF64Vector();
var metaObservations = sut.LearnMetaFeatures(observations, targets);
var model = sut.LearnStackingModel(observations, metaObservations, targets);
var predictions = model.Predict(observations);
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.63551401869158874, actual, 0.0001);
}
public void ClassificationEnsembleModel_Predict_single()
{
var parser = new CsvParser(() => new StringReader(Resources.AptitudeData));
var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix();
var targets = parser.EnumerateRows("Pass").ToF64Vector();
var rows = targets.Length;
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var learner = new ClassificationEnsembleLearner(learners, new MeanProbabilityClassificationEnsembleStrategy());
var sut = learner.Learn(observations, targets);
var predictions = new double[rows];
for (int i = 0; i < rows; i++)
{
predictions[i] = sut.Predict(observations.Row(i));
}
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.076923076923076927, actual, 0.0000001);
}
public void ClassificationStackingEnsembleLearner_CreateMetaFeatures_Then_Learn()
{
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var sut = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), false);
var(observations, targets) = DataSetUtilities.LoadGlassDataSet();
var metaObservations = sut.LearnMetaFeatures(observations, targets);
var model = sut.LearnStackingModel(observations, metaObservations, targets);
var predictions = model.Predict(observations);
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.63551401869158874, actual, 0.0001);
}
public void ClassificationStackingEnsembleLearner_Learn_Indexed()
{
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var sut = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), false);
var parser = new CsvParser(() => new StringReader(Resources.Glass));
var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix();
var targets = parser.EnumerateRows("Target").ToF64Vector();
var indices = Enumerable.Range(0, 25).ToArray();
var model = sut.Learn(observations, targets, indices);
var predictions = model.Predict(observations);
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.67289719626168221, actual, 0.0001);
}
public void ClassificationNeuralNetLearner_Learn_Early_Stopping()
{
var numberOfObservations = 500;
var numberOfFeatures = 5;
var numberOfClasses = 5;
var random = new Random(32);
var(observations, targets) = CreateData(numberOfObservations,
numberOfFeatures, numberOfClasses, random);
var(validationObservations, validationTargets) = CreateData(numberOfObservations,
numberOfFeatures, numberOfClasses, random);
var net = new NeuralNet();
net.Add(new InputLayer(numberOfFeatures));
net.Add(new DenseLayer(10));
net.Add(new SvmLayer(numberOfClasses));
var sut = new ClassificationNeuralNetLearner(net, new AccuracyLoss());
var model = sut.Learn(observations, targets,
validationObservations, validationTargets);
var validationPredictions = model.Predict(validationObservations);
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(validationTargets, validationPredictions);
Assert.AreEqual(0.798, actual);
}
public void ClassificationNeuralNetModel_PredictProbability_Single()
{
var numberOfObservations = 500;
var numberOfFeatures = 5;
var numberOfClasses = 5;
var random = new Random(32);
var observations = new F64Matrix(numberOfObservations, numberOfFeatures);
observations.Map(() => random.NextDouble());
var targets = Enumerable.Range(0, numberOfObservations)
.Select(i => (double)random.Next(0, numberOfClasses)).ToArray();
var sut = ClassificationNeuralNetModel.Load(() => new StringReader(m_classificationNeuralNetModelText));
var predictions = new ProbabilityPrediction[numberOfObservations];
for (int i = 0; i < numberOfObservations; i++)
{
predictions[i] = sut.PredictProbability(observations.Row(i));
}
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(targets, predictions.Select(p => p.Prediction).ToArray());
Assert.AreEqual(0.762, actual);
}
public void ClassificationNeuralNetLearner_Learn()
{
var numberOfObservations = 500;
var numberOfFeatures = 5;
var numberOfClasses = 5;
var random = new Random(32);
F64Matrix observations;
double[] targets;
CreateData(numberOfObservations, numberOfFeatures, numberOfClasses, random, out observations, out targets);
var net = new NeuralNet();
net.Add(new InputLayer(numberOfFeatures));
net.Add(new DenseLayer(10));
net.Add(new SvmLayer(numberOfClasses));
var sut = new ClassificationNeuralNetLearner(net, new AccuracyLoss());
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(targets, predictions);
Assert.AreEqual(0.762, actual);
}
public void ClassificationEnsembleModel_Predict_single()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var learner = new ClassificationEnsembleLearner(learners, new MeanProbabilityClassificationEnsembleStrategy());
var sut = learner.Learn(observations, targets);
var rows = targets.Length;
var predictions = new double[rows];
for (int i = 0; i < rows; i++)
{
predictions[i] = sut.Predict(observations.Row(i));
}
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.076923076923076927, actual, 0.0000001);
}
public void ClassificationEnsembleLearner_Learn_Indexed()
{
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var sut = new ClassificationEnsembleLearner(learners,
new MeanProbabilityClassificationEnsembleStrategy());
var(observations, targets) = DataSetUtilities.LoadGlassDataSet();
var indices = Enumerable.Range(0, 25).ToArray();
var model = sut.Learn(observations, targets, indices);
var predictions = model.Predict(observations);
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.67289719626168221, actual, 0.0001);
}
public void ClassificationStackingEnsembleModel_Predict_Multiple()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var learner = new ClassificationStackingEnsembleLearner(learners,
new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), false);
var sut = learner.Learn(observations, targets);
var predictions = sut.Predict(observations);
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.34615384615384615, actual, 0.0000001);
}
public void ClassificationStackingEnsembleModel_Predict_Multiple()
{
var parser = new CsvParser(() => new StringReader(Resources.AptitudeData));
var observations = parser.EnumerateRows(v => v != "Pass").ToF64Matrix();
var targets = parser.EnumerateRows("Pass").ToF64Vector();
var rows = targets.Length;
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var learner = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), false);
var sut = learner.Learn(observations, targets);
var predictions = sut.Predict(observations);
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.34615384615384615, actual, 0.0000001);
}
public void ClassificationStackingEnsembleLearner_Learn_Include_Original_Features()
{
var learners = new IIndexedLearner <ProbabilityPrediction>[]
{
new ClassificationDecisionTreeLearner(2),
new ClassificationDecisionTreeLearner(5),
new ClassificationDecisionTreeLearner(7),
new ClassificationDecisionTreeLearner(9)
};
var sut = new ClassificationStackingEnsembleLearner(learners, new ClassificationDecisionTreeLearner(9),
new RandomCrossValidation <ProbabilityPrediction>(5, 23), true);
var parser = new CsvParser(() => new StringReader(Resources.Glass));
var observations = parser.EnumerateRows(v => v != "Target").ToF64Matrix();
var targets = parser.EnumerateRows("Target").ToF64Vector();
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var metric = new TotalErrorClassificationMetric <double>();
var actual = metric.Error(targets, predictions);
Assert.AreEqual(0.26168224299065418, actual, 0.0001);
}
public void ClassificationMetric_Error()
{
var targets = new double[] { 1, 2, 2, 2, 3, 1, 1, 2, 3 };
var predictions = new double[] { 1, 2, 2, 2, 1, 2, 2, 1, 3 };
var metric = new TotalErrorClassificationMetric <double>();
Trace.WriteLine("Error: " + metric.Error(targets, predictions));
}
public void Classification_Neural_Net_Using_ValidtionSet_For_Selecting_The_best_Model()
{
#region Read Data
// Use StreamReader(filepath) when running from filesystem
var trainingParser = new CsvParser(() => new StringReader(Resources.mnist_small_train));
var testParser = new CsvParser(() => new StringReader(Resources.mnist_small_test));
var targetName = "Class";
var featureNames = trainingParser.EnumerateRows(c => c != targetName).First().ColumnNameToIndex.Keys.ToArray();
// read feature matrix (training)
var trainingObservations = trainingParser
.EnumerateRows(featureNames)
.ToF64Matrix();
// read classification targets (training)
var trainingTargets = trainingParser.EnumerateRows(targetName)
.ToF64Vector();
// read feature matrix (test)
var testObservations = testParser
.EnumerateRows(featureNames)
.ToF64Matrix();
// read classification targets (test)
var testTargets = testParser.EnumerateRows(targetName)
.ToF64Vector();
#endregion
// transform pixel values to be between 0 and 1.
trainingObservations.Map(p => p / 255);
testObservations.Map(p => p / 255);
// create training validation split
var splitter = new StratifiedTrainingTestIndexSplitter <double>(trainingPercentage: 0.7, seed: 24);
var split = splitter.SplitSet(trainingObservations, trainingTargets);
// the output layer must know the number of classes.
var numberOfClasses = trainingTargets.Distinct().Count();
var net = new NeuralNet();
net.Add(new InputLayer(width: 28, height: 28, depth: 1)); // MNIST data is 28x28x1.
net.Add(new DenseLayer(800, Activation.Relu));
net.Add(new SoftMaxLayer(numberOfClasses));
// using classification accuracy as error metric.
// When using a validation set, the error metric
// is used for selecting the best iteration based on models error on the validation set.
var learner = new ClassificationNeuralNetLearner(net, iterations: 10, loss: new AccuracyLoss());
var model = learner.Learn(split.TrainingSet.Observations, split.TrainingSet.Targets, //);
split.TestSet.Observations, split.TestSet.Targets); // the validation set for estimating how well the network generalises to new data.
var metric = new TotalErrorClassificationMetric <double>();
var predictions = model.Predict(testObservations);
Trace.WriteLine("Test Error: " + metric.Error(testTargets, predictions));
}
public void Classification_Convolutional_Neural_Net()
{
#region Read Data
// Use StreamReader(filepath) when running from filesystem
var trainingParser = new CsvParser(() => new StringReader(Resources.mnist_small_train));
var testParser = new CsvParser(() => new StringReader(Resources.mnist_small_test));
var targetName = "Class";
var featureNames = trainingParser.EnumerateRows(c => c != targetName).First().ColumnNameToIndex.Keys.ToArray();
// read feature matrix (training)
var trainingObservations = trainingParser
.EnumerateRows(featureNames)
.ToF64Matrix();
// read classification targets (training)
var trainingTargets = trainingParser.EnumerateRows(targetName)
.ToF64Vector();
// read feature matrix (test)
var testObservations = testParser
.EnumerateRows(featureNames)
.ToF64Matrix();
// read classification targets (test)
var testTargets = testParser.EnumerateRows(targetName)
.ToF64Vector();
#endregion
// transform pixel values to be between 0 and 1.
trainingObservations.Map(p => p / 255);
testObservations.Map(p => p / 255);
// the output layer must know the number of classes.
var numberOfClasses = trainingTargets.Distinct().Count();
// define the neural net.
var net = new NeuralNet();
net.Add(new InputLayer(width: 28, height: 28, depth: 1)); // MNIST data is 28x28x1.
net.Add(new Conv2DLayer(filterWidth: 5, filterHeight: 5, filterCount: 32));
net.Add(new MaxPool2DLayer(poolWidth: 2, poolHeight: 2));
net.Add(new DropoutLayer(0.5));
net.Add(new DenseLayer(256, Activation.Relu));
net.Add(new DropoutLayer(0.5));
net.Add(new SoftMaxLayer(numberOfClasses));
// using only 10 iteration to make the example run faster.
// using classification accuracy as error metric. This is only used for reporting progress.
var learner = new ClassificationNeuralNetLearner(net, iterations: 10, loss: new AccuracyLoss());
var model = learner.Learn(trainingObservations, trainingTargets);
var metric = new TotalErrorClassificationMetric <double>();
var predictions = model.Predict(testObservations);
Trace.WriteLine("Test Error: " + metric.Error(testTargets, predictions));
}
double CrossValidate(int folds)
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var sut = new StratifiedCrossValidation <double>(folds, 42);
var predictions = sut.CrossValidate(new ClassificationDecisionTreeLearner(), observations, targets);
var metric = new TotalErrorClassificationMetric <double>();
return(metric.Error(targets, predictions));
}
public void TotalErrorClassificationMetric_Error_All_Error()
{
var predictions = new double[] { 1, 1, 1, 1 };
var targets = new double[] { 0, 0, 0, 0 };
var sut = new TotalErrorClassificationMetric <double>();
var actual = sut.Error(targets, predictions);
Assert.AreEqual(1.0, actual);
}
public void TotalErrorClassificationMetric_Error_Zero_Error()
{
var predictions = new double[] { 0, 1, 1, 2, 3, 4, 4 };
var targets = new double[] { 0, 1, 1, 2, 3, 4, 4 };
var sut = new TotalErrorClassificationMetric <double>();
var actual = sut.Error(targets, predictions);
Assert.AreEqual(0.0, actual);
}
double CrossValidate(int folds)
{
var targetName = "Pass";
var parser = new CsvParser(() => new StringReader(Resources.AptitudeData));
var observations = parser.EnumerateRows(v => !v.Contains(targetName)).ToF64Matrix();
var targets = parser.EnumerateRows(targetName).ToF64Vector();
var sut = new StratifiedCrossValidation<double>(folds, 42);
var predictions = sut.CrossValidate(new ClassificationDecisionTreeLearner(), observations, targets);
var metric = new TotalErrorClassificationMetric<double>();
return metric.Error(targets, predictions);
}
double ClassificationExtremelyRandomizedTreesLearner_Learn_Glass(int trees, double subSampleRatio = 1.0)
{
var(observations, targets) = DataSetUtilities.LoadGlassDataSet();
var sut = new ClassificationExtremelyRandomizedTreesLearner(trees, 1, 100, 1, 0.0001, subSampleRatio, 42, false);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, predictions);
return(error);
}
public void ClassificationAdaBoostModel_Precit_Multiple()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learner = new ClassificationAdaBoostLearner(10);
var sut = learner.Learn(observations, targets);
var predictions = sut.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.038461538461538464, error, 0.0000001);
}
public void ClassificationAdaBoostModel_Load()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var reader = new StringReader(m_classificationAdaBoostModelString);
var sut = ClassificationAdaBoostModel.Load(() => reader);
var predictions = sut.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.19230769230769232, error, 0.0000001);
}
public void ClassificationGradientBoostModel_Predict_Multiple()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learner = new ClassificationGradientBoostLearner(100, 0.1, 3, 1, 1e-6, 1, 0, new GradientBoostBinomialLoss(), false);
var sut = learner.Learn(observations, targets);
var predictions = sut.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.038461538461538464, error, 0.0000001);
}
public void ClassificationAdaBoostLearner_Learn_Glass()
{
var(observations, targets) = DataSetUtilities.LoadGlassDataSet();
var sut = new ClassificationAdaBoostLearner(10, 1, 5);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(targets, predictions);
Assert.AreEqual(0.0, actual);
}
double ClassificationDecisionTreeLearner_Learn_Aptitude(int treeDepth)
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var sut = new ClassificationDecisionTreeLearner(treeDepth, 1, 2, 0.001, 42);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, predictions);
return(error);
}
public void ClassificationBinomialGradientBoostLearner_Stochastic_Learn()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var sut = new ClassificationBinomialGradientBoostLearner(50, 0.1, 3, 1, 1e-6, .3, 0, false);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(targets, predictions);
Assert.AreEqual(0.076923076923076927, actual);
}
public void ClassificationForestModel_Predict_Multiple()
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var learner = new ClassificationRandomForestLearner(100, 5, 100, 1, 0.0001, 1.0, 42, false);
var sut = learner.Learn(observations, targets);
var predictions = sut.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, predictions);
Assert.AreEqual(0.23076923076923078, error, m_delta);
}
public void ClassificationBinomialGradientBoostLearner_MultiClass_Learn()
{
var(observations, targets) = DataSetUtilities.LoadGlassDataSet();
var sut = new ClassificationBinomialGradientBoostLearner(30, 0.1, 3, 1, 1e-6, 1.0, 0, false);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var actual = evaluator.Error(targets, predictions);
Assert.AreEqual(0.018691588785046728, actual);
}
double ClassificationRandomLearner_Learn_Aptitude(int trees, double subSampleRatio = 1.0)
{
var(observations, targets) = DataSetUtilities.LoadAptitudeDataSet();
var sut = new ClassificationRandomForestLearner(trees, 5, 100, 1, 0.0001, subSampleRatio, 42, false);
var model = sut.Learn(observations, targets);
var predictions = model.Predict(observations);
var evaluator = new TotalErrorClassificationMetric <double>();
var error = evaluator.Error(targets, predictions);
return(error);
}