public static void TestSVM(double[][] inputs, int[] outputs) { var crossValidation = new CrossValidation(inputs.Length, 10); crossValidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { var trainingInputs = inputs.Submatrix(indicesTrain); var trainingOutputs = outputs.Submatrix(indicesTrain); // And now the validation data: var validationInputs = inputs.Submatrix(indicesValidation); var validationOutputs = outputs.Submatrix(indicesValidation); var sw1 = Stopwatch.StartNew(); var svm = new SVM(); var trainingError = svm.TrainSVM(new RationalQuadratic(1), 3, trainingInputs, trainingOutputs); sw1.Stop(); Console.WriteLine("Training for: " + sw1.ElapsedMilliseconds + "ms with errors: " + trainingError); var validationError = svm.GetSMO().ComputeError(validationInputs, validationOutputs); // Return a new information structure containing the model and the errors achieved. return(new CrossValidationValues(svm, trainingError, validationError)); }; // Compute the cross-validation var result = crossValidation.Compute(); // Finally, access the measured performance. var trainingErrors = result.Training.Mean; var validationErrors = result.Validation.Mean; Console.WriteLine("Finished with " + trainingErrors + " training errors and " + validationErrors + " validation errors"); }
public CrossValidationResult <object> Validate(IClassifier classifier, TrainingData trainingData, int folds = 10) { var crossValidation = new CrossValidation(size: trainingData.Inputs.Length, folds: folds); crossValidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { var trainingInputs = trainingData.Inputs.Get(indicesTrain); var trainingOutputs = trainingData.Outputs.Get(indicesTrain); var validationInputs = trainingData.Inputs.Get(indicesValidation); var validationOutputs = trainingData.Outputs.Get(indicesValidation); var foldClassifier = classifier.CreateInstance(trainingData.FeatureDefaultsValueTypes, trainingData.FeatureGranularities); foldClassifier.Train(trainingInputs, trainingOutputs); var trainingPredicted = foldClassifier.Decide(trainingInputs); var validationPredicted = foldClassifier.Decide(validationInputs); double trainingError = new ZeroOneLoss(trainingOutputs).Loss(trainingPredicted); double validationError = new ZeroOneLoss(validationOutputs).Loss(validationPredicted); var confusionMatrix = new ConfusionMatrix(validationPredicted, validationOutputs, positiveValue: 1, negativeValue: 0); Console.WriteLine($"{k}\t{trainingError}\t{validationError}\t{confusionMatrix.Accuracy}\t{confusionMatrix.TruePositives}\t{confusionMatrix.TrueNegatives}\t{confusionMatrix.FalsePositives}\t{confusionMatrix.FalseNegatives}\t{confusionMatrix.FalsePositiveRate}"); return(new CrossValidationValues(foldClassifier, trainingError, validationError)); }; var result = crossValidation.Compute(); return(result); }
public static void TestKNN(double[][] inputs, int[] outputs, int kValue) { var crossValidation = new CrossValidation(inputs[0].Length, 10); crossValidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { var trainingInputs = inputs.Submatrix(indicesTrain); var trainingOutputs = outputs.Submatrix(indicesTrain); // And now the validation data: var validationInputs = inputs.Submatrix(indicesValidation); var validationOutputs = outputs.Submatrix(indicesValidation); var sw = Stopwatch.StartNew(); var knn = new KNN(); knn.TrainKNN(trainingInputs, trainingOutputs, kValue); sw.Stop(); //Console.WriteLine("Training for: " + sw.ElapsedMilliseconds + "ms"); var error = knn.ComputeError(validationInputs, validationOutputs); return(new CrossValidationValues(knn, 0, error)); }; // Compute the cross-validation var result = crossValidation.Compute(); // Finally, access the measured performance. var trainingErrors = result.Training.Mean; var validationErrors = result.Validation.Mean; Console.WriteLine("Finished with " + trainingErrors + " training errors and " + validationErrors + " validation errors"); }
public void NotEnoughSamplesTest2() { Accord.Math.Tools.SetupGenerator(0); int[] labels = Matrix.Vector(10, 1).Concatenate(Matrix.Vector(30, 0)); Vector.Shuffle(labels); var crossvalidation = new CrossValidation <MulticlassSupportVectorMachine>(labels, 2, folds: 10) { RunInParallel = false, Fitting = (int index, int[] indicesTrain, int[] indicesValidation) => { var labelsValidation = labels.Submatrix(indicesValidation); int countValidation = labelsValidation.Count(x => x == 1); Assert.AreEqual(1, countValidation); var labelsTraining = labels.Submatrix(indicesTrain); int countTraining = labelsTraining.Count(x => x == 1); Assert.AreEqual(9, countTraining); return(new CrossValidationValues <MulticlassSupportVectorMachine>(null, 0, 0)); } }; crossvalidation.Compute(); }
public void FittingTest() { int[] folds = CrossValidation.Splittings(100, 10); int[] samples = Matrix.Indices(0, 100); CrossValidation val = new CrossValidation(folds, 10); val.RunInParallel = false; int current = 0; val.Fitting = (k, trainingSamples, validationSamples) => { Assert.AreEqual(current, k); Assert.AreEqual(90, trainingSamples.Length); Assert.AreEqual(10, validationSamples.Length); int[] trainingSet = samples.Submatrix(trainingSamples); int[] validationSet = samples.Submatrix(validationSamples); for (int i = 0; i < trainingSet.Length; i++) { Assert.AreEqual(samples[trainingSamples[i]], trainingSet[i]); } for (int i = 0; i < validationSet.Length; i++) { Assert.AreEqual(samples[validationSamples[i]], validationSet[i]); } current++; return(new CrossValidationValues(k, 2 * k)); }; var result = val.Compute(); Assert.AreEqual(10, current); Assert.AreEqual(4.5, result.Training.Mean); Assert.AreEqual(9.0, result.Validation.Mean); Assert.AreEqual( 2 * result.Training.StandardDeviation, result.Validation.StandardDeviation); Assert.AreEqual(val.Folds.Length, result.Training.Sizes.Length); Assert.AreEqual(val.Folds.Length, result.Validation.Sizes.Length); for (int i = 0; i < result.Training.Sizes.Length; i++) { Assert.AreEqual(90, result.Training.Sizes[i]); } for (int i = 0; i < result.Validation.Sizes.Length; i++) { Assert.AreEqual(10, result.Validation.Sizes[i]); } }
public override Task <List <GeneralConfusionMatrix> > ComputeFoldedConfusionMatrixAsync(ClassificationModel classificationModel, int folds) { return(Task.Factory.StartNew(() => { int numFeatures = classificationModel.FeatureVectors.Count; DecisionVariable[] decisionVariables = Enumerable.ToArray(classificationModel.Bands.Select(b => DecisionVariable.Continuous(b.ToString()))); double[][] input = new double[numFeatures][]; int[] responses = new int[numFeatures]; for (int featureIndex = 0; featureIndex < classificationModel.FeatureVectors.Count; ++featureIndex) { var featureVector = classificationModel.FeatureVectors[featureIndex]; input[featureIndex] = Array.ConvertAll(featureVector.FeatureVector.BandIntensities, s => (double)s / ushort.MaxValue); responses[featureIndex] = featureVector.FeatureClass; } List <GeneralConfusionMatrix> confusionMatrices = new List <GeneralConfusionMatrix>(); // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation(input.Length, folds); crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // Lets now grab the training data: var trainingInputs = input.Get(indicesTrain); var trainingOutputs = responses.Get(indicesTrain); // And now the validation data: var validationInputs = input.Get(indicesValidation); var validationOutputs = responses.Get(indicesValidation); var tree = new DecisionTree(decisionVariables, Enum.GetValues(typeof(LandcoverTypeViewModel)).Length); C45Learning id3Learning = new C45Learning(tree); id3Learning.Learn(trainingInputs, trainingOutputs); var predictedTraining = tree.Decide(trainingInputs); var predictedValidation = tree.Decide(validationInputs); double trainingError = new ZeroOneLoss(trainingOutputs).Loss(predictedTraining); double validationError = new ZeroOneLoss(validationOutputs).Loss(predictedValidation); GeneralConfusionMatrix confusionMatrix = new GeneralConfusionMatrix(Enum.GetValues(typeof(LandcoverTypeViewModel)).Length - 1, validationOutputs, predictedValidation); confusionMatrices.Add(confusionMatrix); // Return a new information structure containing the model and the errors achieved. return new CrossValidationValues(trainingError, validationError); }; var result = crossvalidation.Compute(); return confusionMatrices; })); }
public void NotEnoughSamplesTest1() { Accord.Math.Random.Generator.Seed = 0; int[] labels = Matrix.Vector(10, 1).Concatenate(Matrix.Vector(30, 0)); Vector.Shuffle(labels); var crossvalidation = new CrossValidation <MulticlassSupportVectorMachine>(size: 40, folds: 10) { RunInParallel = false, Fitting = (int index, int[] indicesTrain, int[] indicesValidation) => { var labelsValidation = labels.Submatrix(indicesValidation); int countValidation = labelsValidation.Count(x => x == 1); Assert.AreEqual(2, countValidation); var labelsTraining = labels.Submatrix(indicesTrain); int countTraining = labelsTraining.Count(x => x == 1); Assert.AreEqual(9 * 2, countTraining); return(new CrossValidationValues <MulticlassSupportVectorMachine>(null, 0, 0)); } }; bool thrown = false; try { crossvalidation.Compute(); } catch (Exception) { thrown = true; } Assert.IsTrue(thrown); crossvalidation = new CrossValidation <MulticlassSupportVectorMachine>(labels, 2, folds: 10) { RunInParallel = false, Fitting = (int index, int[] indicesTrain, int[] indicesValidation) => { var labelsValidation = labels.Submatrix(indicesValidation); int countValidation = labelsValidation.Count(x => x == 1); Assert.AreEqual(1, countValidation); var labelsTraining = labels.Submatrix(indicesTrain); int countTraining = labelsTraining.Count(x => x == 1); Assert.AreEqual(9, countTraining); return(new CrossValidationValues <MulticlassSupportVectorMachine>(null, 0, 0)); } }; crossvalidation.Compute(); }
public override Task <List <GeneralConfusionMatrix> > ComputeFoldedConfusionMatrixAsync(ClassificationModel classificationModel, int folds) { return(Task.Factory.StartNew(() => { int numFeatures = classificationModel.FeatureVectors.Count; double[][] input = new double[numFeatures][]; int[] responses = new int[numFeatures]; for (int featureIndex = 0; featureIndex < classificationModel.FeatureVectors.Count; ++featureIndex) { var featureVector = classificationModel.FeatureVectors[featureIndex]; input[featureIndex] = Array.ConvertAll(featureVector.FeatureVector.BandIntensities, s => (double)s / ushort.MaxValue); responses[featureIndex] = featureVector.FeatureClass; } List <GeneralConfusionMatrix> confusionMatrices = new List <GeneralConfusionMatrix>(); // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation(input.Length, folds); crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // Lets now grab the training data: var trainingInputs = input.Get(indicesTrain); var trainingOutputs = responses.Get(indicesTrain); // And now the validation data: var validationInputs = input.Get(indicesValidation); var validationOutputs = responses.Get(indicesValidation); int[] predictedTraining; int[] predictedValidation; TrainAndPredict(Complexity, Gamma, Degree, trainingInputs, trainingOutputs, validationInputs, out predictedTraining, out predictedValidation); double trainingError = new ZeroOneLoss(trainingOutputs).Loss(predictedTraining); double validationError = new ZeroOneLoss(validationOutputs).Loss(predictedValidation); GeneralConfusionMatrix confusionMatrix = new GeneralConfusionMatrix(classificationModel.LandCoverTypes.Count, validationOutputs, predictedValidation); confusionMatrices.Add(confusionMatrix); // Return a new information structure containing the model and the errors achieved. return new CrossValidationValues(trainingError, validationError); }; crossvalidation.Compute(); return confusionMatrices; })); }
private void button1_Click(object sender, EventArgs e) { // Creates a matrix from the source data table double[,] sourceMatrix = (dgvLearningSource.DataSource as DataTable).ToMatrix(out sourceColumns); // Get only the input vector values var inputs = sourceMatrix.Submatrix(0, sourceMatrix.GetLength(0) - 1, 0, 1).ToArray(); // Get only the label outputs var outputs = new int[sourceMatrix.GetLength(0)]; for (int i = 0; i < outputs.Length; i++) { outputs[i] = (int)sourceMatrix[i, 2]; } var cv = new CrossValidation <KernelSupportVectorMachine>(inputs.Length, 10); cv.Fitting = (int k, int[] training, int[] testing) => { var trainingInputs = inputs.Submatrix(training); var trainingOutputs = outputs.Submatrix(training); var testingInputs = inputs.Submatrix(testing); var testingOutputs = outputs.Submatrix(testing); // Create the specified Kernel IKernel kernel = getKernel(); // Creates the Support Vector Machine using the selected kernel var svm = new KernelSupportVectorMachine(kernel, 2); // Creates a new instance of the SMO Learning Algortihm var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs); // Set learning parameters smo.Complexity = (double)numC.Value; smo.Tolerance = (double)numT.Value; // Run double trainingError = smo.Run(); double validationError = smo.ComputeError(testingInputs, testingOutputs); return(new CrossValidationValues <KernelSupportVectorMachine>(svm, trainingError, validationError)); }; var result = cv.Compute(); }
public CrossValidationResult <RandomForest> GetCrossValidationResultsOfRandomForestModel(AppIdentAcordSource appIdentAcordSource, GridSearchParameterCollection bestParameters, int folds = 10) { var samples = appIdentAcordSource.Samples; var labels = appIdentAcordSource.LabelsAsIntegers; var decisionVariables = appIdentAcordSource.DecisionVariables; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation <RandomForest>(samples.Length, folds) { Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. Console.WriteLine($"{DateTime.Now} RandomForest cross validation."); // Lets now grab the training data: var trainingInputs = samples.Get(indicesTrain); var trainingOutputs = labels.Get(indicesTrain); // And now the validation data: var validationInputs = samples.Get(indicesValidation); var validationOutputs = labels.Get(indicesValidation); // create random forest model with the best parameters from grid search results var rfcModel = CreateRandomForestModel(decisionVariables, bestParameters, trainingInputs, trainingOutputs); // compute the training error rate with ZeroOneLoss function var trainingError = new ZeroOneLoss(trainingOutputs).Loss(rfcModel.Decide(trainingInputs)); // Now we can compute the validation error on the validation data: var validationError = new ZeroOneLoss(validationOutputs).Loss(rfcModel.Decide(validationInputs)); // Return a new information structure containing the model and the errors achieved. var tag = new ValidationDataSource(validationInputs, validationOutputs); return(new CrossValidationValues <RandomForest>(rfcModel, trainingError, validationError) { Tag = tag }); } }; // Compute the cross-validation return(crossvalidation.Compute()); }
public void NotEnoughSamplesTest2() { Accord.Math.Tools.SetupGenerator(0); int[] labels = Matrix.Vector(10, 1).Concatenate(Matrix.Vector(30, 0)); Accord.Statistics.Tools.Shuffle(labels); var crossvalidation = new CrossValidation<MulticlassSupportVectorMachine>(labels, 2, folds: 10) { RunInParallel = false, Fitting = (int index, int[] indicesTrain, int[] indicesValidation) => { var labelsValidation = labels.Submatrix(indicesValidation); int countValidation = labelsValidation.Count(x => x == 1); Assert.AreEqual(1, countValidation); var labelsTraining = labels.Submatrix(indicesTrain); int countTraining = labelsTraining.Count(x => x == 1); Assert.AreEqual(9, countTraining); return new CrossValidationValues<MulticlassSupportVectorMachine>(null, 0, 0); } }; crossvalidation.Compute(); }
private int OptimizeK() { Tuple <KNearestNeighbors, double> bestModel = null; for (int k = 50; k <= 200; k++) { var crossvalidation = new CrossValidation <KNearestNeighbors>(_input.Length, 5); crossvalidation.Fitting = delegate(int fold, int[] indicesTrain, int[] indicesValidation) { var trainingInputs = _input.Submatrix(indicesTrain); var trainingOutputs = _output.Submatrix(indicesTrain); var validationInputs = _input.Submatrix(indicesValidation); var validationOutputs = _output.Submatrix(indicesValidation); var predictor = new KNearestNeighbors(k, _classes, trainingInputs, trainingOutputs); // Create a training algorithm and learn the training data var trainingError = 0.0; for (int i = 0; i < trainingInputs.Length; i++) { int[] nearest; predictor.GetNearestNeighbors(trainingInputs[i], out nearest); var prediction = InverseDistanceWeightedAverage(nearest); if (prediction > 0 && trainingOutputs[i] > 0 || prediction < 0 && trainingOutputs[i] < 0 || prediction.Equals(trainingOutputs[i])) { continue; } trainingError++; } double validationError = 0.0; for (int i = 0; i < validationInputs.Length; i++) { int[] nearest; predictor.GetNearestNeighbors(validationInputs[i], out nearest); var prediction = InverseDistanceWeightedAverage(nearest); if (prediction > 0 && validationOutputs[i] > 0 || prediction < 0 && validationOutputs[i] < 0 || prediction.Equals(validationOutputs[i])) { continue; } validationError++; } trainingError /= trainingInputs.Length; validationError /= validationInputs.Length; return(new CrossValidationValues <KNearestNeighbors>(predictor, trainingError, validationError)); }; var result = crossvalidation.Compute(); //var minError = result.Models.Select(y => y.ValidationValue).Min(); var minError = result.Models.Select(y => Math.Sqrt(Math.Pow(y.TrainingValue + y.ValidationValue, 2.0))).Min(); if (bestModel == null || minError < bestModel.Item2) { var bestFit = result.Models.FirstOrDefault(x => minError.Equals(x.ValidationValue))?.Model; bestModel = bestFit == null ? bestModel : new Tuple <KNearestNeighbors, double>(bestFit, minError); } } return(bestModel?.Item1.K ?? 80); }
public void CrossvalidationConstructorTest2() { Accord.Math.Tools.SetupGenerator(0); // This is a sample code on how to use Cross-Validation // to assess the performance of Hidden Markov Models. // Declare some testing data int[][] inputs = new int[][] { new int[] { 0, 1, 1, 0 }, // Class 0 new int[] { 0, 0, 1, 0 }, // Class 0 new int[] { 0, 1, 1, 1, 0 }, // Class 0 new int[] { 0, 1, 1, 1, 0 }, // Class 0 new int[] { 0, 1, 1, 0 }, // Class 0 new int[] { 0, 1, 1, 1, 0 }, // Class 0 new int[] { 0, 1, 1, 1, 0 }, // Class 0 new int[] { 0, 1, 0, 1, 0 }, // Class 0 new int[] { 0, 1, 0 }, // Class 0 new int[] { 0, 1, 1, 0 }, // Class 0 new int[] { 1, 0, 0, 1 }, // Class 1 new int[] { 1, 1, 0, 1 }, // Class 1 new int[] { 1, 0, 0, 0, 1 }, // Class 1 new int[] { 1, 0, 1 }, // Class 1 new int[] { 1, 1, 0, 1 }, // Class 1 new int[] { 1, 0, 1 }, // Class 1 new int[] { 1, 0, 0, 1 }, // Class 1 new int[] { 1, 0, 0, 0, 1 }, // Class 1 new int[] { 1, 0, 1 }, // Class 1 new int[] { 1, 0, 0, 0, 1 }, // Class 1 }; int[] outputs = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // First 10 sequences are of class 0 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // Last 10 sequences are of class 1 }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation <HiddenMarkovClassifier>(size: inputs.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = inputs.Submatrix(indicesTrain); var trainingOutputs = outputs.Submatrix(indicesTrain); // And now the validation data: var validationInputs = inputs.Submatrix(indicesValidation); var validationOutputs = outputs.Submatrix(indicesValidation); // We are trying to predict two different classes int classes = 2; // Each sequence may have up to two symbols (0 or 1) int symbols = 2; // Nested models will have two states each int[] states = new int[] { 2, 2 }; // Creates a new Hidden Markov Model Classifier with the given parameters HiddenMarkovClassifier classifier = new HiddenMarkovClassifier(classes, states, symbols); // Create a new learning algorithm to train the sequence classifier var teacher = new HiddenMarkovClassifierLearning(classifier, // Train each model until the log-likelihood changes less than 0.001 modelIndex => new BaumWelchLearning(classifier.Models[modelIndex]) { Tolerance = 0.001, Iterations = 0 } ); // Train the sequence classifier using the algorithm double likelihood = teacher.Run(trainingInputs, trainingOutputs); double trainingError = teacher.ComputeError(trainingInputs, trainingOutputs); // Now we can compute the validation error on the validation data: double validationError = teacher.ComputeError(validationInputs, validationOutputs); // Return a new information structure containing the model and the errors achieved. return(new CrossValidationValues <HiddenMarkovClassifier>(classifier, trainingError, validationError)); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; Assert.AreEqual(3, crossvalidation.K); Assert.AreEqual(0, result.Training.Mean); Assert.AreEqual(0, result.Validation.Mean); Assert.AreEqual(3, crossvalidation.Folds.Length); Assert.AreEqual(3, result.Models.Length); }
public void CrossvalidationConstructorTest() { Accord.Math.Tools.SetupGenerator(0); // This is a sample code on how to use Cross-Validation // to assess the performance of Support Vector Machines. // Consider the example binary data. We will be trying // to learn a XOR problem and see how well does SVMs // perform on this data. double[][] data = { new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, }; int[] xor = // result of xor for the sample input data { -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation <KernelSupportVectorMachine>(size: data.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = data.Submatrix(indicesTrain); var trainingOutputs = xor.Submatrix(indicesTrain); // And now the validation data: var validationInputs = data.Submatrix(indicesValidation); var validationOutputs = xor.Submatrix(indicesValidation); // Create a Kernel Support Vector Machine to operate on the set var svm = new KernelSupportVectorMachine(new Polynomial(2), 2); // Create a training algorithm and learn the training data var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs); double trainingError = smo.Run(); // Now we can compute the validation error on the validation data: double validationError = smo.ComputeError(validationInputs, validationOutputs); // Return a new information structure containing the model and the errors achieved. return(new CrossValidationValues <KernelSupportVectorMachine>(svm, trainingError, validationError)); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; Assert.AreEqual(3, crossvalidation.K); Assert.AreEqual(0, result.Training.Mean); Assert.AreEqual(0, result.Validation.Mean); Assert.AreEqual(3, crossvalidation.Folds.Length); Assert.AreEqual(3, result.Models.Length); }
public void NotEnoughSamplesTest1() { Accord.Math.Random.Generator.Seed = 0; int[] labels = Matrix.Vector(10, 1).Concatenate(Matrix.Vector(30, 0)); Vector.Shuffle(labels); var crossvalidation = new CrossValidation<MulticlassSupportVectorMachine>(size: 40, folds: 10) { RunInParallel = false, Fitting = (int index, int[] indicesTrain, int[] indicesValidation) => { var labelsValidation = labels.Submatrix(indicesValidation); int countValidation = labelsValidation.Count(x => x == 1); Assert.AreEqual(2, countValidation); var labelsTraining = labels.Submatrix(indicesTrain); int countTraining = labelsTraining.Count(x => x == 1); Assert.AreEqual(9 * 2, countTraining); return new CrossValidationValues<MulticlassSupportVectorMachine>(null, 0, 0); } }; bool thrown = false; try { crossvalidation.Compute(); } catch (Exception) { thrown = true; } Assert.IsTrue(thrown); crossvalidation = new CrossValidation<MulticlassSupportVectorMachine>(labels, 2, folds: 10) { RunInParallel = false, Fitting = (int index, int[] indicesTrain, int[] indicesValidation) => { var labelsValidation = labels.Submatrix(indicesValidation); int countValidation = labelsValidation.Count(x => x == 1); Assert.AreEqual(1, countValidation); var labelsTraining = labels.Submatrix(indicesTrain); int countTraining = labelsTraining.Count(x => x == 1); Assert.AreEqual(9, countTraining); return new CrossValidationValues<MulticlassSupportVectorMachine>(null, 0, 0); } }; crossvalidation.Compute(); }
private void Test_Load(object sender, EventArgs e) { // TODO: This line of code loads data into the 'diabetesDataSetB.ContinuousData' table. You can move, or remove it, as needed. this.continuousDataTableAdapter.Fill(this.diabetesDataSetB.ContinuousData); // This is a sample code on how to use Cross-Validation // to access the performance of Support Vector Machines. // Consider the example binary data. We will be trying // to learn a XOR problem and see how well does SVMs // perform on this data. double[][] data = { new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, }; int[] xor = // result of xor for the sample input data { -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation<KernelSupportVectorMachine>(size: data.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data dicted by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = data.Submatrix(indicesTrain); var trainingOutputs = xor.Submatrix(indicesTrain); // And now the validation data: var validationInputs = data.Submatrix(indicesValidation); var validationOutputs = xor.Submatrix(indicesValidation); // Create a Kernel Support Vector Machine to operate on the set var svm = new KernelSupportVectorMachine(new Polynomial(2), 2); // Create a training algorithm and learn the training data var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs); double trainingError = smo.Run(); // Now we can compute the validation error on the validation data: double validationError = smo.ComputeError(validationInputs, validationOutputs); // Return a new information structure containing the model and the errors achieved. return new CrossValidationValues<KernelSupportVectorMachine>(svm, trainingError, validationError); }; //crossvalidation.CreatePartitions(2, data,out xor); // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; }
private CrossValidationResult _performCrossValidation() { // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation(size: _inputs.Length, folds: _configuration.CrossValidationNumFolds); Console.WriteLine("Num Samples: " + crossvalidation.Samples); // ConcurrentDictionary<string, NodeClassification> nodeClassifications = new ConcurrentDictionary<string, NodeClassification>(); CrossValidationResult cvResult = new CrossValidationResult(); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = _inputs.Submatrix(indicesTrain); var trainingOutputs = _outputs.Submatrix(indicesTrain); var trainingNames = _names.Submatrix(indicesTrain); // And now the validation data: var validationInputs = _inputs.Submatrix(indicesValidation); var validationOutputs = _outputs.Submatrix(indicesValidation); var validationNames = _names.Submatrix(indicesValidation); if (validationNames.Intersect <String>(trainingNames).Count() > 0) { Console.WriteLine("Warning, Training and Validation Set not disjunct."); Utility.writeToConsole <String>(validationNames.Intersect(trainingNames).ToArray()); } //Console.WriteLine("=== TRAINING ==="); //Utility.writeToConsole<String>(trainingNames); //Utility.writeToConsole<int>(trainingInputs); //Utility.writeToConsole<int>(trainingOutputs); //Console.WriteLine("=== VALIDATION ==="); //Utility.writeToConsole<String>(validationNames); //Utility.writeToConsole<double>(validationInputs); //Utility.writeToConsole<int>(validationOutputs); //Console.WriteLine("=== INTERSECTION ==="); //String[] intersection = Utility.Intersection(validationNames, trainingNames); //Utility.writeToConsole<String>(intersection); // Create a Kernel Support Vector Machine to operate on the set var svm = new KernelSupportVectorMachine(_kernel, _inputs[0].Length); //Accord.MachineLearning.Boosting.Boost<KernelSupportVectorMachine> b; // Create a training algorithm and learn the training data var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs) { // Set learning parameters Tolerance = _configuration.Tolerance, PositiveWeight = _configuration.WeightPositiveClass, NegativeWeight = _configuration.WeightNegativeClass, UseClassProportions = _configuration.UseComputedWeights, UseComplexityHeuristic = true }; //if (!_configuration.UseHeuristicalComplexity) _smo.Complexity = _configuration.Complexity; double trainingError = smo.Run(); // Now we can compute the validation error on the validation data: double validationError = smo.ComputeError(validationInputs, validationOutputs); // Predictions & Confusion Matrix List <int> predictions = new List <int>(); List <double> rawPredictions = new List <double>(); int index = 0; foreach (double[] inputVector in validationInputs) { // Compute the decision output for vector // Console.WriteLine(validationNames[index]); // Utility.writeToConsole<double>(inputVector); double rawPrediction = svm.Compute(inputVector); rawPredictions.Add(rawPrediction); int prediction = rawPrediction > 0.0d ? +1 : -1; predictions.Add(prediction); try { // Update Node Classifications cvResult.AddOrUpdateClassification(validationNames[index], rawPrediction, validationOutputs[index]); } catch (IndexOutOfRangeException) { Console.WriteLine("Failed to update CV Result: Input Vector larger than Name Vector."); } index++; } ConfusionMatrix confusionMatrix = new ConfusionMatrix(predictions.ToArray(), validationOutputs, 1, -1); cvResult.ConfusionMatrices.Add(confusionMatrix); // Return a new information structure containing the model and the errors achieved. return(new CrossValidationValues(svm, trainingError, validationError)); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; Console.WriteLine("Training Errors: " + result.Training.Mean); Console.WriteLine("Validation Errors: " + result.Validation.Mean); ConfusionMatrix aggregatedConfusionMatrix = ConfusionMatrix.Combine(cvResult.ConfusionMatrices.ToArray()); return(cvResult); }
private void Test_Load(object sender, EventArgs e) { // TODO: This line of code loads data into the 'diabetesDataSetB.ContinuousData' table. You can move, or remove it, as needed. this.continuousDataTableAdapter.Fill(this.diabetesDataSetB.ContinuousData); // This is a sample code on how to use Cross-Validation // to access the performance of Support Vector Machines. // Consider the example binary data. We will be trying // to learn a XOR problem and see how well does SVMs // perform on this data. double[][] data = { new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, }; int[] xor = // result of xor for the sample input data { -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation <KernelSupportVectorMachine>(size: data.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data dicted by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = data.Submatrix(indicesTrain); var trainingOutputs = xor.Submatrix(indicesTrain); // And now the validation data: var validationInputs = data.Submatrix(indicesValidation); var validationOutputs = xor.Submatrix(indicesValidation); // Create a Kernel Support Vector Machine to operate on the set var svm = new KernelSupportVectorMachine(new Polynomial(2), 2); // Create a training algorithm and learn the training data var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs); double trainingError = smo.Run(); // Now we can compute the validation error on the validation data: double validationError = smo.ComputeError(validationInputs, validationOutputs); // Return a new information structure containing the model and the errors achieved. return(new CrossValidationValues <KernelSupportVectorMachine>(svm, trainingError, validationError)); }; //crossvalidation.CreatePartitions(2, data,out xor); // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; }
public void CrossvalidationConstructorTest2() { Accord.Math.Tools.SetupGenerator(0); // This is a sample code on how to use Cross-Validation // to assess the performance of Hidden Markov Models. // Declare some testing data int[][] inputs = new int[][] { new int[] { 0,1,1,0 }, // Class 0 new int[] { 0,0,1,0 }, // Class 0 new int[] { 0,1,1,1,0 }, // Class 0 new int[] { 0,1,1,1,0 }, // Class 0 new int[] { 0,1,1,0 }, // Class 0 new int[] { 0,1,1,1,0 }, // Class 0 new int[] { 0,1,1,1,0 }, // Class 0 new int[] { 0,1,0,1,0 }, // Class 0 new int[] { 0,1,0 }, // Class 0 new int[] { 0,1,1,0 }, // Class 0 new int[] { 1,0,0,1 }, // Class 1 new int[] { 1,1,0,1 }, // Class 1 new int[] { 1,0,0,0,1 }, // Class 1 new int[] { 1,0,1 }, // Class 1 new int[] { 1,1,0,1 }, // Class 1 new int[] { 1,0,1 }, // Class 1 new int[] { 1,0,0,1 }, // Class 1 new int[] { 1,0,0,0,1 }, // Class 1 new int[] { 1,0,1 }, // Class 1 new int[] { 1,0,0,0,1 }, // Class 1 }; int[] outputs = new int[] { 0,0,0,0,0,0,0,0,0,0, // First 10 sequences are of class 0 1,1,1,1,1,1,1,1,1,1, // Last 10 sequences are of class 1 }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation<HiddenMarkovClassifier>(size: inputs.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = inputs.Submatrix(indicesTrain); var trainingOutputs = outputs.Submatrix(indicesTrain); // And now the validation data: var validationInputs = inputs.Submatrix(indicesValidation); var validationOutputs = outputs.Submatrix(indicesValidation); // We are trying to predict two different classes int classes = 2; // Each sequence may have up to two symbols (0 or 1) int symbols = 2; // Nested models will have two states each int[] states = new int[] { 2, 2 }; // Creates a new Hidden Markov Model Classifier with the given parameters HiddenMarkovClassifier classifier = new HiddenMarkovClassifier(classes, states, symbols); // Create a new learning algorithm to train the sequence classifier var teacher = new HiddenMarkovClassifierLearning(classifier, // Train each model until the log-likelihood changes less than 0.001 modelIndex => new BaumWelchLearning(classifier.Models[modelIndex]) { Tolerance = 0.001, Iterations = 0 } ); // Train the sequence classifier using the algorithm double likelihood = teacher.Run(trainingInputs, trainingOutputs); double trainingError = teacher.ComputeError(trainingInputs, trainingOutputs); // Now we can compute the validation error on the validation data: double validationError = teacher.ComputeError(validationInputs, validationOutputs); // Return a new information structure containing the model and the errors achieved. return new CrossValidationValues<HiddenMarkovClassifier>(classifier, trainingError, validationError); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; Assert.AreEqual(3, crossvalidation.K); Assert.AreEqual(0, result.Training.Mean); Assert.AreEqual(0.055555555555555552, result.Validation.Mean); Assert.AreEqual(3, crossvalidation.Folds.Length); Assert.AreEqual(3, result.Models.Length); }
public void KNearestNeighbor_CrossValidation() { // Create some sample learning data. In this data, // the first two instances belong to a class, the // four next belong to another class and the last // three to yet another. double[][] inputs = { // The first two are from class 0 new double[] { -5, -2, -1 }, new double[] { -5, -5, -6 }, // The next four are from class 1 new double[] { 2, 1, 1 }, new double[] { 1, 1, 2 }, new double[] { 1, 2, 2 }, new double[] { 3, 1, 2 }, // The last three are from class 2 new double[] { 11, 5, 4 }, new double[] { 15, 5, 6 }, new double[] { 10, 5, 6 }, }; int[] outputs = { 0, 0, // First two from class 0 1, 1, 1, 1, // Next four from class 1 2, 2, 2 // Last three from class 2 }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation(size: inputs.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = inputs.Submatrix(indicesTrain); var trainingOutputs = outputs.Submatrix(indicesTrain); // And now the validation data: var validationInputs = inputs.Submatrix(indicesValidation); var validationOutputs = outputs.Submatrix(indicesValidation); // Now we will create the K-Nearest Neighbors algorithm. For this // example, we will be choosing k = 4. This means that, for a given // instance, its nearest 4 neighbors will be used to cast a decision. KNearestNeighbors knn = new KNearestNeighbors(k: 4, classes: 3, inputs: inputs, outputs: outputs); // After the algorithm has been created, we can classify instances: int[] train_predicted = trainingInputs.Apply(knn.Compute); int[] test_predicted = validationInputs.Apply(knn.Compute); // Compute classification error var cmTrain = new ConfusionMatrix(train_predicted, trainingOutputs); double trainingAcc = cmTrain.Accuracy; // Now we can compute the validation error on the validation data: var cmTest = new ConfusionMatrix(test_predicted, validationOutputs); double validationAcc = cmTest.Accuracy; // Return a new information structure containing the model and the errors achieved. return(new CrossValidationValues(knn, trainingAcc, validationAcc)); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingAccs = result.Training.Mean; double validationAccs = result.Validation.Mean; Assert.AreEqual(1, trainingAccs); Assert.AreEqual(1, validationAccs); }
public void KNearestNeighbor_CrossValidation() { // Create some sample learning data. In this data, // the first two instances belong to a class, the // four next belong to another class and the last // three to yet another. double[][] inputs = { // The first two are from class 0 new double[] { -5, -2, -1 }, new double[] { -5, -5, -6 }, // The next four are from class 1 new double[] { 2, 1, 1 }, new double[] { 1, 1, 2 }, new double[] { 1, 2, 2 }, new double[] { 3, 1, 2 }, // The last three are from class 2 new double[] { 11, 5, 4 }, new double[] { 15, 5, 6 }, new double[] { 10, 5, 6 }, }; int[] outputs = { 0, 0, // First two from class 0 1, 1, 1, 1, // Next four from class 1 2, 2, 2 // Last three from class 2 }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation(size: inputs.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = inputs.Submatrix(indicesTrain); var trainingOutputs = outputs.Submatrix(indicesTrain); // And now the validation data: var validationInputs = inputs.Submatrix(indicesValidation); var validationOutputs = outputs.Submatrix(indicesValidation); // Now we will create the K-Nearest Neighbors algorithm. For this // example, we will be choosing k = 4. This means that, for a given // instance, its nearest 4 neighbors will be used to cast a decision. KNearestNeighbors knn = new KNearestNeighbors(k: 4, classes: 3, inputs: inputs, outputs: outputs); // After the algorithm has been created, we can classify instances: int[] train_predicted = trainingInputs.Apply(knn.Compute); int[] test_predicted = validationInputs.Apply(knn.Compute); // Compute classification error var cmTrain = new ConfusionMatrix(train_predicted, trainingOutputs); double trainingAcc = cmTrain.Accuracy; // Now we can compute the validation error on the validation data: var cmTest = new ConfusionMatrix(test_predicted, validationOutputs); double validationAcc = cmTest.Accuracy; // Return a new information structure containing the model and the errors achieved. return new CrossValidationValues(knn, trainingAcc, validationAcc); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingAccs = result.Training.Mean; double validationAccs = result.Validation.Mean; Assert.AreEqual(1, trainingAccs); Assert.AreEqual(1, validationAccs); }
public override Task <GridSearchParameterCollection> GridSearchAsync(ClassificationModel classificationModel) { return(Task.Factory.StartNew(() => { // Declare the parameters and ranges to be searched List <GridSearchRange> ranges = new List <GridSearchRange> { new GridSearchRange("complexity", new[] { 150.0, 100.0, 50, 10, 1 }), }; switch (Kernel) { case Kernel.Gaussian: ranges.Add(new GridSearchRange("gamma", new[] { 0.1, 1.0, 2.0, 5.0, 10.0, 20.0 })); break; case Kernel.Polynomial: ranges.Add(new GridSearchRange("degree", new[] { 1.0, 2.0, 3.0, 4.0 })); break; } int numFeatures = classificationModel.FeatureVectors.Count; double[][] input = new double[numFeatures][]; int[] responses = new int[numFeatures]; for (int featureIndex = 0; featureIndex < classificationModel.FeatureVectors.Count; ++featureIndex) { var featureVector = classificationModel.FeatureVectors[featureIndex]; input[featureIndex] = Array.ConvertAll(featureVector.FeatureVector.BandIntensities, s => (double)s / ushort.MaxValue); responses[featureIndex] = (int)featureVector.FeatureClass; } // Instantiate a new Grid Search algorithm for Kernel Support Vector Machines var gridsearch = new GridSearch <MulticlassSupportVectorMachine <Gaussian> >(ranges.ToArray()); // Set the fitting function for the algorithm gridsearch.Fitting = delegate(GridSearchParameterCollection parameters, out double error) { // The parameters to be tried will be passed as a function parameter. double complexity = parameters["complexity"].Value; double gamma = parameters.Contains("gamma") ? parameters["gamma"].Value : 0; int degree = parameters.Contains("degree") ? (int)parameters["degree"].Value : 0; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation(size: input.Length, folds: 10); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // Lets now grab the training data: var trainingInputs = input.Get(indicesTrain); var trainingOutputs = responses.Get(indicesTrain); // And now the validation data: var validationInputs = input.Get(indicesValidation); var validationOutputs = responses.Get(indicesValidation); int[] predictedTraining; int[] predictedValidation; TrainAndPredict(complexity, gamma, degree, trainingInputs, trainingOutputs, validationInputs, out predictedTraining, out predictedValidation); double trainingError = new ZeroOneLoss(trainingOutputs).Loss(predictedTraining); double validationError = new ZeroOneLoss(validationOutputs).Loss(predictedValidation); // Return a new information structure containing the model and the errors achieved. return new CrossValidationValues(trainingError, validationError); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; error = validationErrors; return null; // Return the current model }; // Declare some out variables to pass to the grid search algorithm GridSearchParameterCollection bestParameters; double minError; // Compute the grid search to find the best Support Vector Machine gridsearch.Compute(out bestParameters, out minError); return bestParameters; })); }
public void CrossvalidationConstructorTest() { Accord.Math.Tools.SetupGenerator(0); // This is a sample code on how to use Cross-Validation // to assess the performance of Support Vector Machines. // Consider the example binary data. We will be trying // to learn a XOR problem and see how well does SVMs // perform on this data. double[][] data = { new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, new double[] { -1, -1 }, new double[] { 1, -1 }, new double[] { -1, 1 }, new double[] { 1, 1 }, }; int[] xor = // result of xor for the sample input data { -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, }; // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation<KernelSupportVectorMachine>(size: data.Length, folds: 3); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = data.Submatrix(indicesTrain); var trainingOutputs = xor.Submatrix(indicesTrain); // And now the validation data: var validationInputs = data.Submatrix(indicesValidation); var validationOutputs = xor.Submatrix(indicesValidation); // Create a Kernel Support Vector Machine to operate on the set var svm = new KernelSupportVectorMachine(new Polynomial(2), 2); // Create a training algorithm and learn the training data var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs); double trainingError = smo.Run(); // Now we can compute the validation error on the validation data: double validationError = smo.ComputeError(validationInputs, validationOutputs); // Return a new information structure containing the model and the errors achieved. return new CrossValidationValues<KernelSupportVectorMachine>(svm, trainingError, validationError); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; Assert.AreEqual(3, crossvalidation.K); Assert.AreEqual(0, result.Training.Mean); Assert.AreEqual(0, result.Validation.Mean); Assert.AreEqual(3, crossvalidation.Folds.Length); Assert.AreEqual(3, result.Models.Length); }
public void FittingTest() { int[] folds = CrossValidation.Splittings(100, 10); int[] samples = Matrix.Indices(0, 100); CrossValidation val = new CrossValidation(folds, 10); val.RunInParallel = false; int current = 0; val.Fitting = (k, trainingSamples, validationSamples) => { Assert.AreEqual(current, k); Assert.AreEqual(90, trainingSamples.Length); Assert.AreEqual(10, validationSamples.Length); int[] trainingSet = samples.Submatrix(trainingSamples); int[] validationSet = samples.Submatrix(validationSamples); for (int i = 0; i < trainingSet.Length; i++) Assert.AreEqual(samples[trainingSamples[i]], trainingSet[i]); for (int i = 0; i < validationSet.Length; i++) Assert.AreEqual(samples[validationSamples[i]], validationSet[i]); current++; return new CrossValidationValues<object>(new object(), k, 2 * k); }; var result = val.Compute(); Assert.AreEqual(10, current); Assert.AreEqual(4.5, result.Training.Mean); Assert.AreEqual(9.0, result.Validation.Mean); Assert.AreEqual( 2 * result.Training.StandardDeviation, result.Validation.StandardDeviation); Assert.AreEqual(val.Folds.Length, result.Training.Sizes.Length); Assert.AreEqual(val.Folds.Length, result.Validation.Sizes.Length); for (int i = 0; i < result.Training.Sizes.Length; i++) Assert.AreEqual(90, result.Training.Sizes[i]); for (int i = 0; i < result.Validation.Sizes.Length; i++) Assert.AreEqual(10, result.Validation.Sizes[i]); }
private void button1_Click(object sender, EventArgs e) { // Creates a matrix from the source data table double[,] sourceMatrix = (dgvLearningSource.DataSource as DataTable).ToMatrix(out sourceColumns); // Get only the input vector values var inputs = sourceMatrix.Submatrix(0, sourceMatrix.GetLength(0) - 1, 0, 1).ToArray(); // Get only the label outputs var outputs = new int[sourceMatrix.GetLength(0)]; for (int i = 0; i < outputs.Length; i++) outputs[i] = (int)sourceMatrix[i, 2]; var cv = new CrossValidation<KernelSupportVectorMachine>(inputs.Length, 10); cv.Fitting = (int k, int[] training, int[] testing) => { var trainingInputs = inputs.Submatrix(training); var trainingOutputs = outputs.Submatrix(training); var testingInputs = inputs.Submatrix(testing); var testingOutputs = outputs.Submatrix(testing); // Create the specified Kernel IKernel kernel = getKernel(); // Creates the Support Vector Machine using the selected kernel var svm = new KernelSupportVectorMachine(kernel, 2); // Creates a new instance of the SMO Learning Algortihm var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs); // Set learning parameters smo.Complexity = (double)numC.Value; smo.Tolerance = (double)numT.Value; // Run double trainingError = smo.Run(); double validationError = smo.ComputeError(testingInputs, testingOutputs); return new CrossValidationValues<KernelSupportVectorMachine>(svm, trainingError, validationError); }; var result = cv.Compute(); }