public void RunTest()
{
var dataset = SequentialMinimalOptimizationTest.yinyang;
double[][] inputs = dataset.Submatrix(null, 0, 1).ToJagged();
int[] labels = dataset.GetColumn(2).ToInt32();
Accord.Math.Random.Generator.Seed = 0;
var svm = new SupportVectorMachine(inputs: 2);
var teacher = new ProbabilisticDualCoordinateDescent(svm, inputs, labels);
teacher.Tolerance = 1e-10;
teacher.UseComplexityHeuristic = true;
Assert.IsFalse(svm.IsProbabilistic);
double error = teacher.Run();
Assert.IsTrue(svm.IsProbabilistic);
double[] weights = svm.ToWeights();
Assert.AreEqual(0.13, error);
Assert.AreEqual(3, weights.Length);
Assert.AreEqual(-0.52913278486359605, weights[0], 1e-4);
Assert.AreEqual(-1.6426069611746976, weights[1], 1e-4);
Assert.AreEqual(-0.77766953652287762, weights[2], 1e-4);
Assert.AreEqual(svm.Threshold, weights[0]);
}
public void Run()
{
// Example AND problem
double[][] inputs =
{
new double[] { 0, 0 }, // 0 and 0: 0 (label -1)
new double[] { 0, 1 }, // 0 and 1: 0 (label -1)
new double[] { 1, 0 }, // 1 and 0: 0 (label -1)
new double[] { 1, 1 } // 1 and 1: 1 (label +1)
};
// Dichotomy SVM outputs should be given as [-1;+1]
int[] labels =
{
// 0, 0, 0, 1
-1, -1, -1, 1
};
// Create a Support Vector Machine for the given inputs
SupportVectorMachine machine = new SupportVectorMachine(inputs[0].Length);
// Instantiate a new learning algorithm for SVMs
SequentialMinimalOptimization smo = new SequentialMinimalOptimization(machine, inputs, labels);
// Set up the learning algorithm
smo.Complexity = 1.0;
// Run the learning algorithm
double error = smo.Run();
// Compute the decision output for one of the input vectors
int decision = System.Math.Sign(machine.Compute(inputs[0]));
}
public void RunTest()
{
double[][] input =
{
new double[] { 55, 0 }, // 0 - no cancer
new double[] { 28, 0 }, // 0
new double[] { 65, 1 }, // 0
new double[] { 46, 0 }, // 1 - have cancer
new double[] { 86, 1 }, // 1
new double[] { 56, 1 }, // 1
new double[] { 85, 0 }, // 0
new double[] { 33, 0 }, // 0
new double[] { 21, 1 }, // 0
new double[] { 42, 1 }, // 1
};
double[] output =
{
0, 0, 0, 1, 1, 1, 0, 0, 0, 1
};
int[] labels = output.Apply(x => x > 0 ? +1 : -1);
var svm = new SupportVectorMachine(inputs: 2);
var teacher = new ProbabilisticNewtonMethod(svm, input, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
Assert.IsFalse(svm.IsProbabilistic);
double error = teacher.Run();
Assert.IsTrue(svm.IsProbabilistic);
Assert.AreEqual(0.2, error);
Assert.AreEqual(0.02064511826338301, svm.SupportVectors[0][0]);
Assert.AreEqual(1.767889310996118, svm.SupportVectors[0][1]);
Assert.AreEqual(-2.4577464317497455, svm.Threshold);
var regression = LogisticRegression.FromWeights(svm.ToWeights());
double[] actual = new double[output.Length];
for (int i = 0; i < actual.Length; i++)
actual[i] = regression.Compute(input[i]);
double ageOdds = regression.GetOddsRatio(1); // 1.0208597028836701
double smokeOdds = regression.GetOddsRatio(2); // 5.8584748789881331
Assert.AreEqual(1.0208597028836701, ageOdds, 1e-4);
Assert.AreEqual(5.8584748789881331, smokeOdds, 1e-4);
Assert.IsFalse(Double.IsNaN(ageOdds));
Assert.IsFalse(Double.IsNaN(smokeOdds));
Assert.AreEqual(-2.4577464307294092, regression.Intercept, 1e-8);
Assert.AreEqual(-2.4577464307294092, regression.Coefficients[0], 1e-8);
Assert.AreEqual(0.020645118265359252, regression.Coefficients[1], 1e-8);
Assert.AreEqual(1.7678893101571855, regression.Coefficients[2], 1e-8);
}
public void Setup()
{
ksvm = new SupportVectorMachine<Polynomial>(inputs: 2, kernel: new Polynomial(2));
smo = new SequentialMinimalOptimization<Polynomial>()
{
Model = ksvm
};
}
public SupportVectorMachine<Polynomial> v3_1_0()
{
ksvm = new SupportVectorMachine<Polynomial>(inputs: 2, kernel: new Polynomial(2));
smo = new SequentialMinimalOptimization<Polynomial>()
{
Model = ksvm
};
smo.Learn(inputs, outputs);
return ksvm;
}
private static void and()
{
// Create a simple binary AND
// classification problem:
double[][] problem =
{
// a b a + b
new double[] { 0, 0, 0 },
new double[] { 0, 1, 0 },
new double[] { 1, 0, 0 },
new double[] { 1, 1, 1 },
};
// Get the two first columns as the problem
// inputs and the last column as the output
// input columns
double[][] inputs = problem.GetColumns(0, 1);
// output column
int[] outputs = problem.GetColumn(2).ToInt32();
// Plot the problem on screen
ScatterplotBox.Show("AND", inputs, outputs).Hold();
// However, SVMs expect the output value to be
// either -1 or +1. As such, we have to convert
// it so the vector contains { -1, -1, -1, +1 }:
//
outputs = outputs.Apply(x => x == 0 ? -1 : 1);
// Create a new linear-SVM for two inputs (a and b)
SupportVectorMachine svm = new SupportVectorMachine(inputs: 2);
// Create a L2-regularized L2-loss support vector classification
var teacher = new LinearDualCoordinateDescent(svm, inputs, outputs)
{
Loss = Loss.L2,
Complexity = 1000,
Tolerance = 1e-5
};
// Learn the machine
double error = teacher.Run(computeError: true);
// Compute the machine's answers for the learned inputs
int[] answers = inputs.Apply(x => Math.Sign(svm.Compute(x)));
// Plot the results
ScatterplotBox.Show("SVM's answer", inputs, answers).Hold();
}
public void RunTest()
{
Accord.Math.Tools.SetupGenerator(0);
// Example regression problem. Suppose we are trying
// to model the following equation: f(x, y) = 2x + y
double[][] inputs = // (x, y)
{
new double[] { 0, 1 }, // 2*0 + 1 = 1
new double[] { 4, 3 }, // 2*4 + 3 = 11
new double[] { 8, -8 }, // 2*8 - 8 = 8
new double[] { 2, 2 }, // 2*2 + 2 = 6
new double[] { 6, 1 }, // 2*6 + 1 = 13
new double[] { 5, 4 }, // 2*5 + 4 = 14
new double[] { 9, 1 }, // 2*9 + 1 = 19
new double[] { 1, 6 }, // 2*1 + 6 = 8
};
double[] outputs = // f(x, y)
{
1, 11, 8, 6, 13, 14, 19, 8
};
// Create a new linear Support Vector Machine
var machine = new SupportVectorMachine(inputs: 2);
// Create the linear regression coordinate descent teacher
var learn = new LinearRegressionCoordinateDescent(machine, inputs, outputs)
{
Complexity = 10000000,
Epsilon = 1e-10
};
// Run the learning algorithm
double error = learn.Run();
// Compute the answer for one particular example
double fxy = machine.Compute(inputs[0]); // 1.000
// Check for correct answers
double[] answers = new double[inputs.Length];
for (int i = 0; i < answers.Length; i++)
answers[i] = machine.Compute(inputs[i]);
Assert.AreEqual(1.0, fxy, 1e-5);
for (int i = 0; i < outputs.Length; i++)
Assert.AreEqual(outputs[i], answers[i], 1e-2);
}
public void LearnTest()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] xor =
{
-1,
1,
1,
-1
};
var kernel = new Polynomial(2, 0.0);
double[][] augmented = new double[inputs.Length][];
for (int i = 0; i < inputs.Length; i++)
augmented[i] = kernel.Transform(inputs[i]);
SupportVectorMachine machine = new SupportVectorMachine(augmented[0].Length);
// Create the Least Squares Support Vector Machine teacher
var learn = new LinearDualCoordinateDescent(machine, augmented, xor);
// Run the learning algorithm
double error = learn.Run();
Assert.AreEqual(0, error);
int[] output = augmented.Apply(p => Math.Sign(machine.Compute(p)));
for (int i = 0; i < output.Length; i++)
Assert.AreEqual(System.Math.Sign(xor[i]), System.Math.Sign(output[i]));
}
public void ComplexityHeuristicTest()
{
var dataset = yinyang;
double[][] inputs = dataset.Submatrix(null, 0, 1).ToArray();
int[] labels = dataset.GetColumn(2).ToInt32();
var linear = new SupportVectorMachine(inputs[0].Length);
Linear kernel = new Linear(0);
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo1 = new SequentialMinimalOptimization(machine, inputs, labels);
smo1.UseClassProportions = true;
smo1.UseComplexityHeuristic = true;
double e1 = smo1.Run();
var smo2 = new SequentialMinimalOptimization(linear, inputs, labels);
smo2.UseClassProportions = true;
smo2.UseComplexityHeuristic = true;
double e2 = smo2.Run();
Assert.AreEqual(smo1.Complexity, smo2.Complexity);
Assert.AreEqual(e1, e2);
}
private static void linearSvm(double[][] inputs, int[] outputs)
{
// Create a linear binary machine with 2 inputs
var svm = new SupportVectorMachine(inputs: 2);
// Create a L2-regularized L2-loss optimization algorithm for
// the dual form of the learning problem. This is *exactly* the
// same method used by LIBLINEAR when specifying -s 1 in the
// command line (i.e. L2R_L2LOSS_SVC_DUAL).
//
var teacher = new LinearCoordinateDescent(svm, inputs, outputs);
// Teach the vector machine
double error = teacher.Run();
// Classify the samples using the model
int[] answers = inputs.Apply(svm.Compute).Apply(System.Math.Sign);
// Plot the results
ScatterplotBox.Show("Expected results", inputs, outputs);
ScatterplotBox.Show("LinearSVM results", inputs, answers);
// Grab the index of multipliers higher than 0
int[] idx = teacher.Lagrange.Find(x => x > 0);
// Select the input vectors for those
double[][] sv = inputs.Submatrix(idx);
// Plot the support vectors selected by the machine
ScatterplotBox.Show("Support vectors", sv).Hold();
}
public void ComputeTest5()
{
var dataset = SequentialMinimalOptimizationTest.yinyang;
var inputs = dataset.Submatrix(null, 0, 1).ToArray();
var labels = dataset.GetColumn(2).ToInt32();
var kernel = new Polynomial(2, 0);
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.UseComplexityHeuristic = true;
double error = smo.Run();
Assert.AreEqual(0.2, error);
Assert.AreEqual(0.11714451552090824, smo.Complexity);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
actual[i] = Math.Sign(machine.Compute(inputs[i]));
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(20, matrix.FalseNegatives);
Assert.AreEqual(0, matrix.FalsePositives);
Assert.AreEqual(30, matrix.TruePositives);
Assert.AreEqual(50, matrix.TrueNegatives);
}
{
Accord.Math.Tools.SetupGenerator(0);
var projection = inputs.Apply(kernel.Transform);
var machine = new SupportVectorMachine(projection[0].Length);
var smo = new LinearNewtonMethod(machine, projection, labels);
smo.UseComplexityHeuristic = true;
double error = smo.Run();
Assert.AreEqual(0.18, error);
Assert.AreEqual(0.11714451552090821, smo.Complexity, 1e-15);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
actual[i] = Math.Sign(machine.Compute(projection[i]));
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(17, matrix.FalseNegatives);
Assert.AreEqual(1, matrix.FalsePositives);
Assert.AreEqual(33, matrix.TruePositives);
Assert.AreEqual(49, matrix.TrueNegatives);
}
}
private static void cancer()
{
// Create a new LibSVM sparse format data reader
// to read the Wisconsin's Breast Cancer dataset
//
var reader = new SparseReader("examples-sparse.txt");
int[] outputs; // Read the classification problem into dense memory
double[][] inputs = reader.ReadToEnd(sparse: false, labels: out outputs);
// The dataset has output labels as 4 and 2. We have to convert them
// into negative and positive labels so they can be properly processed.
//
outputs = outputs.Apply(x => x == 2 ? -1 : +1);
// Create a new linear-SVM for the problem dimensions
var svm = new SupportVectorMachine(inputs: reader.Dimensions);
// Create a learning algorithm for the problem's dimensions
var teacher = new LinearDualCoordinateDescent(svm, inputs, outputs)
{
Loss = Loss.L2,
Complexity = 1000,
Tolerance = 1e-5
};
// Learn the classification
double error = teacher.Run();
// Compute the machine's answers for the learned inputs
int[] answers = inputs.Apply(x => Math.Sign(svm.Compute(x)));
// Create a confusion matrix to show the machine's performance
var m = new ConfusionMatrix(predicted: answers, expected: outputs);
// Show it onscreen
DataGridBox.Show(new ConfusionMatrixView(m));
}
public void ReceiverOperatingCharacteristicConstructorTest3()
{
// This example shows how to measure the accuracy of a
// binary classifier using a ROC curve. For this example,
// we will be creating a Support Vector Machine trained
// on the following instances:
double[][] inputs =
{
// Those are from class -1
new double[] { 2, 4, 0 },
new double[] { 5, 5, 1 },
new double[] { 4, 5, 0 },
new double[] { 2, 5, 5 },
new double[] { 4, 5, 1 },
new double[] { 4, 5, 0 },
new double[] { 6, 2, 0 },
new double[] { 4, 1, 0 },
// Those are from class +1
new double[] { 1, 4, 5 },
new double[] { 7, 5, 1 },
new double[] { 2, 6, 0 },
new double[] { 7, 4, 7 },
new double[] { 4, 5, 0 },
new double[] { 6, 2, 9 },
new double[] { 4, 1, 6 },
new double[] { 7, 2, 9 },
};
int[] outputs =
{
-1, -1, -1, -1, -1, -1, -1, -1, // fist eight from class -1
+1, +1, +1, +1, +1, +1, +1, +1 // last eight from class +1
};
// Create a linear Support Vector Machine with 4 inputs
SupportVectorMachine machine = new SupportVectorMachine(inputs: 3);
// Create the sequential minimal optimization teacher
SequentialMinimalOptimization learn = new SequentialMinimalOptimization(machine, inputs, outputs);
// Run the learning algorithm
double error = learn.Run();
// Extract the input labels predicted by the machine
double[] predicted = new double[inputs.Length];
for (int i = 0; i < predicted.Length; i++)
predicted[i] = machine.Compute(inputs[i]);
// Create a new ROC curve to assess the performance of the model
var roc = new ReceiverOperatingCharacteristic(outputs, predicted);
roc.Compute(100); // Compute a ROC curve with 100 points
/*
// Generate a connected scatter plot for the ROC curve and show it on-screen
ScatterplotBox.Show(roc.GetScatterplot(includeRandom: true), nonBlocking: true)
.SetSymbolSize(0) // do not display data points
.SetLinesVisible(true) // show lines connecting points
.SetScaleTight(true) // tighten the scale to points
.WaitForClose();
*/
Assert.AreEqual(0.7890625, roc.Area);
// Assert.AreEqual(0.1174774, roc.StandardError, 1e-6); HanleyMcNeil
Assert.AreEqual(0.11958120746409709, roc.StandardError, 1e-6);
}
private static void xor()
{
// Create a simple binary XOR
// classification problem:
double[][] problem =
{
// a b a XOR b
new double[] { 0, 0, 0 },
new double[] { 0, 1, 1 },
new double[] { 1, 0, 1 },
new double[] { 1, 1, 0 },
};
// Get the two first columns as the problem
// inputs and the last column as the output
// input columns
double[][] inputs = problem.GetColumns(0, 1);
// output column
int[] outputs = problem.GetColumn(2).ToInt32();
// Plot the problem on screen
ScatterplotBox.Show("XOR", inputs, outputs).Hold();
// However, SVMs expect the output value to be
// either -1 or +1. As such, we have to convert
// it so the vector contains { -1, -1, -1, +1 }:
//
outputs = outputs.Apply(x => x == 0 ? -1 : 1);
// Create a new linear-SVM for two inputs (a and b)
SupportVectorMachine svm = new SupportVectorMachine(inputs: 2);
// Create a L2-regularized L2-loss support vector classification
var teacher = new LinearDualCoordinateDescent(svm, inputs, outputs)
{
Loss = Loss.L2,
Complexity = 1000,
Tolerance = 1e-5
};
// Learn the machine
double error = teacher.Run(computeError: true);
// Compute the machine's answers for the learned inputs
int[] answers = inputs.Apply(x => Math.Sign(svm.Compute(x)));
// Plot the results
ScatterplotBox.Show("SVM's answer", inputs, answers).Hold();
// Use an explicit kernel expansion to transform the
// non-linear classification problem into a linear one
//
// Create a quadratic kernel
Quadratic quadratic = new Quadratic(constant: 1);
// Project the inptus into a higher dimensionality space
double[][] expansion = inputs.Apply(quadratic.Transform);
// Create a new linear-SVM for the transformed input space
svm = new SupportVectorMachine(inputs: expansion[0].Length);
// Create the same learning algorithm in the expanded input space
teacher = new LinearDualCoordinateDescent(svm, expansion, outputs)
{
Loss = Loss.L2,
Complexity = 1000,
Tolerance = 1e-5
};
// Learn the machine
error = teacher.Run(computeError: true);
// Compute the machine's answers for the learned inputs
answers = expansion.Apply(x => Math.Sign(svm.Compute(x)));
// Plot the results
ScatterplotBox.Show("SVM's answer", inputs, answers).Hold();
}
public void SequentialMinimalOptimizationConstructorTest()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] or =
{
0,
0,
0,
+1
};
// Create Kernel Support Vector Machine with a Polynomial Kernel of 2nd degree
SupportVectorMachine machine = new SupportVectorMachine(inputs[0].Length);
bool thrown = false;
try
{
SequentialMinimalOptimization learn = new SequentialMinimalOptimization(machine, inputs, or);
}
catch (ArgumentOutOfRangeException)
{
thrown = true;
}
Assert.IsTrue(thrown);
}
public void RunTest2()
{
var dataset = SequentialMinimalOptimizationTest.yinyang;
double[][] inputs = dataset.Submatrix(null, 0, 1).ToArray();
int[] labels = dataset.GetColumn(2).ToInt32();
var svm = new SupportVectorMachine(inputs: 2);
var teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
double error = teacher.Run();
double[] weights = svm.ToWeights();
Assert.AreEqual(0.12, error);
Assert.AreEqual(3, weights.Length);
Assert.AreEqual(-1.3231203367770932, weights[0]);
Assert.AreEqual(-3.0227742288788493, weights[1]);
Assert.AreEqual(-0.73074823290553259, weights[2]);
Assert.AreEqual(svm.Threshold, weights[0]);
}
/// <summary>
/// Creates a Support Vector Machine and teaches it to recognize
/// the previously loaded dataset using the current UI settings.
/// </summary>
///
private void btnCreate_Click(object sender, EventArgs e)
{
if (dgvLearningSource.DataSource == null)
{
MessageBox.Show("Please load some data first.");
return;
}
// Finishes and save any pending changes to the given data
dgvLearningSource.EndEdit();
// Creates a matrix from the entire source data table
double[,] table = (dgvLearningSource.DataSource as DataTable).ToMatrix(out columnNames);
// Get only the input vector values (first two columns)
double[][] inputs = table.GetColumns(0, 1).ToJagged();
// Get only the output labels (last column)
int[] outputs = table.GetColumn(2).ToInt32();
// Creates a new instance of the SMO learning algorithm
var smo = new SequentialMinimalOptimization<IKernel>()
{
// Set learning parameters
Complexity = (double)numC.Value,
Tolerance = (double)numT.Value,
PositiveWeight = (double)numPositiveWeight.Value,
NegativeWeight = (double)numNegativeWeight.Value,
Kernel = createKernel()
};
try
{
// Run
svm = smo.Learn(inputs, outputs);
lbStatus.Text = "Training complete!";
}
catch (ConvergenceException)
{
lbStatus.Text = "Convergence could not be attained. "+
"The learned machine might still be usable.";
}
createSurface(table);
// Check if we got support vectors
if (svm.SupportVectors == null || svm.SupportVectors.Length == 0)
{
dgvSupportVectors.DataSource = null;
graphSupportVectors.GraphPane.CurveList.Clear();
return;
}
// Show support vectors on the Support Vectors tab page
double[][] supportVectorsWeights = svm.SupportVectors.InsertColumn(svm.Weights);
string[] supportVectorNames = columnNames.RemoveAt(columnNames.Length - 1).Concatenate("Weight");
dgvSupportVectors.DataSource = new ArrayDataView(supportVectorsWeights, supportVectorNames);
// Show the support vector labels on the scatter plot
double[] supportVectorLabels = new double[svm.SupportVectors.Length];
for (int i = 0; i < supportVectorLabels.Length; i++)
{
int j = inputs.Find(sv => sv == svm.SupportVectors[i])[0];
supportVectorLabels[i] = outputs[j];
}
double[][] graph = svm.SupportVectors.InsertColumn(supportVectorLabels);
CreateScatterplot(graphSupportVectors, graph.ToMatrix());
}
public void TransformTest()
{
var inputs = yinyang.Submatrix(null, 0, 1).ToArray();
var labels = yinyang.GetColumn(2).ToInt32();
ConfusionMatrix actual, expected;
SequentialMinimalOptimization a, b;
var kernel = new Polynomial(2, 0);
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
a = new SequentialMinimalOptimization(machine, inputs, labels);
a.UseComplexityHeuristic = true;
a.Run();
int[] values = new int[labels.Length];
for (int i = 0; i < values.Length; i++)
values[i] = Math.Sign(machine.Compute(inputs[i]));
expected = new ConfusionMatrix(values, labels);
}
{
var projection = inputs.Apply(kernel.Transform);
var machine = new SupportVectorMachine(projection[0].Length);
b = new SequentialMinimalOptimization(machine, projection, labels);
b.UseComplexityHeuristic = true;
b.Run();
int[] values = new int[labels.Length];
for (int i = 0; i < values.Length; i++)
values[i] = Math.Sign(machine.Compute(projection[i]));
actual = new ConfusionMatrix(values, labels);
}
Assert.AreEqual(a.Complexity, b.Complexity, 1e-15);
Assert.AreEqual(expected.TrueNegatives, actual.TrueNegatives);
Assert.AreEqual(expected.TruePositives, actual.TruePositives);
Assert.AreEqual(expected.FalseNegatives, actual.FalseNegatives);
Assert.AreEqual(expected.FalsePositives, actual.FalsePositives);
}
/// <summary>
/// Creates a Support Vector Machine and teaches it to recognize
/// the previously loaded dataset using the current UI settings.
/// </summary>
///
private void btnCreate_Click(object sender, EventArgs e)
{
if (dgvLearningSource.DataSource == null)
{
MessageBox.Show("Please load some data first.");
return;
}
// Finishes and save any pending changes to the given data
dgvLearningSource.EndEdit();
// Creates a matrix from the entire source data table
double[,] table = (dgvLearningSource.DataSource as DataTable).ToMatrix(out columnNames);
// Get only the input vector values (first two columns)
double[][] inputs = table.GetColumns(0, 1).ToArray();
// Get only the output labels (last column)
int[] outputs = table.GetColumn(2).ToInt32();
// Create a sparse logistic learning algorithm
var pcd = new ProbabilisticCoordinateDescent()
{
// Set learning parameters
Complexity = (double)numC.Value,
Tolerance = (double)numT.Value,
PositiveWeight = (double)numPositiveWeight.Value,
NegativeWeight = (double)numNegativeWeight.Value,
};
try
{
// Run
svm = pcd.Learn(inputs, outputs);
lbStatus.Text = "Training complete!";
}
catch (ConvergenceException)
{
lbStatus.Text = "Convergence could not be attained. " +
"The learned machine might still be usable.";
}
svm.Compress(); // reduce support vectors to a single weight vector
Trace.Assert(svm.SupportVectors.Length == 1);
Trace.Assert(svm.Weights.Length == 1);
createSurface(table);
// Show feature weight importance
double[] weights = svm.SupportVectors[0].Abs();
string[] featureNames = columnNames.RemoveAt(columnNames.Length - 1);
dgvSupportVectors.DataSource = new ArrayDataView(weights, featureNames);
CreateBarGraph(weights, featureNames);
}
private void btnCreate_Click(object sender, EventArgs e)
{
if (dgvLearningSource.DataSource == null)
{
MessageBox.Show("Please load some data first.");
return;
}
// Finishes and save any pending changes to the given data
dgvLearningSource.EndEdit();
// Creates a matrix from the entire source data table
double[,] table = (dgvLearningSource.DataSource as DataTable).ToMatrix(out columnNames);
// Get only the input vector values (first two columns)
double[][] inputs = table.GetColumns(0).ToArray();
// Get only the outputs (last column)
double[] outputs = table.GetColumn(1);
// Create the specified Kernel
IKernel kernel = createKernel();
// Creates a new SMO for regression learning algorithm
var smo = new SequentialMinimalOptimizationRegression()
{
// Set learning parameters
Complexity = (double)numC.Value,
Tolerance = (double)numT.Value,
Epsilon = (double)numEpsilon.Value
};
try
{
// Run
svm = smo.Learn(inputs, outputs);
lbStatus.Text = "Training complete!";
}
catch (ConvergenceException)
{
lbStatus.Text = "Convergence could not be attained. " +
"The learned machine might still be usable.";
}
// Check if we got support vectors
if (svm.SupportVectors.Length == 0)
{
dgvSupportVectors.DataSource = null;
graphSupportVectors.GraphPane.CurveList.Clear();
return;
}
// Show support vectors on the Support Vectors tab page
double[][] supportVectorsWeights = svm.SupportVectors.InsertColumn(svm.Weights);
string[] supportVectorNames = columnNames.RemoveAt(columnNames.Length - 1).Concatenate("Weight");
dgvSupportVectors.DataSource = new ArrayDataView(supportVectorsWeights, supportVectorNames);
// Show the support vector labels on the scatter plot
double[] supportVectorLabels = new double[svm.SupportVectors.Length];
for (int i = 0; i < supportVectorLabels.Length; i++)
{
int j = inputs.Find(sv => sv == svm.SupportVectors[i])[0];
supportVectorLabels[i] = outputs[j];
}
double[][] graph = svm.SupportVectors.InsertColumn(supportVectorLabels);
CreateScatterplot(graphSupportVectors, graph.ToMatrix());
// Get the ranges for each variable (X and Y)
DoubleRange range = table.GetColumn(0).GetRange();
double[][] map = Vector.Interval(range, 0.05).ToArray();
// Classify each point in the Cartesian coordinate system
double[] result = map.Apply(svm.Compute);
double[,] surface = map.ToMatrix().InsertColumn(result);
CreateScatterplot(zedGraphControl2, surface);
}
public void TrainTest2()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] or =
{
-1,
-1,
-1,
+1
};
// Create Kernel Support Vector Machine with a Polynomial Kernel of 2nd degree
SupportVectorMachine machine = new SupportVectorMachine(inputs[0].Length);
// Create the sequential minimal optimization teacher
SequentialMinimalOptimization learn = new SequentialMinimalOptimization(machine, inputs, or);
learn.Complexity = 1;
// Run the learning algorithm
learn.Run();
double[] output = machine.Compute(inputs);
for (int i = 0; i < output.Length; i++)
{
bool sor = or[i] >= 0;
bool sou = output[i] >= 0;
Assert.AreEqual(sor, sou);
}
}
public static void train_one(Problem prob, Parameters param, out double[] w, double Cp, double Cn)
{
double[][] inputs = prob.Inputs;
int[] labels = prob.Outputs.Apply(x => x >= 0 ? 1 : -1);
double eps = param.Tolerance;
int pos = 0;
for (int i = 0; i < labels.Length; i++)
if (labels[i] >= 0) pos++;
int neg = prob.Outputs.Length - pos;
double primal_solver_tol = eps * Math.Max(Math.Min(pos, neg), 1.0) / prob.Inputs.Length;
SupportVectorMachine svm = new SupportVectorMachine(prob.Dimensions);
ISupportVectorMachineLearning teacher = null;
switch (param.Solver)
{
case LibSvmSolverType.L2RegularizedLogisticRegression:
// l2r_lr_fun
teacher = new ProbabilisticNewtonMethod(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol
}; break;
case LibSvmSolverType.L2RegularizedL2LossSvc:
// fun_obj=new l2r_l2_svc_fun(prob, C);
teacher = new LinearNewtonMethod(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol
}; break;
case LibSvmSolverType.L2RegularizedL2LossSvcDual:
// solve_l2r_l1l2_svc(prob, w, eps, Cp, Cn, L2R_L2LOSS_SVC_DUAL);
teacher = new LinearCoordinateDescent(svm, inputs, labels)
{
Loss = Loss.L2,
PositiveWeight = Cp,
NegativeWeight = Cn,
}; break;
case LibSvmSolverType.L2RegularizedL1LossSvcDual:
// solve_l2r_l1l2_svc(prob, w, eps, Cp, Cn, L2R_L1LOSS_SVC_DUAL);
teacher = new LinearCoordinateDescent(svm, inputs, labels)
{
Loss = Loss.L1,
PositiveWeight = Cp,
NegativeWeight = Cn,
}; break;
case LibSvmSolverType.L1RegularizedLogisticRegression:
// solve_l1r_lr(&prob_col, w, primal_solver_tol, Cp, Cn);
teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol
}; break;
case LibSvmSolverType.L2RegularizedLogisticRegressionDual:
// solve_l2r_lr_dual(prob, w, eps, Cp, Cn);
teacher = new ProbabilisticDualCoordinateDescent(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol,
}; break;
}
Trace.WriteLine("Training " + param.Solver);
// run the learning algorithm
var sw = Stopwatch.StartNew();
double error = teacher.Run();
sw.Stop();
// save the solution
w = svm.ToWeights();
Trace.WriteLine(String.Format("Finished {0}: {1} in {2}",
param.Solver, error, sw.Elapsed));
}
public void SequentialMinimalOptimizationConstructorTest()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] or =
{
0,
0,
0,
+1
};
// Create Kernel Support Vector Machine with a Polynomial Kernel of 2nd degree
SupportVectorMachine machine = new SupportVectorMachine(inputs[0].Length);
var learn = new SequentialMinimalOptimization(machine, inputs, or);
learn.Run();
for (int i = 0; i < inputs.Length; i++)
{
bool actual = machine.Decide(inputs[i]);
Assert.AreEqual(or[i] > 0, actual);
}
}
/// <summary>
/// Creates a new <see cref="SupportVectorMachine"/> that is
/// completely equivalent to a <see cref="LogisticRegression"/>.
/// </summary>
///
/// <param name="regression">The <see cref="LogisticRegression"/> to be converted.</param>
///
/// <returns>
/// A <see cref="SupportVectorMachine"/> whose linear weights are
/// equivalent to the given <see cref="LogisticRegression"/>'s
/// <see cref="GeneralizedLinearRegression.Coefficients"> linear
/// coefficients</see>, properly configured with a <see cref="LogLinkFunction"/>.
/// </returns>
///
public static SupportVectorMachine FromLogisticRegression(LogisticRegression regression)
{
double[] weights = regression.Coefficients;
var svm = new SupportVectorMachine(regression.Inputs);
for (int i = 0; i < svm.weights.Length; i++)
svm.Weights[i] = weights[i + 1];
svm.Threshold = regression.Intercept;
svm.Link = new LogitLinkFunction(1, 0);
return svm;
}
public void KernelTest1()
{
var dataset = SequentialMinimalOptimizationTest.yinyang;
double[][] inputs = dataset.Submatrix(null, 0, 1).ToArray();
int[] labels = dataset.GetColumn(2).ToInt32();
double e1, e2;
double[] w1, w2;
{
Accord.Math.Tools.SetupGenerator(0);
var svm = new SupportVectorMachine(inputs: 2);
var teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
e1 = teacher.Run();
w1 = svm.ToWeights();
}
{
Accord.Math.Tools.SetupGenerator(0);
var svm = new KernelSupportVectorMachine(new Linear(0), inputs: 2);
var teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
e2 = teacher.Run();
w2 = svm.ToWeights();
}
Assert.AreEqual(e1, e2);
Assert.AreEqual(w1.Length, w2.Length);
Assert.AreEqual(w1[0], w2[0]);
Assert.AreEqual(w1[1], w2[1]);
Assert.AreEqual(w1[2], w2[2]);
}
/// <summary>
/// Creates a new linear <see cref="SupportVectorMachine"/>
/// with the given set of linear <paramref name="weights"/>.
/// </summary>
///
/// <param name="weights">The machine's linear coefficients.</param>
///
/// <returns>
/// A <see cref="SupportVectorMachine"/> whose linear coefficients
/// are defined by the given <paramref name="weights"/> vector.
/// </returns>
///
public static SupportVectorMachine FromWeights(double[] weights)
{
var svm = new SupportVectorMachine(weights.Length - 1);
svm.Weights = new double[svm.Inputs];
for (int i = 0; i < svm.weights.Length; i++)
svm.Weights[i] = weights[i + 1];
svm.Threshold = weights[0];
return svm;
}
public void LearnTest5()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] positives =
{
1,
1,
1,
1
};
// Create Kernel Support Vector Machine with a Polynomial Kernel of 2nd degree
SupportVectorMachine machine = new SupportVectorMachine(inputs[0].Length);
// Create the sequential minimal optimization teacher
SequentialMinimalOptimization learn = new SequentialMinimalOptimization(machine, inputs, positives);
learn.Complexity = 1;
// Run the learning algorithm
double error = learn.Run();
Assert.AreEqual(0, error);
int[] output = inputs.Apply(p => (int)machine.Compute(p));
for (int i = 0; i < output.Length; i++)
{
bool sor = positives[i] >= 0;
bool sou = output[i] >= 0;
Assert.AreEqual(sor, sou);
}
}
public void GridsearchConstructorTest2()
{
Accord.Math.Tools.SetupGenerator(0);
// Example binary data
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] xor = // xor labels
{
-1, 1, 1, -1
};
// Declare the parameters and ranges to be searched
GridSearchRange[] ranges =
{
new GridSearchRange("complexity", new double[] { 0.00000001, 5.20, 0.30, 1000000, 0.50 } ),
};
// Instantiate a new Grid Search algorithm for Kernel Support Vector Machines
var gridsearch = new GridSearch<SupportVectorMachine>(ranges);
// Set the fitting function for the algorithm
gridsearch.Fitting = delegate(GridSearchParameterCollection parameters, out double error)
{
// The parameters to be tried will be passed as a function parameter.
double complexity = parameters["complexity"].Value;
// Use the parameters to build the SVM model
SupportVectorMachine svm = new SupportVectorMachine(2);
// Create a new learning algorithm for SVMs
SequentialMinimalOptimization smo = new SequentialMinimalOptimization(svm, inputs, xor);
smo.Complexity = complexity;
// Measure the model performance to return as an out parameter
error = smo.Run();
return svm; // Return the current model
};
{
// Declare some out variables to pass to the grid search algorithm
GridSearchParameterCollection bestParameters; double minError;
// Compute the grid search to find the best Support Vector Machine
SupportVectorMachine bestModel = gridsearch.Compute(out bestParameters, out minError);
// The minimum error should be zero because the problem is well-known.
Assert.AreEqual(minError, 0.5);
Assert.IsNotNull(bestModel);
Assert.IsNotNull(bestParameters);
Assert.AreEqual(bestParameters.Count, 1);
}
{
// Compute the grid search to find the best Support Vector Machine
var result = gridsearch.Compute();
// The minimum error should be zero because the problem is well-known.
Assert.AreEqual(result.Error, 0.5);
Assert.IsNotNull(result.Model);
Assert.AreEqual(5, result.Errors.Length);
Assert.AreEqual(5, result.Models.Length);
}
}
public void ComputeTest5()
{
var dataset = SequentialMinimalOptimizationTest.yinyang;
double[][] inputs = dataset.Submatrix(null, 0, 1).ToArray();
int[] labels = dataset.GetColumn(2).ToInt32();
var kernel = new Polynomial(2, 1);
Accord.Math.Tools.SetupGenerator(0);
var projection = inputs.Apply(kernel.Transform);
var machine = new SupportVectorMachine(projection[0].Length);
var smo = new LinearCoordinateDescent(machine, projection, labels)
{
Complexity = 1000000,
Tolerance = 1e-15
};
double error = smo.Run();
Assert.AreEqual(1000000.0, smo.Complexity, 1e-15);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
actual[i] = Math.Sign(machine.Compute(projection[i]));
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(6, matrix.FalseNegatives);
Assert.AreEqual(7, matrix.FalsePositives);
Assert.AreEqual(44, matrix.TruePositives);
Assert.AreEqual(43, matrix.TrueNegatives);
}
public void LeastSquaresConstructorTest()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] or =
{
0,
0,
0,
+1
};
// Create Kernel Support Vector Machine with a Polynomial Kernel of 2nd degree
var machine = new SupportVectorMachine(inputs[0].Length);
var learn = new LeastSquaresLearning(machine, inputs, or);
double error = learn.Run();
Assert.AreEqual(0, error);
{
int[] iout = new int[inputs.Length];
machine.ToMulticlass().Decide(inputs, iout);
for (int i = 0; i < iout.Length; i++)
Assert.AreEqual(or[i], iout[i]);
}
{
double[] dout = new double[inputs.Length];
machine.ToMulticlass().Decide(inputs, dout);
for (int i = 0; i < dout.Length; i++)
Assert.AreEqual(or[i], dout[i]);
}
{
bool[] bout = new bool[inputs.Length];
machine.Decide(inputs, bout);
Assert.IsFalse(bout[0]);
Assert.IsFalse(bout[1]);
Assert.IsFalse(bout[2]);
Assert.IsTrue(bout[3]);
}
{
int[][] iiout = Jagged.Create<int>(inputs.Length, 2);
machine.ToMulticlass().Decide(inputs, iiout);
for (int i = 0; i < iiout.Length; i++)
{
Assert.AreEqual(or[i], iiout[i][0]);
Assert.AreEqual(or[i], iiout[i][1] == 1 ? 0 : 1);
}
}
{
bool[][] bbout = Jagged.Create<bool>(inputs.Length, 2);
machine.ToMulticlass().Decide(inputs, bbout);
for (int i = 0; i < bbout.Length; i++)
{
Assert.AreEqual(or[i], bbout[i][0] ? 1 : 0);
Assert.AreEqual(or[i], bbout[i][1] ? 0 : 1);
}
}
}
