public void KernelFunctionCacheConstructorTest()
{
IKernel kernel = new Linear(1);
int cacheSize = 0;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(0, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
for (int i = 0; i < inputs.Length; i++)
{
double expected = i * i + 1;
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Usage);
}
//---------------------------------------------
/// <summary>
/// Runs the SMO algorithm.
/// </summary>
///
/// <param name="computeError">
/// True to compute error after the training
/// process completes, false otherwise. Default is true.
/// </param>
///
/// <returns>
/// The misclassification error rate of
/// the resulting support vector machine.
/// </returns>
///
public double Run(bool computeError)
{
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Initialize variables
int samples = inputs.Length;
int dimension = inputs[0].Length;
if (useComplexityHeuristic)
c = EstimateComplexity(kernel, inputs);
// Lagrange multipliers
Array.Clear(alpha, 0, alpha.Length);
if (isLinear) // Hyperplane weights
Array.Clear(weights, 0, weights.Length);
// Error cache
Array.Clear(errors, 0, errors.Length);
// Kernel evaluations cache
this.kernelCache = new KernelFunctionCache(kernel, inputs, cacheSize);
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = outputs.Find(x => x == +1)[0];
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = outputs.Find(x => x == -1)[0];
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Prepare indice sets
activeExamples = new HashSet<int>();
nonBoundExamples = new HashSet<int>();
atBoundsExamples = new HashSet<int>();
// Algorithm:
int numChanged = 0;
bool examineAll = true;
while (numChanged > 0 || examineAll)
{
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
if (examineExample(i)) numChanged++;
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c)
{
if (examineExample(i)) numChanged++;
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else // strategy == Strategy.WorstPair
{
bool success;
do { success = takeStep(i_upper, i_lower); }
while ((b_upper <= b_lower - 2.0 * tolerance) && success);
numChanged = 0;
}
}
if (examineAll)
examineAll = false;
else if (numChanged == 0)
examineAll = true;
}
// Store information about bounded examples
for (int i = 0; i < alpha.Length; i++)
if (alpha[i] == c) atBoundsExamples.Add(i);
if (isCompact)
{
// Store the hyperplane directly
machine.SupportVectors = null;
machine.Weights = weights;
machine.Threshold = -(b_lower + b_upper) / 2.0;
}
else
{
// Store Support Vectors in the SV Machine. Only vectors which have lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
int[] idx = new int[activeCount];
activeExamples.CopyTo(idx);
machine.SupportVectors = new double[activeCount][];
machine.Weights = new double[activeCount];
for (int i = 0; i < idx.Length; i++)
{
int j = idx[i];
machine.SupportVectors[i] = inputs[j];
machine.Weights[i] = alpha[j] * outputs[j];
}
machine.Threshold = -(b_lower + b_upper) / 2;
}
// Clear function cache
this.kernelCache.Clear();
// Compute error if required.
return (computeError) ? ComputeError(inputs, outputs) : 0.0;
}
/// <summary>
/// Runs the learning algorithm.
/// </summary>
///
/// <param name="token">A token to stop processing when requested.</param>
/// <param name="c">The complexity for each sample.</param>
protected override void Run(CancellationToken token, double[] c)
{
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Initialize variables
int samples = Inputs.Length;
int dimension = Inputs[0].Length;
this.c = c;
// Lagrange multipliers
Array.Clear(alpha, 0, alpha.Length);
if (IsLinear) // Hyperplane weights
Array.Clear(weights, 0, weights.Length);
// Error cache
Array.Clear(errors, 0, errors.Length);
// Kernel evaluations cache
this.kernelCache = new KernelFunctionCache(Kernel, Inputs, cacheSize);
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = Outputs.First(x => x > 0);
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = Outputs.First(x => x < 0);
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Prepare indices sets
activeExamples.Clear();
nonBoundExamples.Clear();
atBoundsExamples.Clear();
// Algorithm:
int numChanged = 0;
int wholeSetChecks = 0;
bool examineAll = true;
bool diverged = false;
bool shouldStop = false;
while ((numChanged > 0 || examineAll) && !shouldStop)
{
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
if (examineExample(i))
numChanged++;
wholeSetChecks++;
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c[i])
{
if (examineExample(i))
numChanged++;
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else // strategy == Strategy.WorstPair
{
int attempts = 0;
do
{
attempts++;
if (!takeStep(i_upper, i_lower))
break;
if (attempts > samples * maxChecks)
break;
}
while ((b_upper <= b_lower - 2.0 * tolerance));
numChanged = 0;
}
}
if (examineAll)
examineAll = false;
else if (numChanged == 0)
examineAll = true;
if (wholeSetChecks > maxChecks)
shouldStop = diverged = true;
if (token.IsCancellationRequested)
shouldStop = true;
}
// Store information about bounded examples
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] == c[i])
atBoundsExamples.Add(i);
}
if (isCompact)
{
// Store the hyperplane directly
Machine.SupportVectors = null;
Machine.Weights = weights;
Machine.Threshold = -(b_lower + b_upper) / 2.0;
}
else
{
// Store Support Vectors in the SV Machine. Only vectors which have Lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
int[] idx = new int[activeCount];
activeExamples.CopyTo(idx);
Machine.SupportVectors = new double[activeCount][];
Machine.Weights = new double[activeCount];
for (int i = 0; i < idx.Length; i++)
{
int j = idx[i];
Machine.SupportVectors[i] = Inputs[j];
Machine.Weights[i] = alpha[j] * Outputs[j];
}
Machine.Threshold = -(b_lower + b_upper) / 2;
}
// Clear function cache
this.kernelCache.Clear();
this.kernelCache = null;
if (diverged)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
}
public void KernelFunctionCacheConstructorTest2()
{
IKernel kernel = new Linear(1);
int cacheSize = inputs.Length;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(10, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
for (int i = 0; i < inputs.Length; i++)
{
double expected = i * i + 1;
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
int[] hits = { 0, 1, 3, 6, 10, 15, 21, 28, 36, 45 };
int[] miss = { 9, 17, 24, 30, 35, 39, 42, 44, 45, 45 };
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(hits[i], target.Hits);
Assert.AreEqual(miss[i], target.Misses);
}
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(45, target.Misses);
Assert.AreEqual(135, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
public void KernelFunctionCacheConstructorTest4()
{
IKernel kernel = new Linear();
int cacheSize = 100;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(inputs.Length, target.Size);
}
public void KernelFunctionCacheConstructorTest3()
{
IKernel kernel = new Linear(1);
int cacheSize = 5;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(5, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
for (int i = 0; i < inputs.Length; i++)
{
double expected = i * i + 1;
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(9, target.Hits);
Assert.AreEqual(81, target.Misses);
var snapshot = target.GetDataCache();
foreach (var entry in snapshot)
{
double a = target.GetOrCompute(entry.Key.Item1, entry.Key.Item2);
double b = target.GetOrCompute(entry.Key.Item2, entry.Key.Item1);
Assert.AreEqual(a, b);
}
Assert.AreEqual(81, target.Misses);
Assert.AreEqual(29, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
public void KernelFunctionCacheConstructorTest7()
{
double[][] inputs =
{
new double[] { 0, 1 },
new double[] { 1, 0 },
new double[] { 1, 1 },
};
IKernel kernel = new Polynomial(2);
int cacheSize = inputs.Length;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(3, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
// upper half
for (int i = 0; i < inputs.Length; i++)
{
for (int j = i + 1; j < inputs.Length; j++)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
var lruList1 = target.GetLeastRecentlyUsedList();
Assert.AreEqual(3, target.Misses);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(1.0, target.Usage);
// upper half, backwards
for (int i = inputs.Length - 1; i >= 0; i--)
{
for (int j = inputs.Length - 1; j >= i; j--)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(j, i);
Assert.AreEqual(expected, actual);
}
}
var lruList2 = target.GetLeastRecentlyUsedList();
Assert.IsTrue(lruList2.SequenceEqual(lruList1.Reverse()));
Assert.AreEqual(3, target.Misses);
Assert.AreEqual(3, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
public void KernelFunctionCacheConstructorTest6()
{
IKernel kernel = new Gaussian(0.6);
int cacheSize = inputs.Length;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(10, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
for (int i = 0; i < inputs.Length; i++)
{
double expected = kernel.Function(inputs[i], inputs[i]);
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(45, target.Misses);
Assert.AreEqual(135, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
/// <summary>
/// Runs the SMO algorithm.
/// </summary>
///
/// <param name="computeError">
/// True to compute error after the training
/// process completes, false otherwise. Default is true.
/// </param>
/// <param name="token">
/// A <see cref="CancellationToken"/> which can be used
/// to request the cancellation of the learning algorithm
/// when it is being run in another thread.
/// </param>
///
/// <returns>
/// The misclassification error rate of
/// the resulting support vector machine.
/// </returns>
///
public double Run(bool computeError, CancellationToken token)
{
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Initialize variables
int samples = inputs.Length;
int dimension = inputs[0].Length;
// Initialization heuristics
if (useComplexityHeuristic)
c = EstimateComplexity(kernel, inputs);
int[] positives = outputs.Find(x => x == +1);
int[] negatives = outputs.Find(x => x == -1);
// If all examples are positive or negative, terminate
// learning early by directly setting the threshold.
if (positives.Length == 0)
{
machine.SupportVectors = new double[0][];
machine.Weights = new double[0];
machine.Threshold = -1;
return 0;
}
if (negatives.Length == 0)
{
machine.SupportVectors = new double[0][];
machine.Weights = new double[0];
machine.Threshold = +1;
return 0;
}
if (useClassLabelProportion)
WeightRatio = positives.Length / (double)negatives.Length;
// Lagrange multipliers
Array.Clear(alpha, 0, alpha.Length);
if (isLinear) // Hyperplane weights
Array.Clear(weights, 0, weights.Length);
// Error cache
Array.Clear(errors, 0, errors.Length);
// Kernel evaluations cache
this.kernelCache = new KernelFunctionCache(kernel, inputs, cacheSize);
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = positives[0];
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = negatives[0];
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Prepare indice sets
activeExamples.Clear();
nonBoundExamples.Clear();
atBoundsExamples.Clear();
// Balance classes
bool balanced = positiveWeight == 1 && negativeWeight == 1;
positiveCost = c * positiveWeight;
negativeCost = c * negativeWeight;
// Algorithm:
int numChanged = 0;
int wholeSetChecks = 0;
bool examineAll = true;
bool diverged = false;
bool shouldStop = false;
while ((numChanged > 0 || examineAll) && !shouldStop)
{
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
if (examineExample(i)) numChanged++;
wholeSetChecks++;
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
if (balanced) // Assume balanced data
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c)
{
if (examineExample(i)) numChanged++;
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else // Use different weights for classes
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0)
{
if (outputs[i] == +1)
{
if (alpha[i] == positiveCost) continue;
}
else // outputs[i] == -1
{
if (alpha[i] == negativeCost) continue;
}
if (examineExample(i)) numChanged++;
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
}
else // strategy == Strategy.WorstPair
{
int attempts = 0;
do
{
attempts++;
if (!takeStep(i_upper, i_lower))
break;
if (attempts > samples * maxChecks)
break;
}
while ((b_upper <= b_lower - 2.0 * tolerance));
numChanged = 0;
}
}
if (examineAll)
examineAll = false;
else if (numChanged == 0)
examineAll = true;
if (wholeSetChecks > maxChecks)
shouldStop = diverged = true;
if (token.IsCancellationRequested)
shouldStop = true;
}
// Store information about bounded examples
if (balanced)
{
// Assume equal weights for classes
for (int i = 0; i < alpha.Length; i++)
if (alpha[i] == c) atBoundsExamples.Add(i);
}
else
{
// Use different weights for classes
for (int i = 0; i < alpha.Length; i++)
{
if (outputs[i] == +1)
{
if (alpha[i] == positiveCost)
atBoundsExamples.Add(i);
}
else // outputs[i] == -1
{
if (alpha[i] == negativeCost)
atBoundsExamples.Add(i);
}
}
}
if (isCompact)
{
// Store the hyperplane directly
machine.SupportVectors = null;
machine.Weights = weights;
machine.Threshold = -(b_lower + b_upper) / 2.0;
}
else
{
// Store Support Vectors in the SV Machine. Only vectors which have lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
int[] idx = new int[activeCount];
activeExamples.CopyTo(idx);
machine.SupportVectors = new double[activeCount][];
machine.Weights = new double[activeCount];
for (int i = 0; i < idx.Length; i++)
{
int j = idx[i];
machine.SupportVectors[i] = inputs[j];
machine.Weights[i] = alpha[j] * outputs[j];
}
machine.Threshold = -(b_lower + b_upper) / 2;
}
// Clear function cache
this.kernelCache.Clear();
this.kernelCache = null;
if (diverged)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
// Compute error if required.
return (computeError) ? ComputeError(inputs, outputs) : 0.0;
}
/// <summary>
/// Runs the LS-SVM algorithm.
/// </summary>
///
/// <param name="computeError">
/// True to compute error after the training
/// process completes, false otherwise. Default is true.
/// </param>
///
/// <returns>
/// The misclassification error rate of
/// the resulting support vector machine.
/// </returns>
///
public double Run(bool computeError)
{
// Create kernel function cache
cache = new KernelFunctionCache(kernel, inputs);
diagonal = new double[inputs.Length];
for (int i = 0; i < diagonal.Length; i++)
diagonal[i] = kernel.Function(inputs[i], inputs[i]) + gamma;
// 1. Solve to find nu and eta
double[] eta = conjugateGradient(outputs);
double[] nu = conjugateGradient(ones);
// 2. Compute s = Y' eta
double s = 0;
for (int i = 0; i < outputs.Length; i++)
s += outputs[i] * eta[i];
// 3. Find solution
double b = 0;
for (int i = 0; i < eta.Length; i++)
b += eta[i];
b /= s;
double[] alpha = new double[nu.Length];
for (int i = 0; i < alpha.Length; i++)
alpha[i] = (nu[i] - eta[i] * b) * outputs[i];
machine.SupportVectors = inputs;
machine.Weights = alpha;
machine.Threshold = b;
// Compute error if required.
return (computeError) ? ComputeError(inputs, outputs) : 0.0;
}
