public void KernelFunctionCacheConstructorTest8()
{
double[][] inputs =
{
new double[] { 0, 1 },
new double[] { 1, 0 },
new double[] { 1, 1 },
};
IKernel kernel = new Polynomial(2);
int cacheSize = inputs.Length;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(3, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
Assert.IsTrue(target.Enabled);
// upper half
for (int i = 0; i < inputs.Length - 1; i++)
{
for (int j = i + 1; j < inputs.Length - 1; j++)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.Throws <InvalidOperationException>(() => target.GetLeastRecentlyUsedList(), "The cache is not using a LRU list.");
}
public void KernelFunctionCacheConstructorTest7()
{
double[][] inputs =
{
new double[] { 0, 1 },
new double[] { 1, 0 },
new double[] { 1, 1 },
new double[] { },
};
IKernel kernel = new Polynomial(2);
int cacheSize = inputs.Length - 1;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(3, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
Assert.IsTrue(target.Enabled);
// upper half
for (int i = 0; i < inputs.Length - 1; i++)
{
for (int j = i + 1; j < inputs.Length - 1; j++)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
var lruList1 = target.GetLeastRecentlyUsedList();
Assert.AreEqual(2, target.Misses);
Assert.AreEqual(1, target.Hits);
Assert.AreEqual(0.66666, target.Usage, 1e-4);
// upper half, backwards
for (int i = inputs.Length - 2; i >= 0; i--)
{
for (int j = inputs.Length - 2; j >= i; j--)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(j, i);
Assert.AreEqual(expected, actual);
}
}
var lruList2 = target.GetLeastRecentlyUsedList();
Assert.IsTrue(lruList2.SequenceEqual(new[] { lruList1[1], lruList1[2], lruList1[0] }));
Assert.AreEqual(2, target.Misses);
Assert.AreEqual(4, target.Hits);
Assert.AreEqual(0.666666, target.Usage, 1e-5);
}
public void KernelFunctionCacheConstructorTest4()
{
IKernel kernel = new Linear();
int cacheSize = 100;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(inputs.Length, target.Size);
}
public void KernelFunctionCacheConstructorTest2()
{
IKernel kernel = new Linear(1);
int cacheSize = inputs.Length - 1;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(9, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
Assert.IsTrue(target.Enabled);
for (int i = 0; i < inputs.Length; i++)
{
double expected = i * i + 1;
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
int[] hits = { 0, 9, 18, 27, 36, 45, 54, 63, 72, 81 };
int[] miss = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(hits[i], target.Hits);
Assert.AreEqual(miss[i], target.Misses);
}
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(9, target.Misses);
Assert.AreEqual(171, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
public void KernelFunctionCacheConstructorTest2()
{
IKernel kernel = new Linear(1);
int cacheSize = inputs.Length - 1;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(9, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
Assert.IsTrue(target.Enabled);
for (int i = 0; i < inputs.Length; i++)
{
double expected = i * i + 1;
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
int[] hits = { 0, 1, 3, 6, 10, 15, 20, 25, 30, 35 };
int[] miss = { 9, 17, 24, 30, 35, 39, 43, 47, 51, 55 };
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(hits[i], target.Hits);
Assert.AreEqual(miss[i], target.Misses);
}
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(100, target.Misses);
Assert.AreEqual(80, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
public void KernelFunctionCacheConstructorTest3()
{
IKernel kernel = new Linear();
int cacheSize = 5;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(5, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
for (int i = 0; i < inputs.Length; i++)
{
double expected = i * i + 1;
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(9, target.Hits);
Assert.AreEqual(81, target.Misses);
var snapshot = target.GetDataCache();
foreach (var entry in snapshot)
{
double a = target.GetOrCompute(entry.Key.Item1, entry.Key.Item2);
double b = target.GetOrCompute(entry.Key.Item2, entry.Key.Item1);
Assert.AreEqual(a, b);
}
Assert.AreEqual(81, target.Misses);
Assert.AreEqual(29, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
public void KernelFunctionCacheConstructorTest6()
{
IKernel kernel = new Gaussian(0.6);
int cacheSize = inputs.Length;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(10, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
Assert.IsTrue(target.Enabled);
for (int i = 0; i < inputs.Length; i++)
{
double expected = kernel.Function(inputs[i], inputs[i]);
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = kernel.Function(inputs[i], inputs[j]);
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(0, target.Misses);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Usage);
}
/// <summary>
/// Runs the main body of the learning algorithm.
/// </summary>
protected override void InnerRun()
{
TInput[] inputs = Inputs;
int[] outputs = Outputs;
this.ones = Vector.Create(outputs.Length, 1);
// Create kernel function cache
diagonal = new double[inputs.Length];
cache = new KernelFunctionCache <TKernel, TInput>(Kernel, inputs);
for (int i = 0; i < diagonal.Length; i++)
{
diagonal[i] = Kernel.Function(inputs[i], inputs[i]) + 1.0 / C[i];
}
// 1. Solve to find nu and eta
double[] eta = conjugateGradient(outputs);
double[] nu = conjugateGradient(ones);
// 2. Compute s = Y' eta
double s = 0;
for (int i = 0; i < outputs.Length; i++)
{
s += outputs[i] * eta[i];
}
// 3. Find solution
double b = 0;
for (int i = 0; i < eta.Length; i++)
{
b += eta[i];
}
b /= s;
double[] alpha = new double[nu.Length];
for (int i = 0; i < alpha.Length; i++)
{
alpha[i] = (nu[i] - eta[i] * b) * outputs[i];
}
Model.SupportVectors = inputs;
Model.Weights = alpha;
Model.Threshold = b;
}
/// <summary>
/// Runs the LS-SVM algorithm.
/// </summary>
///
/// <param name="computeError">
/// True to compute error after the training
/// process completes, false otherwise. Default is true.
/// </param>
///
/// <returns>
/// The misclassification error rate of
/// the resulting support vector machine.
/// </returns>
///
public double Run(bool computeError)
{
// Create kernel function cache
cache = new KernelFunctionCache(kernel, inputs);
diagonal = new double[inputs.Length];
for (int i = 0; i < diagonal.Length; i++)
{
diagonal[i] = kernel.Function(inputs[i], inputs[i]) + gamma;
}
// 1. Solve to find nu and eta
double[] eta = conjugateGradient(outputs);
double[] nu = conjugateGradient(ones);
// 2. Compute s = Y' eta
double s = 0;
for (int i = 0; i < outputs.Length; i++)
{
s += outputs[i] * eta[i];
}
// 3. Find solution
double b = 0;
for (int i = 0; i < eta.Length; i++)
{
b += eta[i];
}
b /= s;
double[] alpha = new double[nu.Length];
for (int i = 0; i < alpha.Length; i++)
{
alpha[i] = (nu[i] - eta[i] * b) * outputs[i];
}
machine.SupportVectors = inputs;
machine.Weights = alpha;
machine.Threshold = b;
// Compute error if required.
return((computeError) ? ComputeError(inputs, outputs) : 0.0);
}
public void KernelFunctionCacheConstructorTest5()
{
IKernel kernel = new Linear(1);
int cacheSize = inputs.Length - 1;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(9, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
Assert.IsTrue(target.Enabled);
// upper half
for (int i = 0; i < inputs.Length; i++)
{
for (int j = i + 1; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(9, target.Misses);
Assert.AreEqual(36, target.Hits);
Assert.AreEqual(1.0, target.Usage);
// lower half
for (int i = 0; i < inputs.Length; i++)
{
for (int j = i + 1; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(j, i);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(9, target.Misses);
Assert.AreEqual(81, target.Hits);
Assert.AreEqual(1.0, target.Usage);
}
public void KernelFunctionCacheConstructorTest()
{
IKernel kernel = new Linear(1);
int cacheSize = 0;
KernelFunctionCache target = new KernelFunctionCache(kernel, inputs, cacheSize);
Assert.AreEqual(0, target.Size);
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Misses);
Assert.IsFalse(target.Enabled);
for (int i = 0; i < inputs.Length; i++)
{
double expected = i * i + 1;
double actual = target.GetOrCompute(i);
Assert.AreEqual(expected, actual);
}
Assert.AreEqual(0, target.Hits);
for (int i = 0; i < inputs.Length; i++)
{
for (int j = 0; j < inputs.Length; j++)
{
double expected = i * j + 1;
double actual = target.GetOrCompute(i, j);
Assert.AreEqual(expected, actual);
}
}
Assert.AreEqual(0, target.Hits);
Assert.AreEqual(0, target.Usage);
}
/// <summary>
/// Runs the SMO algorithm.
/// </summary>
///
/// <param name="computeError">
/// True to compute error after the training
/// process completes, false otherwise. Default is true.
/// </param>
/// <param name="token">
/// A <see cref="CancellationToken"/> which can be used
/// to request the cancellation of the learning algorithm
/// when it is being run in another thread.
/// </param>
///
/// <returns>
/// The misclassification error rate of
/// the resulting support vector machine.
/// </returns>
///
public double Run(bool computeError, CancellationToken token)
{
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Initialize variables
int samples = inputs.Length;
int dimension = inputs[0].Length;
// Initialization heuristics
if (useComplexityHeuristic)
{
c = EstimateComplexity(kernel, inputs);
}
int[] positives = outputs.Find(x => x == +1);
int[] negatives = outputs.Find(x => x == -1);
// If all examples are positive or negative, terminate
// learning early by directly setting the threshold.
if (positives.Length == 0)
{
machine.SupportVectors = new double[0][];
machine.Weights = new double[0];
machine.Threshold = -1;
return(0);
}
if (negatives.Length == 0)
{
machine.SupportVectors = new double[0][];
machine.Weights = new double[0];
machine.Threshold = +1;
return(0);
}
if (useClassLabelProportion)
{
WeightRatio = positives.Length / (double)negatives.Length;
}
// Lagrange multipliers
Array.Clear(alpha, 0, alpha.Length);
if (isLinear) // Hyperplane weights
{
Array.Clear(weights, 0, weights.Length);
}
// Error cache
Array.Clear(errors, 0, errors.Length);
// Kernel evaluations cache
this.kernelCache = new KernelFunctionCache(kernel, inputs, cacheSize);
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = positives[0];
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = negatives[0];
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Prepare indices sets
activeExamples.Clear();
nonBoundExamples.Clear();
atBoundsExamples.Clear();
// Balance classes
bool balanced = positiveWeight == 1 && negativeWeight == 1;
positiveCost = c * positiveWeight;
negativeCost = c * negativeWeight;
// Algorithm:
int numChanged = 0;
int wholeSetChecks = 0;
bool examineAll = true;
bool diverged = false;
bool shouldStop = false;
while ((numChanged > 0 || examineAll) && !shouldStop)
{
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
{
if (examineExample(i))
{
numChanged++;
}
}
wholeSetChecks++;
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
if (balanced) // Assume balanced data
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c)
{
if (examineExample(i))
{
numChanged++;
}
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else // Use different weights for classes
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0)
{
if (outputs[i] == +1)
{
if (alpha[i] == positiveCost)
{
continue;
}
}
else // outputs[i] == -1
{
if (alpha[i] == negativeCost)
{
continue;
}
}
if (examineExample(i))
{
numChanged++;
}
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
}
else // strategy == Strategy.WorstPair
{
int attempts = 0;
do
{
attempts++;
if (!takeStep(i_upper, i_lower))
{
break;
}
if (attempts > samples * maxChecks)
{
break;
}
}while ((b_upper <= b_lower - 2.0 * tolerance));
numChanged = 0;
}
}
if (examineAll)
{
examineAll = false;
}
else if (numChanged == 0)
{
examineAll = true;
}
if (wholeSetChecks > maxChecks)
{
shouldStop = diverged = true;
}
if (token.IsCancellationRequested)
{
shouldStop = true;
}
}
// Store information about bounded examples
if (balanced)
{
// Assume equal weights for classes
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] == c)
{
atBoundsExamples.Add(i);
}
}
}
else
{
// Use different weights for classes
for (int i = 0; i < alpha.Length; i++)
{
if (outputs[i] == +1)
{
if (alpha[i] == positiveCost)
{
atBoundsExamples.Add(i);
}
}
else // outputs[i] == -1
{
if (alpha[i] == negativeCost)
{
atBoundsExamples.Add(i);
}
}
}
}
if (isCompact)
{
// Store the hyperplane directly
machine.SupportVectors = null;
machine.Weights = weights;
machine.Threshold = -(b_lower + b_upper) / 2.0;
}
else
{
// Store Support Vectors in the SV Machine. Only vectors which have Lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
int[] idx = new int[activeCount];
activeExamples.CopyTo(idx);
machine.SupportVectors = new double[activeCount][];
machine.Weights = new double[activeCount];
for (int i = 0; i < idx.Length; i++)
{
int j = idx[i];
machine.SupportVectors[i] = inputs[j];
machine.Weights[i] = alpha[j] * outputs[j];
}
machine.Threshold = -(b_lower + b_upper) / 2;
}
// Clear function cache
this.kernelCache.Clear();
this.kernelCache = null;
if (diverged)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
// Compute error if required.
return((computeError) ? ComputeError(inputs, outputs) : 0.0);
}
/// <summary>
/// Runs the learning algorithm.
/// </summary>
///
protected override void InnerRun()
{
// Initialize variables
int samples = Inputs.Length;
double[] c = C;
TInput[] x = Inputs;
int[] y = Outputs;
// Lagrange multipliers
this.alpha = new double[samples];
// Prepare indices sets
activeExamples = new HashSet <int>();
nonBoundExamples = new HashSet <int>();
atBoundsExamples = new HashSet <int>();
// Kernel cache
if (this.cacheSize == -1)
{
this.cacheSize = samples;
}
using (this.kernelCache = new KernelFunctionCache <TKernel, TInput>(Kernel, Inputs, cacheSize))
{
bool diverged = false;
if (Strategy == SelectionStrategy.SecondOrder)
{
double[] minusOnes = Vector.Create(Inputs.Length, -1.0);
Func <int, int[], int, double[], double[]> Q;
if (kernelCache.Enabled)
{
Q = (int i, int[] indices, int length, double[] row) =>
{
for (int j = 0; j < length; j++)
{
row[j] = y[i] * y[indices[j]] * kernelCache.GetOrCompute(i, indices[j]);
}
return(row);
};
}
else
{
Q = (int i, int[] indices, int length, double[] row) =>
{
for (int j = 0; j < length; j++)
{
row[j] = y[i] * y[indices[j]] * Kernel.Function(x[i], x[indices[j]]);
}
return(row);
};
}
var s = new FanChenLinQuadraticOptimization(alpha.Length, Q, minusOnes, y)
{
Tolerance = tolerance,
Shrinking = this.shrinking,
Solution = alpha,
Token = Token,
UpperBounds = c
};
diverged = !s.Minimize();
// Store information about active examples
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] > 0)
{
activeExamples.Add(i);
}
}
b_lower = b_upper = s.Rho;
}
else // Strategy is Strategy.WorstPair or Strategy.Sequential
{
if (shrinking)
{
throw new InvalidOperationException("Shrinking heuristic can only be used if Strategy is set to SelectionStrategy.SecondOrder.");
}
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Error cache
this.errors = new double[samples];
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = y.First(y_i => y_i > 0);
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = y.First(y_i => y_i < 0);
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Algorithm:
int numChanged = 0;
int wholeSetChecks = 0;
bool examineAll = true;
bool shouldStop = false;
while ((numChanged > 0 || examineAll) && !shouldStop)
{
if (Token.IsCancellationRequested)
{
break;
}
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
{
if (examineExample(i))
{
numChanged++;
}
}
wholeSetChecks++;
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c[i])
{
if (examineExample(i))
{
numChanged++;
}
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else if (strategy == SelectionStrategy.WorstPair)
{
int attempts = 0;
do
{
attempts++;
if (!takeStep(i_upper, i_lower))
{
break;
}
if (attempts > samples * maxChecks)
{
break;
}
}while ((b_upper <= b_lower - 2.0 * tolerance));
numChanged = 0;
}
else
{
throw new InvalidOperationException("Unknown strategy");
}
}
if (examineAll)
{
examineAll = false;
}
else if (numChanged == 0)
{
examineAll = true;
}
if (wholeSetChecks > maxChecks)
{
shouldStop = diverged = true;
}
if (Token.IsCancellationRequested)
{
shouldStop = true;
}
}
}
// Store information about bounded examples
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] == c[i])
{
atBoundsExamples.Add(i);
}
}
// Store Support Vectors in the SV Machine. Only vectors which have Lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
Model.SupportVectors = new TInput[activeCount];
Model.Weights = new double[activeCount];
int index = 0;
foreach (var j in activeExamples)
{
Model.SupportVectors[index] = x[j];
Model.Weights[index] = alpha[j] * y[j];
index++;
}
Model.Threshold = -(b_lower + b_upper) / 2;
if (isCompact)
{
Model.Compress();
}
if (diverged)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
}
}
//---------------------------------------------
/// <summary>
/// Runs the SMO algorithm.
/// </summary>
///
/// <param name="computeError">
/// True to compute error after the training
/// process completes, false otherwise. Default is true.
/// </param>
///
/// <returns>
/// The misclassification error rate of
/// the resulting support vector machine.
/// </returns>
///
public double Run(bool computeError)
{
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Initialize variables
int samples = inputs.Length;
int dimension = inputs[0].Length;
if (useComplexityHeuristic)
{
c = EstimateComplexity(kernel, inputs);
}
// Lagrange multipliers
Array.Clear(alpha, 0, alpha.Length);
if (isLinear) // Hyperplane weights
{
Array.Clear(weights, 0, weights.Length);
}
// Error cache
Array.Clear(errors, 0, errors.Length);
// Kernel evaluations cache
this.kernelCache = new KernelFunctionCache(kernel, inputs, cacheSize);
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = outputs.Find(x => x == +1)[0];
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = outputs.Find(x => x == -1)[0];
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Prepare indice sets
activeExamples = new HashSet <int>();
nonBoundExamples = new HashSet <int>();
atBoundsExamples = new HashSet <int>();
// Algorithm:
int numChanged = 0;
bool examineAll = true;
while (numChanged > 0 || examineAll)
{
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
{
if (examineExample(i))
{
numChanged++;
}
}
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c)
{
if (examineExample(i))
{
numChanged++;
}
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else // strategy == Strategy.WorstPair
{
bool success;
do
{
success = takeStep(i_upper, i_lower);
}while ((b_upper <= b_lower - 2.0 * tolerance) && success);
numChanged = 0;
}
}
if (examineAll)
{
examineAll = false;
}
else if (numChanged == 0)
{
examineAll = true;
}
}
// Store information about bounded examples
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] == c)
{
atBoundsExamples.Add(i);
}
}
if (isCompact)
{
// Store the hyperplane directly
machine.SupportVectors = null;
machine.Weights = weights;
machine.Threshold = -(b_lower + b_upper) / 2.0;
}
else
{
// Store Support Vectors in the SV Machine. Only vectors which have lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
int[] idx = new int[activeCount];
activeExamples.CopyTo(idx);
machine.SupportVectors = new double[activeCount][];
machine.Weights = new double[activeCount];
for (int i = 0; i < idx.Length; i++)
{
int j = idx[i];
machine.SupportVectors[i] = inputs[j];
machine.Weights[i] = alpha[j] * outputs[j];
}
machine.Threshold = -(b_lower + b_upper) / 2;
}
// Clear function cache
this.kernelCache.Clear();
// Compute error if required.
return((computeError) ? ComputeError(inputs, outputs) : 0.0);
}
/// <summary>
/// Runs the learning algorithm.
/// </summary>
///
protected override void InnerRun()
{
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Initialize variables
int samples = Inputs.Length;
double[] c = C;
TInput[] x = Inputs;
int[] y = Outputs;
// Lagrange multipliers
this.alpha = new double[samples];
// Error cache
this.errors = new double[samples];
// Kernel cache
if (this.cacheSize == -1)
{
this.cacheSize = samples;
}
// Lagrange multipliers
Array.Clear(alpha, 0, alpha.Length);
// Error cache
Array.Clear(errors, 0, errors.Length);
// Kernel evaluations cache
this.kernelCache = new KernelFunctionCache <TKernel, TInput>(Kernel, Inputs, cacheSize);
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = y.First(y_i => y_i > 0);
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = y.First(y_i => y_i < 0);
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Prepare indices sets
activeExamples = new HashSet <int>();
nonBoundExamples = new HashSet <int>();
atBoundsExamples = new HashSet <int>();
// Algorithm:
int numChanged = 0;
int wholeSetChecks = 0;
bool examineAll = true;
bool diverged = false;
bool shouldStop = false;
while ((numChanged > 0 || examineAll) && !shouldStop)
{
if (Token.IsCancellationRequested)
{
break;
}
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
{
if (examineExample(i))
{
numChanged++;
}
}
wholeSetChecks++;
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c[i])
{
if (examineExample(i))
{
numChanged++;
}
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else // strategy == Strategy.WorstPair
{
int attempts = 0;
do
{
attempts++;
if (!takeStep(i_upper, i_lower))
{
break;
}
if (attempts > samples * maxChecks)
{
break;
}
}while ((b_upper <= b_lower - 2.0 * tolerance));
numChanged = 0;
}
}
if (examineAll)
{
examineAll = false;
}
else if (numChanged == 0)
{
examineAll = true;
}
if (wholeSetChecks > maxChecks)
{
shouldStop = diverged = true;
}
if (Token.IsCancellationRequested)
{
shouldStop = true;
}
}
// Store information about bounded examples
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] == c[i])
{
atBoundsExamples.Add(i);
}
}
// Store Support Vectors in the SV Machine. Only vectors which have Lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
Model.SupportVectors = new TInput[activeCount];
Model.Weights = new double[activeCount];
int index = 0;
foreach (var j in activeExamples)
{
Model.SupportVectors[index] = x[j];
Model.Weights[index] = alpha[j] * y[j];
index++;
}
Model.Threshold = -(b_lower + b_upper) / 2;
if (isCompact)
{
Model.Compress();
}
// Clear function cache
this.kernelCache.Clear();
this.kernelCache = null;
if (diverged)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
}
