/// <summary>
/// Runs the learning algorithm.
/// </summary>
///
/// <param name="computeError">True to compute error after the training
/// process completes, false otherwise.</param>
/// <returns>
/// The misclassification error rate of the resulting support
/// vector machine if <paramref name="computeError" /> is true,
/// returns zero otherwise.
/// </returns>
///
public double Run(bool computeError)
{
int l = inputs.Length;
int n = (int)(nu * l); // # of alpha's at upper bound
for (int i = 0; i < n; i++)
{
alpha[i] = 1;
}
if (n < inputs.Length)
{
alpha[n] = nu * l - n;
}
for (int i = n + 1; i < l; i++)
{
alpha[i] = 0;
}
var s = new FanChenLinQuadraticOptimization(alpha.Length, Q, zeros, ones)
{
Tolerance = eps,
Shrinking = true,
Solution = alpha
};
bool success = s.Minimize();
int sv = 0;
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] > 0)
{
sv++;
}
}
machine.SupportVectors = new double[sv][];
machine.Weights = new double[sv];
for (int i = 0, j = 0; i < alpha.Length; i++)
{
if (alpha[i] > 0)
{
machine.SupportVectors[j] = inputs[i];
machine.Weights[j] = alpha[i];
j++;
}
}
machine.Threshold = s.Rho;
if (success == false)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
if (computeError)
{
return(ComputeError(inputs));
}
return(0.0);
}
/// <summary>
/// Learns a model that can map the given inputs to the desired outputs.
/// </summary>
/// <param name="x">The model inputs.</param>
/// <param name="weights">The weight of importance for each input sample.</param>
/// <returns>
/// A model that has learned how to produce suitable outputs
/// given the input data <paramref name="x" />.
/// </returns>
public TModel Learn(TInput[] x, double[] weights = null)
{
bool initialized = false;
SupportVectorLearningHelper.CheckArgs(x);
if (kernel == null)
{
kernel = SupportVectorLearningHelper.CreateKernel <TKernel, TInput>(x);
initialized = true;
}
if (!initialized && useKernelEstimation)
{
kernel = SupportVectorLearningHelper.EstimateKernel(kernel, x);
}
if (Model == null)
{
Model = Create(SupportVectorLearningHelper.GetNumberOfInputs(kernel, x), kernel);
}
Model.Kernel = kernel;
try
{
this.inputs = x;
double[] zeros = new double[inputs.Length];
int[] ones = Vector.Ones <int>(inputs.Length);
this.alpha = Vector.Ones <double>(inputs.Length);
int l = inputs.Length;
int n = (int)(nu * l); // # of alpha's at upper bound
for (int i = 0; i < n; i++)
{
alpha[i] = 1;
}
if (n < inputs.Length)
{
alpha[n] = nu * l - n;
}
for (int i = n + 1; i < l; i++)
{
alpha[i] = 0;
}
Func <int, int[], int, double[], double[]> Q = (int i, int[] indices, int length, double[] row) =>
{
for (int j = 0; j < length; j++)
{
row[j] = Kernel.Function(x[i], x[indices[j]]);
}
return(row);
};
var s = new FanChenLinQuadraticOptimization(alpha.Length, Q, zeros, ones)
{
Tolerance = eps,
Shrinking = this.shrinking,
Solution = alpha,
Token = Token
};
bool success = s.Minimize();
int sv = 0;
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] > 0)
{
sv++;
}
}
Model.SupportVectors = new TInput[sv];
Model.Weights = new double[sv];
for (int i = 0, j = 0; i < alpha.Length; i++)
{
if (alpha[i] > 0)
{
Model.SupportVectors[j] = inputs[i];
Model.Weights[j] = alpha[i];
j++;
}
}
Model.Threshold = s.Rho;
if (success == false)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
}
finally
{
if (machine != null)
{
// TODO: This block is only necessary to offer compatibility
// to code written using previous versions of the framework,
// and should be removed after a few releases.
machine.SupportVectors = Model.SupportVectors;
machine.Weights = Model.Weights;
machine.Threshold = Model.Threshold;
machine.Kernel = Model.Kernel;
machine.IsProbabilistic = Model.IsProbabilistic;
}
}
return(Model);
}
/// <summary>
/// Runs the learning algorithm.
/// </summary>
///
protected override void InnerRun()
{
// Initialize variables
int samples = Inputs.Length;
double[] c = C;
TInput[] x = Inputs;
int[] y = Outputs;
// Lagrange multipliers
this.alpha = new double[samples];
// Prepare indices sets
activeExamples = new HashSet <int>();
nonBoundExamples = new HashSet <int>();
atBoundsExamples = new HashSet <int>();
// Kernel cache
if (this.cacheSize == -1)
{
this.cacheSize = samples;
}
using (this.kernelCache = new KernelFunctionCache <TKernel, TInput>(Kernel, Inputs, cacheSize))
{
bool diverged = false;
if (Strategy == SelectionStrategy.SecondOrder)
{
double[] minusOnes = Vector.Create(Inputs.Length, -1.0);
Func <int, int[], int, double[], double[]> Q;
if (kernelCache.Enabled)
{
Q = (int i, int[] indices, int length, double[] row) =>
{
for (int j = 0; j < length; j++)
{
row[j] = y[i] * y[indices[j]] * kernelCache.GetOrCompute(i, indices[j]);
}
return(row);
};
}
else
{
Q = (int i, int[] indices, int length, double[] row) =>
{
for (int j = 0; j < length; j++)
{
row[j] = y[i] * y[indices[j]] * Kernel.Function(x[i], x[indices[j]]);
}
return(row);
};
}
var s = new FanChenLinQuadraticOptimization(alpha.Length, Q, minusOnes, y)
{
Tolerance = tolerance,
Shrinking = this.shrinking,
Solution = alpha,
Token = Token,
UpperBounds = c
};
diverged = !s.Minimize();
// Store information about active examples
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] > 0)
{
activeExamples.Add(i);
}
}
b_lower = b_upper = s.Rho;
}
else // Strategy is Strategy.WorstPair or Strategy.Sequential
{
if (shrinking)
{
throw new InvalidOperationException("Shrinking heuristic can only be used if Strategy is set to SelectionStrategy.SecondOrder.");
}
// The SMO algorithm chooses to solve the smallest possible optimization problem
// at every step. At every step, SMO chooses two Lagrange multipliers to jointly
// optimize, finds the optimal values for these multipliers, and updates the SVM
// to reflect the new optimal values.
//
// Reference: http://research.microsoft.com/en-us/um/people/jplatt/smoTR.pdf
// The algorithm has been updated to implement the improvements suggested
// by Keerthi et al. The code has been based on the pseudo-code available
// on the author's technical report.
//
// Reference: http://www.cs.iastate.edu/~honavar/keerthi-svm.pdf
// Error cache
this.errors = new double[samples];
// [Keerthi] Initialize b_up to -1 and
// i_up to any one index of class 1:
this.b_upper = -1;
this.i_upper = y.First(y_i => y_i > 0);
// [Keerthi] Initialize b_low to +1 and
// i_low to any one index of class 2:
this.b_lower = +1;
this.i_lower = y.First(y_i => y_i < 0);
// [Keerthi] Set error cache for i_low and i_up:
this.errors[i_lower] = +1;
this.errors[i_upper] = -1;
// Algorithm:
int numChanged = 0;
int wholeSetChecks = 0;
bool examineAll = true;
bool shouldStop = false;
while ((numChanged > 0 || examineAll) && !shouldStop)
{
if (Token.IsCancellationRequested)
{
break;
}
numChanged = 0;
if (examineAll)
{
// loop I over all training examples
for (int i = 0; i < samples; i++)
{
if (examineExample(i))
{
numChanged++;
}
}
wholeSetChecks++;
}
else
{
if (strategy == SelectionStrategy.Sequential)
{
// loop I over examples not at bounds
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0 && alpha[i] != c[i])
{
if (examineExample(i))
{
numChanged++;
}
if (b_upper > b_lower - 2.0 * tolerance)
{
numChanged = 0; break;
}
}
}
}
else if (strategy == SelectionStrategy.WorstPair)
{
int attempts = 0;
do
{
attempts++;
if (!takeStep(i_upper, i_lower))
{
break;
}
if (attempts > samples * maxChecks)
{
break;
}
}while ((b_upper <= b_lower - 2.0 * tolerance));
numChanged = 0;
}
else
{
throw new InvalidOperationException("Unknown strategy");
}
}
if (examineAll)
{
examineAll = false;
}
else if (numChanged == 0)
{
examineAll = true;
}
if (wholeSetChecks > maxChecks)
{
shouldStop = diverged = true;
}
if (Token.IsCancellationRequested)
{
shouldStop = true;
}
}
}
// Store information about bounded examples
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] == c[i])
{
atBoundsExamples.Add(i);
}
}
// Store Support Vectors in the SV Machine. Only vectors which have Lagrange multipliers
// greater than zero will be stored as only those are actually required during evaluation.
int activeCount = activeExamples.Count;
Model.SupportVectors = new TInput[activeCount];
Model.Weights = new double[activeCount];
int index = 0;
foreach (var j in activeExamples)
{
Model.SupportVectors[index] = x[j];
Model.Weights[index] = alpha[j] * y[j];
index++;
}
Model.Threshold = -(b_lower + b_upper) / 2;
if (isCompact)
{
Model.Compress();
}
if (diverged)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
}
}
/// <summary>
/// Runs the learning algorithm.
/// </summary>
///
protected override void InnerRun()
{
TInput[] x = Inputs;
double[] y = Outputs;
int l = Inputs.Length;
double[] alpha2 = new double[2 * l];
double[] linear_term = new double[2 * l];
int[] signs = new int[2 * l];
double[] c = new double[2 * l];
int[] sign = new int[2 * l];
int[] index = new int[2 * l];
for (int i = 0; i < l; i++)
{
linear_term[i] = rho - y[i];
signs[i] = +1;
c[i] = C[i];
index[i] = i;
linear_term[i + l] = rho + y[i];
signs[i + l] = -1;
c[i + l] = C[i];
index[i + l] = i;
}
Func <int, int[], int, double[], double[]> Q = (int i, int[] indices, int length, double[] row) =>
{
for (int j = 0; j < length; j++)
{
int k = indices[j];
row[j] = signs[i] * signs[k] * Kernel.Function(x[index[i]], x[index[k]]);
}
return(row);
};
var s = new FanChenLinQuadraticOptimization(2 * l, Q, linear_term, signs)
{
Tolerance = eps,
Shrinking = this.shrinking,
Solution = alpha2,
Token = Token,
UpperBounds = c,
};
bool success = s.Minimize();
this.alpha = new double[Inputs.Length];
for (int i = 0; i < alpha.Length; i++)
{
alpha[i] = alpha2[i] - alpha2[i + l];
}
int sv = 0;
for (int i = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0)
{
sv++;
}
}
Model.SupportVectors = new TInput[sv];
Model.Weights = new double[sv];
for (int i = 0, j = 0; i < alpha.Length; i++)
{
if (alpha[i] != 0)
{
Model.SupportVectors[j] = Inputs[i];
Model.Weights[j] = alpha[i];
j++;
}
}
Model.Threshold = -s.Rho;
if (success == false)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
}
/// <summary>
/// Runs the learning algorithm.
/// </summary>
///
/// <param name="computeError">True to compute error after the training
/// process completes, false otherwise.</param>
/// <returns>
/// The misclassification error rate of the resulting support
/// vector machine if <paramref name="computeError" /> is true,
/// returns zero otherwise.
/// </returns>
///
public double Run(bool computeError)
{
int l = inputs.Length;
int n = (int)(nu * l); // # of alpha's at upper bound
for (int i = 0; i < n; i++)
alpha[i] = 1;
if (n < inputs.Length)
alpha[n] = nu * l - n;
for (int i = n + 1; i < l; i++)
alpha[i] = 0;
var s = new FanChenLinQuadraticOptimization(alpha.Length, Q, zeros, ones)
{
Tolerance = eps,
Shrinking = true,
Solution = alpha
};
bool success = s.Minimize();
int sv = 0;
for (int i = 0; i < alpha.Length; i++)
if (alpha[i] > 0) sv++;
machine.SupportVectors = new double[sv][];
machine.Weights = new double[sv];
for (int i = 0, j = 0; i < alpha.Length; i++)
{
if (alpha[i] > 0)
{
machine.SupportVectors[j] = inputs[i];
machine.Weights[j] = alpha[i];
j++;
}
}
machine.Threshold = s.Rho;
if (success == false)
{
throw new ConvergenceException("Convergence could not be attained. " +
"Please reduce the cost of misclassification errors by reducing " +
"the complexity parameter C or try a different kernel function.");
}
if (computeError)
return ComputeError(inputs);
return 0.0;
}
