/// <summary>
/// Attempts to retrieve the value of the kernel function
/// from the diagonal of the kernel matrix. If the value
/// is not available, it is immediatelly computed and inserted
/// in the cache.
/// </summary>
///
/// <param name="i">Index of the point to compute.</param>
///
/// <remarks>The result of the kernel function k(p[i], p[i]).</remarks>
///
public double GetOrCompute(int i)
{
if (!Enabled)
{
return(kernel.Function(inputs[i], inputs[i]));
}
hits++; // Diagonal elements are always available
return(diagonal[i]);
}
/// <summary>
/// Constructs a new <see cref="KernelFunctionCache"/>.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The inputs values.</param>
/// <param name="cacheSize">
/// The size for the cache, measured in number of
/// elements from the <paramref name="inputs"/> set.
/// Default is to use all elements.</param>
///
public KernelFunctionCache(IKernel kernel, double[][] inputs, int cacheSize)
{
if (cacheSize < 0)
{
throw new ArgumentOutOfRangeException("cacheSize",
"The cache size must be non-negative.");
}
this.kernel = kernel;
this.inputs = inputs;
this.size = cacheSize;
if (cacheSize > inputs.Length)
{
this.size = inputs.Length;
}
if (cacheSize > inputs.Length)
{
// Create whole cache.
matrix = new double[inputs.Length][];
for (int i = 0; i < inputs.Length; i++)
{
double[] row = matrix[i] = new double[inputs.Length - 1];
for (int j = 0; j < row.Length - 1; j++)
{
matrix[i][j] = kernel.Function(inputs[i], inputs[j]);
}
}
}
else if (cacheSize > 0)
{
this.capacity = (size * (size - 1)) / 2;
int collectionCapacity = (int)(1.1f * capacity);
// Create lookup tables
this.lruIndices = new LinkedList <int>();
this.lruIndicesLookupTable =
new Dictionary <int, LinkedListNode <int> >(collectionCapacity);
// Create cache for off-diagonal elements
this.data = new Dictionary <int, double>(collectionCapacity);
}
// Create cache for diagonal elements
this.diagonal = new double[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
this.diagonal[i] = kernel.Function(inputs[i], inputs[i]);
}
}
/// <summary>
/// Computes the given input to produce the corresponding output.
/// </summary>
///
/// <remarks>
/// For a binary decision problem, the decision for the negative
/// or positive class is typically computed by taking the sign of
/// the machine's output.
/// </remarks>
///
/// <param name="inputs">An input vector.</param>
/// <param name="output">The output of the machine. If this is a
/// <see cref="SupportVectorMachine.IsProbabilistic">probabilistic
/// </see> machine, the output is the probability of the positive
/// class. If this is a standard machine, the output is the distance
/// to the decision hyperplane in feature space.</param>
///
/// <returns>The decision label for the given input.</returns>
///
public override int Compute(double[] inputs, out double output)
{
output = Threshold;
if (IsCompact)
{
for (int i = 0; i < Weights.Length; i++)
{
output += Weights[i] * inputs[i];
}
}
else
{
for (int i = 0; i < SupportVectors.Length; i++)
{
output += Weights[i] * kernel.Function(SupportVectors[i], inputs);
}
}
if (IsProbabilistic)
{
output = Link.Inverse(output);
return(output >= 0.5 ? +1 : -1);
}
return(output >= 0 ? +1 : -1);
}
/// <summary>
/// Computes the given input to produce the corresponding output.
/// </summary>
///
/// <remarks>
/// For a binary decision problem, the decision for the negative
/// or positive class is typically computed by taking the sign of
/// the machine's output.
/// </remarks>
///
/// <param name="inputs">An input vector.</param>
/// <returns>The output for the given input.</returns>
///
public override double Compute(double[] inputs)
{
double s = Threshold;
for (int i = 0; i < SupportVectors.Length; i++)
{
s += Weights[i] * kernel.Function(SupportVectors[i], inputs);
}
return(s);
}
private double compute(double[] point)
{
double sum = machine.Threshold;
for (int i = 0; i < alpha.Length; i++)
{
sum += alpha[i] * kernel.Function(supportVectors[i], point);
}
return(sum);
}
/// <summary>
/// Estimates the <see cref="Complexity">complexity parameter C</see>
/// for a given kernel and a given data set.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The input samples.</param>
///
/// <returns>A suitable value for C.</returns>
///
public static double EstimateComplexity(IKernel kernel, double[][] inputs)
{
// Compute initial value for C as the number of examples
// divided by the trace of the input sample kernel matrix.
double sum = 0.0;
for (int i = 0; i < inputs.Length; i++)
{
sum += kernel.Function(inputs[i], inputs[i]);
}
return(inputs.Length / sum);
}
/// <summary>
/// Estimates the <see cref="Complexity">complexity parameter C</see>
/// for a given kernel and an unbalanced data set by summing every element
/// on the diagonal of the kernel matrix and using an heuristic based on it.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The input samples.</param>
/// <param name="outputs">The output samples.</param>
///
/// <returns>A suitable value for positive C and negative C, respectively.</returns>
///
public static Tuple <double, double> EstimateComplexity(IKernel kernel, double[][] inputs, int[] outputs)
{
// Compute initial value for C as the number of examples
// divided by the trace of the input sample kernel matrix.
double negativeSum = 0.0;
double positiveSum = 0.0;
int negativeCount = 0;
int positiveCount = 0;
for (int i = 0; i < inputs.Length; i++)
{
if (outputs[i] == -1)
{
negativeSum += kernel.Function(inputs[i], inputs[i]);
negativeCount++;
}
else // outputs[i] == +1
{
positiveSum += kernel.Function(inputs[i], inputs[i]);
positiveCount++;
}
if (Double.IsNaN(positiveSum) || Double.IsNaN(negativeSum))
{
throw new OverflowException();
}
}
return(Tuple.Create
(
positiveCount / positiveSum,
negativeCount / negativeSum
));
}
/// <summary>
/// Runs the LS-SVM algorithm.
/// </summary>
///
/// <param name="computeError">
/// True to compute error after the training
/// process completes, false otherwise. Default is true.
/// </param>
///
/// <returns>
/// The misclassification error rate of
/// the resulting support vector machine.
/// </returns>
///
public double Run(bool computeError)
{
// Create kernel function cache
cache = new KernelFunctionCache(kernel, inputs);
diagonal = new double[inputs.Length];
for (int i = 0; i < diagonal.Length; i++)
{
diagonal[i] = kernel.Function(inputs[i], inputs[i]) + gamma;
}
// 1. Solve to find nu and eta
double[] eta = conjugateGradient(outputs);
double[] nu = conjugateGradient(ones);
// 2. Compute s = Y' eta
double s = 0;
for (int i = 0; i < outputs.Length; i++)
{
s += outputs[i] * eta[i];
}
// 3. Find solution
double b = 0;
for (int i = 0; i < eta.Length; i++)
{
b += eta[i];
}
b /= s;
double[] alpha = new double[nu.Length];
for (int i = 0; i < alpha.Length; i++)
{
alpha[i] = (nu[i] - eta[i] * b) * outputs[i];
}
machine.SupportVectors = inputs;
machine.Weights = alpha;
machine.Threshold = b;
// Compute error if required.
return((computeError) ? ComputeError(inputs, outputs) : 0.0);
}
/// <summary>
/// Computes the SVM output for a given point.
/// </summary>
///
private double compute(double[] point)
{
double sum = -(b_lower + b_upper) / 2;
if (isLinear)
{
for (int i = 0; i < weights.Length; i++)
{
sum += weights[i] * point[i];
}
}
else
{
foreach (int i in activeExamples)
{
sum += alpha[i] * outputs[i] * kernel.Function(inputs[i], point);
}
}
return(sum);
}
/// <summary>
/// Constructs a new <see cref="KernelFunctionCache"/>.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The inputs values.</param>
/// <param name="cacheSize">The size for the cache.</param>
///
public KernelFunctionCache(IKernel kernel, double[][] inputs, int cacheSize)
{
this.kernel = kernel;
this.inputs = inputs;
this.size = inputs.Length;
if (cacheSize < 0)
{
Enabled = false;
return;
}
else
{
Enabled = true;
}
if (cacheSize == 0)
{
cacheSize = inputs.Length;
}
this.cacheSize = cacheSize;
this.capacity = (size * size) / 2 + 1;
// Create lookup tables
this.lruIndices = new LinkedList <int>();
this.lruIndicesLookupTable = new Dictionary <int, LinkedListNode <int> >(capacity);
// Create cache for off-diagonal elements
this.data = new Dictionary <int, double>(capacity);
// Create cache for diagonal elements
this.diagonal = new double[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
this.diagonal[i] = kernel.Function(inputs[i], inputs[i]);
}
}
/// <summary>
/// Estimates the <see cref="Complexity">complexity parameter C</see>
/// for a given kernel and a given data set.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The input samples.</param>
///
/// <returns>A suitable value for C.</returns>
///
public static double EstimateComplexity(IKernel kernel, double[][] inputs)
{
// Compute initial value for C as the number of examples
// divided by the trace of the input sample kernel matrix.
double sum = 0.0;
for (int i = 0; i < inputs.Length; i++)
{
sum += kernel.Function(inputs[i], inputs[i]);
if (Double.IsNaN(sum))
throw new OverflowException();
}
return inputs.Length / sum;
}
/// <summary>
/// Constructs a new <see cref="KernelFunctionCache"/>.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The inputs values.</param>
/// <param name="cacheSize">
/// The size for the cache, measured in number of
/// elements from the <paramref name="inputs"/> set.
/// Default is to use all elements.</param>
///
public KernelFunctionCache(IKernel kernel, double[][] inputs, int cacheSize)
{
if (cacheSize < 0)
throw new ArgumentOutOfRangeException("cacheSize",
"The cache size must be non-negative.");
this.kernel = kernel;
this.inputs = inputs;
this.size = cacheSize;
if (cacheSize > inputs.Length)
this.size = inputs.Length;
if (cacheSize > inputs.Length)
{
// Create whole cache.
matrix = new double[inputs.Length][];
for (int i = 0; i < inputs.Length; i++)
{
double[] row = matrix[i] = new double[inputs.Length - 1];
for (int j = 0; j < row.Length - 1; j++)
matrix[i][j] = kernel.Function(inputs[i], inputs[j]);
}
}
else if (cacheSize > 0)
{
this.capacity = (size * (size - 1)) / 2;
int collectionCapacity = (int)(1.1f * capacity);
// Create lookup tables
this.lruIndices = new LinkedList<int>();
this.lruIndicesLookupTable =
new Dictionary<int, LinkedListNode<int>>(collectionCapacity);
// Create cache for off-diagonal elements
this.data = new Dictionary<int, double>(collectionCapacity);
}
// Create cache for diagonal elements
this.diagonal = new double[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
this.diagonal[i] = kernel.Function(inputs[i], inputs[i]);
}
/// <summary>
/// Computes the kernel distance for a kernel function even if it doesn't
/// implement the <see cref="Accord.Math.Distances.IDistance"/> interface. Can be used to check
/// the proper implementation of the distance function.
/// </summary>
///
/// <param name="kernel">The kernel function whose distance needs to be evaluated.</param>
/// <param name="x">An input point <c>x</c> given in input space.</param>
/// <param name="y">An input point <c>y</c> given in input space.</param>
///
/// <returns>
/// The distance between <paramref name="x"/> and <paramref name="y"/> in kernel (feature) space.
/// </returns>
///
public static double Distance(this IKernel kernel, double[] x, double[] y)
{
return(kernel.Function(x, x) + kernel.Function(y, y) - 2 * kernel.Function(x, y));
}
/// <summary>
/// Constructs a new <see cref="KernelFunctionCache"/>.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The inputs values.</param>
/// <param name="cacheSize">The size for the cache.</param>
///
public KernelFunctionCache(IKernel kernel, double[][] inputs, int cacheSize)
{
this.kernel = kernel;
this.inputs = inputs;
this.size = inputs.Length;
if (cacheSize < 0)
{
Enabled = false;
return;
}
else
{
Enabled = true;
}
if (cacheSize == 0)
cacheSize = inputs.Length;
this.cacheSize = cacheSize;
this.capacity = (size * size) / 2 + 1;
// Create lookup tables
this.lruIndices = new LinkedList<int>();
this.lruIndicesLookupTable = new Dictionary<int, LinkedListNode<int>>(capacity);
// Create cache for off-diagonal elements
this.data = new Dictionary<int, double>(capacity);
// Create cache for diagonal elements
this.diagonal = new double[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
this.diagonal[i] = kernel.Function(inputs[i], inputs[i]);
}
/// <summary>
/// Analytically solves the optimization problem for two Lagrange multipliers.
/// </summary>
///
private bool takeStep(int i1, int i2)
{
if (i1 == i2)
{
return(false);
}
// Lagrange multipliers
double alpha1a = alpha_a[i1];
double alpha1b = alpha_b[i1];
double alpha2a = alpha_a[i2];
double alpha2b = alpha_b[i2];
// Errors
double e1 = errors[i1];
double e2 = errors[i2];
double delta = e1 - e2;
// Kernel evaluation
double k11 = kernel.Function(inputs[i1], inputs[i1]);
double k12 = kernel.Function(inputs[i1], inputs[i2]);
double k22 = kernel.Function(inputs[i2], inputs[i2]);
double eta = k11 + k22 - 2.0 * k12;
double gamma = alpha1a - alpha1b + alpha2a - alpha2b;
// Assume the kernel is positive definite.
if (eta < 0)
{
eta = 0;
}
#region Optimize
bool case1 = false;
bool case2 = false;
bool case3 = false;
bool case4 = false;
bool changed = false;
bool finished = false;
double L, H, a1, a2;
while (!finished)
{
if (!case1 &&
(alpha1a > 0 || (alpha1b == 0 && delta > 0)) &&
(alpha2a > 0 || (alpha2b == 0 && delta < 0)))
{
// Compute L and H (w.r.t. alpha1, alpha2)
L = Math.Max(0, gamma - this.c);
H = Math.Min(this.c, gamma);
if (L < H)
{
if (eta > 0)
{
a2 = alpha2a - (delta / eta);
if (a2 > H)
{
a2 = H;
}
else if (a2 < L)
{
a2 = L;
}
}
else
{
double Lobj = -L * delta;
double Hobj = -H * delta;
if (Lobj > Hobj)
{
a2 = L;
}
else
{
a2 = H;
}
}
a1 = alpha1a - (a2 - alpha2a);
// Update alpha1, alpha2 if change is larger than some epsilon
if (Math.Abs(a1 - alpha1a) > roundingEpsilon ||
Math.Abs(a2 - alpha2a) > roundingEpsilon)
{
alpha1a = a1;
alpha2a = a2;
changed = true;
}
}
else
{
finished = true;
}
case1 = true;
}
else if (!case2 &&
(alpha1a > 0 || (alpha1b == 0 && delta > 2 * epsilon)) &&
(alpha2b > 0 || (alpha2a == 0 && delta > 2 * epsilon)))
{
// Compute L and H (w.r.t. alpha1, alpha2*)
L = Math.Max(0, -gamma);
H = Math.Min(this.c, -gamma + this.c);
if (L < H)
{
if (eta > 0)
{
a2 = alpha2b + ((delta - 2 * epsilon) / eta);
if (a2 > H)
{
a2 = H;
}
else if (a2 < L)
{
a2 = L;
}
}
else
{
double Lobj = L * (-2 * epsilon + delta);
double Hobj = H * (-2 * epsilon + delta);
if (Lobj > Hobj)
{
a2 = L;
}
else
{
a2 = H;
}
}
a1 = alpha1a + (a2 - alpha2b);
// Update alpha1, alpha2* if change is larger than some epsilon
if (Math.Abs(a1 - alpha1a) > roundingEpsilon ||
Math.Abs(a2 - alpha2b) > roundingEpsilon)
{
alpha1a = a1;
alpha2b = a2;
changed = true;
}
}
else
{
finished = true;
}
case2 = true;
}
else if (!case3 &&
(alpha1b > 0 || (alpha1a == 0 && delta < -2 * epsilon)) &&
(alpha2a > 0 || (alpha2b == 0 && delta < -2 * epsilon)))
{
// Compute L and H (w.r.t. alpha1*, alpha2)
L = Math.Max(0, gamma);
H = Math.Min(this.c, this.c + gamma);
if (L < H)
{
if (eta > 0)
{
a2 = alpha2a - ((delta + 2 * epsilon) / eta);
if (a2 > H)
{
a2 = H;
}
else if (a2 < L)
{
a2 = L;
}
}
else
{
double Lobj = -L * (2 * epsilon + delta);
double Hobj = -H * (2 * epsilon + delta);
if (Lobj > Hobj)
{
a2 = L;
}
else
{
a2 = H;
}
}
a1 = alpha1b + (a2 - alpha2a);
// Update alpha1*, alpha2 if change is larger than some epsilon
if (Math.Abs(a1 - alpha1b) > roundingEpsilon ||
Math.Abs(a2 - alpha2a) > roundingEpsilon)
{
alpha1b = a1;
alpha2a = a2;
changed = true;
}
}
else
{
finished = true;
}
case3 = true;
}
else if (!case4 &&
(alpha1b > 0 || (alpha1a == 0 && delta < 0)) &&
(alpha2b > 0 || (alpha2a == 0 && delta > 0)))
{
// Compute L and H (w.r.t. alpha1*, alpha2*)
L = Math.Max(0, -gamma - this.c);
H = Math.Min(this.c, -gamma);
if (L < H)
{
if (eta > 0)
{
a2 = alpha2b + delta / eta;
if (a2 > H)
{
a2 = H;
}
else if (a2 < L)
{
a2 = L;
}
}
else
{
double Lobj = L * delta;
double Hobj = H * delta;
if (Lobj > Hobj)
{
a2 = L;
}
else
{
a2 = H;
}
}
a1 = alpha1b - (a2 - alpha2b);
// Update alpha1*, alpha2* if change is larger than some epsilon
if (Math.Abs(a1 - alpha1b) > roundingEpsilon ||
Math.Abs(a2 - alpha2b) > roundingEpsilon)
{
alpha1b = a1;
alpha2b = a2;
changed = true;
}
}
else
{
finished = true;
}
case4 = true;
}
else
{
finished = true;
}
// Update the delta
delta += eta * ((alpha2a - alpha2b) - (alpha_a[i2] - alpha_b[i2]));
}
// If nothing has changed, return false.
if (!changed)
{
return(false);
}
#endregion
#region Update error cache
// Update error cache using new Lagrange multipliers
foreach (int i in I0)
{
if (i != i1 && i != i2)
{
// Update all in set i0 except i1 and i2 (because we have the kernel function cached for them)
errors[i] += ((alpha_a[i1] - alpha_b[i1]) - (alpha1a - alpha1b)) * kernel.Function(inputs[i1], inputs[i])
+ ((alpha_a[i2] - alpha_b[i2]) - (alpha2a - alpha2b)) * kernel.Function(inputs[i2], inputs[i]);
}
}
// Update error cache using new Lagrange multipliers for i1 and i2
errors[i1] += ((alpha_a[i1] - alpha_b[i1]) - (alpha1a - alpha1b)) * k11
+ ((alpha_a[i2] - alpha_b[i2]) - (alpha2a - alpha2b)) * k12;
errors[i2] += ((alpha_a[i1] - alpha_b[i1]) - (alpha1a - alpha1b)) * k12
+ ((alpha_a[i2] - alpha_b[i2]) - (alpha2a - alpha2b)) * k22;
#endregion
// to prevent precision problems
double m_Del = 1e-10;
if (alpha1a > c - m_Del * c)
{
alpha1a = c;
}
else if (alpha1a <= m_Del * c)
{
alpha1a = 0;
}
if (alpha1b > c - m_Del * c)
{
alpha1b = c;
}
else if (alpha1b <= m_Del * c)
{
alpha1b = 0;
}
if (alpha2a > c - m_Del * c)
{
alpha2a = c;
}
else if (alpha2a <= m_Del * c)
{
alpha2a = 0;
}
if (alpha2b > c - m_Del * c)
{
alpha2b = c;
}
else if (alpha2b <= m_Del * c)
{
alpha2b = 0;
}
#region Store the new Lagrange multipliers
// Store the changes in the alpha, alpha* arrays
alpha_a[i1] = alpha1a;
alpha_b[i1] = alpha1b;
alpha_a[i2] = alpha2a;
alpha_b[i2] = alpha2b;
#endregion
#region Update the sets of indices
// Update the sets of indices (for i1)
if ((0 < alpha1a && alpha1a < this.c) || (0 < alpha1b && alpha1b < this.c))
{
I0.Add(i1);
}
else
{
I0.Remove(i1);
}
if (alpha1a == 0 && alpha1b == 0)
{
I1.Add(i1);
}
else
{
I1.Remove(i1);
}
if (alpha1a == 0 && alpha1b == this.c)
{
I2.Add(i1);
}
else
{
I2.Remove(i1);
}
if (alpha1a == this.c && alpha1b == 0)
{
I3.Add(i1);
}
else
{
I3.Remove(i1);
}
// Update the sets of indices (for i2)
if ((0 < alpha2a && alpha2a < this.c) || (0 < alpha2b && alpha2b < this.c))
{
I0.Add(i2);
}
else
{
I0.Remove(i2);
}
if (alpha2a == 0 && alpha2b == 0)
{
I1.Add(i2);
}
else
{
I1.Remove(i2);
}
if (alpha2a == 0 && alpha2b == this.c)
{
I2.Add(i2);
}
else
{
I2.Remove(i2);
}
if (alpha2a == this.c && alpha2b == 0)
{
I3.Add(i2);
}
else
{
I3.Remove(i2);
}
#endregion
#region Compute the new thresholds
biasLower = Double.MinValue;
biasUpper = Double.MaxValue;
biasLowerIndex = -1;
biasUpperIndex = -1;
foreach (int i in I0)
{
if (0 < alpha_b[i] && alpha_b[i] < this.c &&
errors[i] + epsilon > biasLower)
{
biasLower = errors[i] + epsilon;
biasLowerIndex = i;
}
else if (0 < alpha_a[i] && alpha_a[i] < this.c &&
errors[i] - epsilon > biasLower)
{
biasLower = errors[i] - epsilon;
biasLowerIndex = i;
}
if (0 < alpha_a[i] && alpha_a[i] < this.c &&
errors[i] - epsilon < biasUpper)
{
biasUpper = errors[i] - epsilon;
biasUpperIndex = i;
}
else if (0 < alpha_b[i] && alpha_b[i] < this.c &&
errors[i] + epsilon < biasUpper)
{
biasUpper = errors[i] + epsilon;
biasUpperIndex = i;
}
}
if (!I0.Contains(i1))
{
if (I2.Contains(i1) && errors[i1] + epsilon > biasLower)
{
biasLower = errors[i1] + epsilon;
biasLowerIndex = i1;
}
else if (I1.Contains(i1) && errors[i1] - epsilon > biasLower)
{
biasLower = errors[i1] - epsilon;
biasLowerIndex = i1;
}
if (I3.Contains(i1) && errors[i1] - epsilon < biasUpper)
{
biasUpper = errors[i1] - epsilon;
biasUpperIndex = i1;
}
else if (I1.Contains(i1) && errors[i1] + epsilon < biasUpper)
{
biasUpper = errors[i1] + epsilon;
biasUpperIndex = i1;
}
}
if (!I0.Contains(i2))
{
if (I2.Contains(i2) && errors[i2] + epsilon > biasLower)
{
biasLower = errors[i2] + epsilon;
biasLowerIndex = i2;
}
else if (I1.Contains(i2) && errors[i2] - epsilon > biasLower)
{
biasLower = errors[i2] - epsilon;
biasLowerIndex = i2;
}
if (I3.Contains(i2) && errors[i2] - epsilon < biasUpper)
{
biasUpper = errors[i2] - epsilon;
biasUpperIndex = i2;
}
else if (I1.Contains(i2) && errors[i2] + epsilon < biasUpper)
{
biasUpper = errors[i2] + epsilon;
biasUpperIndex = i2;
}
}
if (biasLowerIndex == -1 || biasUpperIndex == -1)
{
throw new Exception();
}
#endregion
// Success.
return(true);
}
//---------------------------------------------
#region Public Methods
/// <summary>
/// Computes the Multi-Class Kernel Discriminant Analysis algorithm.
/// </summary>
///
public override void Compute()
{
// Get some initial information
int dimension = Source.GetLength(0);
double[,] source = Source;
double total = dimension;
// Create the Gram (Kernel) Matrix
double[,] K = new double[dimension, dimension];
for (int i = 0; i < dimension; i++)
{
double[] row = source.GetRow(i);
for (int j = i; j < dimension; j++)
{
double s = kernel.Function(row, source.GetRow(j));
K[i, j] = s; // Assume K will be symmetric
K[j, i] = s;
}
}
// Compute entire data set measures
base.Means = Statistics.Tools.Mean(K);
base.StandardDeviations = Statistics.Tools.StandardDeviation(K, Means);
// Initialize the kernel analogous scatter matrices
double[,] Sb = new double[dimension, dimension];
double[,] Sw = new double[dimension, dimension];
// For each class
for (int c = 0; c < Classes.Count; c++)
{
// Get the Kernel matrix class subset
double[,] Kc = K.Submatrix(Classes[c].Indices);
int count = Kc.GetLength(0);
// Get the Kernel matrix class mean
double[] mean = Statistics.Tools.Mean(Kc);
// Construct the Kernel equivalent of the Within-Class Scatter matrix
double[,] Swi = Statistics.Tools.Scatter(Kc, mean, (double)count);
// Sw = Sw + Swi
for (int i = 0; i < dimension; i++)
{
for (int j = 0; j < dimension; j++)
{
Sw[i, j] += Swi[i, j];
}
}
// Construct the Kernel equivalent of the Between-Class Scatter matrix
double[] d = mean.Subtract(base.Means);
double[,] Sbi = Matrix.OuterProduct(d, d).Multiply(total);
// Sb = Sb + Sbi
for (int i = 0; i < dimension; i++)
{
for (int j = 0; j < dimension; j++)
{
Sb[i, j] += Sbi[i, j];
}
}
// Store additional information
base.ClassScatter[c] = Swi;
base.ClassCount[c] = count;
base.ClassMeans[c] = mean;
base.ClassStandardDeviations[c] = Statistics.Tools.StandardDeviation(Kc, mean);
}
// Add regularization to avoid singularity
for (int i = 0; i < dimension; i++)
{
Sw[i, i] += regularization;
}
// Compute the generalized eigenvalue decomposition
GeneralizedEigenvalueDecomposition gevd = new GeneralizedEigenvalueDecomposition(Sb, Sw);
if (gevd.IsSingular) // check validity of the results
{
throw new SingularMatrixException("One of the matrices is singular. Please retry " +
"the method with a higher regularization constant.");
}
// Get the eigenvalues and corresponding eigenvectors
double[] evals = gevd.RealEigenvalues;
double[,] eigs = gevd.Eigenvectors;
// Sort eigenvalues and vectors in descending order
eigs = Matrix.Sort <double, double>(evals, eigs, new GeneralComparer(ComparerDirection.Descending, true));
if (threshold > 0)
{
// We will be discarding less important
// eigenvectors to conserve memory.
// Calculate component proportions
double sum = 0.0; // total variance
for (int i = 0; i < dimension; i++)
{
sum += Math.Abs(evals[i]);
}
if (sum > 0)
{
int keep = 0;
// Now we will detect how many components we have
// have to keep in order to achieve the level of
// explained variance specified by the threshold.
while (keep < dimension)
{
// Get the variance explained by the component
double explainedVariance = Math.Abs(evals[keep]);
// Check its proportion
double proportion = explainedVariance / sum;
// Now, if the component explains an
// enough proportion of the variance,
if (proportion > threshold)
{
keep++; // We can keep it.
}
else
{
break; // Otherwise we can stop, since the
}
// components are ordered by variance.
}
if (keep > 0)
{
// Resize the vectors keeping only needed components
eigs = eigs.Submatrix(0, dimension - 1, 0, keep - 1);
evals = evals.Submatrix(0, keep - 1);
}
else
{
// No component will be kept.
eigs = new double[dimension, 0];
evals = new double[0];
}
}
}
// Store information
base.Eigenvalues = evals;
base.DiscriminantMatrix = eigs;
base.ScatterBetweenClass = Sb;
base.ScatterWithinClass = Sw;
// Project into the kernel discriminant space
this.Result = K.Multiply(eigs);
// Compute feature space means for later classification
for (int c = 0; c < Classes.Count; c++)
{
ProjectionMeans[c] = ClassMeans[c].Multiply(eigs);
}
// Computes additional information about the analysis and creates the
// object-oriented structure to hold the discriminants found.
CreateDiscriminants();
}
/// <summary>
/// Analytically solves the optimization problem for two Lagrange multipliers.
/// </summary>
private bool takeStep(int i1, int i2)
{
if (i1 == i2)
{
return(false);
}
double[] p1 = inputs[i1]; // Input point at index i1
double a1 = alpha[i1]; // Classification label for p1
double y1 = outputs[i1]; // Lagrange multiplier for p1
// SVM output on p1 - y1. Check if it has already been computed
double e1 = (a1 > 0 && a1 < c) ? errors[i1] : compute(p1) - y1;
double[] p2 = inputs[i2]; // Input point at index i2
double a2 = alpha[i2]; // Classification label for p2
double y2 = outputs[i2]; // Lagrange multiplier for p2
// SVM output on p2 - y2. Check if it has already been computed
double e2 = (a2 > 0 && a2 < c) ? errors[i2] : compute(p2) - y2;
double s = y1 * y2;
// Compute L and H GABIZ.Baseing to equations (13) and (14) (Platt, 1998)
double L, H;
if (y1 != y2)
{
double d = a2 - a1;
L = System.Math.Max(0, d);
H = System.Math.Min(c, c + d);
}
else
{
double d = a2 + a1;
L = System.Math.Max(0, d - c);
H = System.Math.Min(c, d);
}
if (L == H)
{
return(false);
}
double k11, k22, k12, eta;
k11 = kernel.Function(p1, p1);
k12 = kernel.Function(p1, p2);
k22 = kernel.Function(p2, p2);
eta = k11 + k22 - 2.0 * k12;
// Compute new values for the lagrange multipliers a1 and a2
double new_a1, new_a2;
if (eta > 0)
{
new_a2 = a2 - y2 * (e2 - e1) / eta;
if (new_a2 < L)
{
new_a2 = L;
}
else if (new_a2 > H)
{
new_a2 = H;
}
}
else
{
double c1 = eta / 2;
double c2 = y2 * (e1 - e2) + eta * a2;
double Lobj = c1 * L * L + c2 * L;
double Hobj = c1 * H * H + c2 * H;
if (Lobj > Hobj + epsilon)
{
new_a2 = L;
}
else if (Lobj < Hobj - epsilon)
{
new_a2 = H;
}
else
{
new_a2 = a2;
}
}
if (System.Math.Abs(new_a2 - a2) < epsilon * (new_a2 + a2 + epsilon))
{
return(false);
}
new_a1 = a1 + s * (a2 - new_a2);
if (new_a1 < 0)
{
new_a2 += s * new_a1;
new_a1 = 0;
}
else if (new_a1 > c)
{
double d = new_a1 - c;
new_a2 += s * d;
new_a1 = c;
}
// Update threshold (bias) to reflect change in Lagrange multipliers
double b1 = 0, b2 = 0;
double new_b = 0, delta_b;
if (new_a1 > 0 && new_a1 < c)
{
// a1 is not at bounds
new_b = e1 + y1 * (new_a1 - a1) * k11 + y2 * (new_a2 - a2) * k12 + bias;
}
else
{
if (new_a2 > 0 && new_a2 < c)
{
// a1 is at bounds but a2 is not.
new_b = e2 + y1 * (new_a1 - a1) * k12 + y2 * (new_a2 - a2) * k22 + bias;
}
else
{
// Both new Lagrange multipliers are at bound. SMO algorithm
// chooses the threshold to be halfway in between b1 and b2.
b1 = e1 + y1 * (new_a1 - a1) * k11 + y2 * (new_a2 - a2) * k12 + bias;
b2 = e2 + y1 * (new_a1 - a1) * k12 + y2 * (new_a2 - a2) * k22 + bias;
new_b = (b1 + b2) / 2;
}
}
delta_b = new_b - bias;
bias = new_b;
// Update error cache using new Lagrange multipliers
double t1 = y1 * (new_a1 - a1);
double t2 = y2 * (new_a2 - a2);
for (int i = 0; i < inputs.Length; i++)
{
if (0 < alpha[i] && alpha[i] < c)
{
double[] point = inputs[i];
errors[i] +=
t1 * kernel.Function(p1, point) +
t2 * kernel.Function(p2, point) -
delta_b;
}
}
errors[i1] = 0f;
errors[i2] = 0f;
// Update lagrange multipliers
alpha[i1] = new_a1;
alpha[i2] = new_a2;
return(true);
}
/// <summary>
/// Attempts to retrieve the kernel function evaluated between point at index i
/// and j. If it is not cached, it will be computed and the cache will be updated.
/// </summary>
///
/// <param name="i">The index of the first point <c>p</c> to compute.</param>
/// <param name="j">The index of the second point <c>p</c> to compute.</param>
///
/// <remarks>The result of the kernel function k(p[i], p[j]).</remarks>
///
public double GetOrCompute(int i, int j)
{
if (data == null)
{
return(kernel.Function(inputs[i], inputs[j]));
}
if (i == j)
{
return(diagonal[i]);
}
// Keys should always be given as in
// the order (i, j) with i > j, so we
// always have (higher{i}, lower{j})
if (j > i)
{
int t = i;
i = j;
j = t;
}
if (matrix != null)
{
return(matrix[i][j]);
}
int key = (i * (i - 1)) / 2 + j;
double value;
// Check if the data is in the cache
if (!data.TryGetValue(key, out value))
{
// It is not. Compute the function and update
value = kernel.Function(inputs[i], inputs[j]);
// Save evaluation
data[key] = value;
// If we are over capacity,
if (data.Count > capacity)
{
// The first entry must be removed to leave
// room for the previously computed value
var first = lruIndices.First;
int discardedKey = first.Value;
// Remove the cached value for
// the least recently used key
data.Remove(discardedKey);
lruIndicesLookupTable.Remove(discardedKey);
// Avoid allocating memory by reusing the
// previously first node to hold the new
// data value and re-insert it at the end
lruIndices.RemoveFirst();
first.Value = key;
lruIndices.AddLast(first);
// Update the index lookup table
lruIndicesLookupTable[key] = first;
}
else
{
// Register the use of the variable in the LRU list
lruIndicesLookupTable[key] = lruIndices.AddLast(key);
}
misses++;
}
else
{
// It is. Update the LRU list to
// indicate the item has been used.
var node = lruIndicesLookupTable[key];
// Remove from middle and add to the end
lruIndices.Remove(node);
lruIndices.AddLast(node);
// Update the lookup table
lruIndicesLookupTable[key] = node;
hits++;
}
return(value);
}
double Q(int i, int j)
{
return(kernel.Function(inputs[i], inputs[j]));
}
/// <summary>
/// Estimates the <see cref="Complexity">complexity parameter C</see>
/// for a given kernel and an unbalanced data set.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The input samples.</param>
/// <param name="outputs">The output samples.</param>
///
/// <returns>A suitable value for positive C and negative C, respectively.</returns>
///
public static Tuple<double, double> EstimateComplexity(IKernel kernel, double[][] inputs, int[] outputs)
{
// Compute initial value for C as the number of examples
// divided by the trace of the input sample kernel matrix.
double negativeSum = 0.0;
double positiveSum = 0.0;
int negativeCount = 0;
int positiveCount = 0;
for (int i = 0; i < inputs.Length; i++)
{
if (outputs[i] == -1)
{
negativeSum += kernel.Function(inputs[i], inputs[i]);
negativeCount++;
}
else // outputs[i] == +1
{
positiveSum += kernel.Function(inputs[i], inputs[i]);
positiveCount++;
}
if (Double.IsNaN(positiveSum) || Double.IsNaN(negativeSum))
throw new OverflowException();
}
return Tuple.Create
(
positiveCount / positiveSum,
negativeCount / negativeSum
);
}
/// <summary>
/// Computes the Kernel Principal Component Analysis algorithm.
/// </summary>
///
public void Compute(int components)
{
if (components < 0 || components > Source.GetLength(0))
{
throw new ArgumentException(
"The number of components must be between 0 and " +
"the number of rows in your source data matrix.");
}
int dimension = Source.GetLength(0);
// If needed, center the source matrix
sourceCentered = Adjust(Source, Overwrite);
// Create the Gram (Kernel) Matrix
this.kernelMatrix = new double[dimension, dimension];
for (int i = 0; i < dimension; i++)
{
double[] row = sourceCentered.GetRow(i);
for (int j = i; j < dimension; j++)
{
double k = kernel.Function(row, sourceCentered.GetRow(j));
kernelMatrix[i, j] = k; // Kernel matrix is symmetric
kernelMatrix[j, i] = k;
}
}
// Center the Gram (Kernel) Matrix if requested
double[,] Kc = centerFeatureSpace ? centerKernel(kernelMatrix) : kernelMatrix;
// Perform the Eigenvalue Decomposition (EVD) of the Kernel matrix
EigenvalueDecomposition evd = new EigenvalueDecomposition(Kc, assumeSymmetric: true);
// Gets the Eigenvalues and corresponding Eigenvectors
double[] evals = evd.RealEigenvalues;
double[,] eigs = evd.Eigenvectors;
// Sort eigenvalues and vectors in descending order
eigs = Matrix.Sort(evals, eigs, new GeneralComparer(ComparerDirection.Descending, true));
// Eliminate unwanted components
if (components != Source.GetLength(0))
{
eigs = eigs.Submatrix(null, 0, components - 1);
evals = evals.Submatrix(0, components - 1);
}
if (threshold > 0)
{
// We will be discarding less important
// eigenvectors to conserve memory.
// Calculate component proportions
double sum = 0.0; // total variance
for (int i = 0; i < evals.Length; i++)
{
sum += Math.Abs(evals[i]);
}
if (sum > 0)
{
int keep = 0;
// Now we will detect how many components we have
// have to keep in order to achieve the level of
// explained variance specified by the threshold.
while (keep < components)
{
// Get the variance explained by the component
double explainedVariance = Math.Abs(evals[keep]);
// Check its proportion
double proportion = explainedVariance / sum;
// Now, if the component explains an
// enough proportion of the variance,
if (proportion > threshold)
{
keep++; // We can keep it.
}
else
{
break; // Otherwise we can stop, since the
}
// components are ordered by variance.
}
if (keep != components)
{
if (keep > 0)
{
// Resize the vectors keeping only needed components
eigs = eigs.Submatrix(0, components - 1, 0, keep - 1);
evals = evals.Submatrix(0, keep - 1);
}
else
{
// No component will be kept.
eigs = new double[dimension, 0];
evals = new double[0];
}
}
}
}
// Normalize eigenvectors
if (centerFeatureSpace)
{
for (int j = 0; j < evals.Length; j++)
{
double val = Math.Sqrt(Math.Abs(evals[j]));
for (int i = 0; i < eigs.GetLength(0); i++)
{
eigs[i, j] = eigs[i, j] / val;
}
}
}
// Set analysis properties
this.SingularValues = new double[evals.Length];
this.Eigenvalues = evals;
this.ComponentMatrix = eigs;
// Project the original data into principal component space
this.Result = Kc.Multiply(eigs);
// Computes additional information about the analysis and creates the
// object-oriented structure to hold the principal components found.
CreateComponents();
}
//---------------------------------------------
#region Public Methods
/// <summary>
/// Computes the Multi-Class Kernel Discriminant Analysis algorithm.
/// </summary>
public override void Compute()
{
// Get some initial information
int dimension = Source.GetLength(0);
double[,] source = Source;
double total = dimension;
// Create the Gram (Kernel) Matrix
double[,] K = new double[dimension, dimension];
for (int i = 0; i < dimension; i++)
{
for (int j = i; j < dimension; j++)
{
double s = kernel.Function(source.GetRow(i), source.GetRow(j));
K[i, j] = s; // Assume K will be symmetric
K[j, i] = s;
}
}
// Compute entire data set measures
base.Means = GABIZ.Base.Statistics.Tools.Mean(K);
base.StandardDeviations = GABIZ.Base.Statistics.Tools.StandardDeviation(K, Means);
// Initialize the kernel analogous scatter matrices
double[,] Sb = new double[dimension, dimension];
double[,] Sw = new double[dimension, dimension];
// For each class
for (int c = 0; c < Classes.Count; c++)
{
// Get the Kernel matrix class subset
double[,] Kc = K.Submatrix(Classes[c].Indexes);
int count = Kc.GetLength(0);
// Get the Kernel matrix class mean
double[] mean = GABIZ.Base.Statistics.Tools.Mean(Kc);
// Construct the Kernel equivalent of the Within-Class Scatter matrix
double[,] Swi = GABIZ.Base.Statistics.Tools.Scatter(Kc, mean, (double)count);
// Sw = Sw + Swi
for (int i = 0; i < dimension; i++)
{
for (int j = 0; j < dimension; j++)
{
Sw[i, j] += Swi[i, j];
}
}
// Construct the Kernel equivalent of the Between-Class Scatter matrix
double[] d = mean.Subtract(base.Means);
double[,] Sbi = Matrix.OuterProduct(d, d).Multiply(total);
// Sb = Sb + Sbi
for (int i = 0; i < dimension; i++)
{
for (int j = 0; j < dimension; j++)
{
Sb[i, j] += Sbi[i, j];
}
}
// Store additional information
base.ClassScatter[c] = Swi;
base.ClassCount[c] = count;
base.ClassMeans[c] = mean;
base.ClassStandardDeviations[c] = GABIZ.Base.Statistics.Tools.StandardDeviation(Kc, mean);
}
// Add regularization to avoid singularity
for (int i = 0; i < dimension; i++)
{
Sw[i, i] += regularization;
}
// Compute the generalized eigenvalue decomposition
GeneralizedEigenvalueDecomposition gevd = new GeneralizedEigenvalueDecomposition(Sb, Sw);
if (gevd.IsSingular) // check validity of the results
{
throw new SingularMatrixException("One of the matrices is singular. Please retry " +
"the method with a higher regularization constant.");
}
// Get the eigenvalues and corresponding eigenvectors
double[] evals = gevd.RealEigenvalues;
double[,] eigs = gevd.Eigenvectors;
// Sort eigenvalues and vectors in descending order
eigs = Matrix.Sort(evals, eigs, new GeneralComparer(ComparerDirection.Descending, true));
if (threshold > 0)
{
// Calculate proportions earlier
double sum = 0.0;
for (int i = 0; i < dimension; i++)
{
sum += System.Math.Abs(evals[i]);
}
if (sum > 0)
{
sum = 1.0 / sum;
// Discard less important eigenvectors to conserve memory
int keep = 0; while (keep < dimension &&
System.Math.Abs(evals[keep]) * sum > threshold)
{
keep++;
}
eigs = eigs.Submatrix(0, dimension - 1, 0, keep - 1);
evals = evals.Submatrix(0, keep - 1);
}
}
// Store information
base.Eigenvalues = evals;
base.DiscriminantMatrix = eigs;
base.ScatterBetweenClass = Sb;
base.ScatterWithinClass = Sw;
// Project into the kernel discriminant space
this.Result = K.Multiply(eigs);
// Compute feature space means for later classification
for (int c = 0; c < Classes.Count; c++)
{
double[] mean = new double[eigs.GetLength(1)];
for (int i = 0; i < eigs.GetLength(0); i++)
{
for (int j = 0; j < eigs.GetLength(1); j++)
{
mean[j] += ClassMeans[c][i] * eigs[i, j];
}
}
kernelClassMeans[c] = mean;
}
// Computes additional information about the analysis and creates the
// object-oriented structure to hold the discriminants found.
CreateDiscriminants();
}
/// <summary>
/// Analytically solves the optimization problem for two Lagrange multipliers.
/// </summary>
///
private bool takeStep(int i1, int i2)
{
if (i1 == i2)
{
return(false);
}
double[] p1 = inputs[i1]; // Input point at index i1
double alph1 = alpha[i1]; // Lagrange multiplier for p1
double y1 = outputs[i1]; // Classification label for p1
// SVM output on p1 - y1. Check if it has already been computed
double e1 = (alph1 > 0 && alph1 < c) ? errors[i1] : compute(p1) - y1;
double[] p2 = inputs[i2]; // Input point at index i2
double alph2 = alpha[i2]; // Lagrange multiplier for p2
double y2 = outputs[i2]; // Classification label for p2
// SVM output on p2 - y2. Check if it has already been computed
double e2 = (alph2 > 0 && alph2 < c) ? errors[i2] : compute(p2) - y2;
double s = y1 * y2;
// Compute L and H according to equations (13) and (14) (Platt, 1998)
double L, H;
if (y1 != y2)
{
// If the target y1 does not equal the target (13)
// y2, then the following bounds apply to a2:
L = Math.Max(0, alph2 - alph1);
H = Math.Min(c, c + alph2 - alph1);
}
else
{
// If the target y1 does equal the target (14)
// y2, then the following bounds apply to a2:
L = Math.Max(0, alph2 + alph1 - c);
H = Math.Min(c, alph2 + alph1);
}
if (L == H)
{
return(false);
}
double k11, k22, k12, eta;
k11 = kernel.Function(p1, p1);
k12 = kernel.Function(p1, p2);
k22 = kernel.Function(p2, p2);
eta = k11 + k22 - 2.0 * k12;
double a1, a2;
if (eta > 0)
{
a2 = alph2 - y2 * (e2 - e1) / eta;
if (a2 < L)
{
a2 = L;
}
else if (a2 > H)
{
a2 = H;
}
}
else
{
// Compute objective function Lobj and Hobj at
// a2=L and a2=H respectively, using (eq. 19)
double L1 = alph1 + s * (alph2 - L);
double H1 = alph1 + s * (alph2 - H);
double f1 = y1 * (e1 + bias) - alph1 * k11 - s * alph2 * k12;
double f2 = y2 * (e2 + bias) - alph2 * k22 - s * alph1 * k12;
double Lobj = -0.5 * L1 * L1 * k11 - 0.5 * L * L * k22 - s * L * L1 * k12 - L1 * f1 - L * f2;
double Hobj = -0.5 * H1 * H1 * k11 - 0.5 * H * H * k22 - s * H * H1 * k12 - H1 * f1 - H * f2;
if (Lobj > Hobj + epsilon)
{
a2 = L;
}
else if (Lobj < Hobj - epsilon)
{
a2 = H;
}
else
{
a2 = alph2;
}
}
if (Math.Abs(a2 - alph2) < epsilon * (a2 + alph2 + epsilon))
{
return(false);
}
a1 = alph1 + s * (alph2 - a2);
if (a1 < 0)
{
a2 += s * a1;
a1 = 0;
}
else if (a1 > c)
{
double d = a1 - c;
a2 += s * d;
a1 = c;
}
// Update threshold (bias) to reflect change in Lagrange multipliers
double b1 = 0, b2 = 0;
double new_b = 0, delta_b;
if (a1 > 0 && a1 < c)
{
// a1 is not at bounds
new_b = e1 + y1 * (a1 - alph1) * k11 + y2 * (a2 - alph2) * k12 + bias;
}
else
{
if (a2 > 0 && a2 < c)
{
// a1 is at bounds but a2 is not.
new_b = e2 + y1 * (a1 - alph1) * k12 + y2 * (a2 - alph2) * k22 + bias;
}
else
{
// Both new Lagrange multipliers are at bound. SMO algorithm
// chooses the threshold to be halfway in between b1 and b2.
b1 = e1 + y1 * (a1 - alph1) * k11 + y2 * (a2 - alph2) * k12 + bias;
b2 = e2 + y1 * (a1 - alph1) * k12 + y2 * (a2 - alph2) * k22 + bias;
new_b = (b1 + b2) / 2;
}
}
delta_b = new_b - bias;
bias = new_b;
// Update error cache using new Lagrange multipliers
double t1 = y1 * (a1 - alph1);
double t2 = y2 * (a2 - alph2);
for (int i = 0; i < inputs.Length; i++)
{
if (0 < alpha[i] && alpha[i] < c)
{
double[] point = inputs[i];
errors[i] +=
t1 * kernel.Function(p1, point) +
t2 * kernel.Function(p2, point) -
delta_b;
}
}
errors[i1] = 0f;
errors[i2] = 0f;
// Update lagrange multipliers
alpha[i1] = a1;
alpha[i2] = a2;
return(true);
}
//---------------------------------------------
#region Public Methods
/// <summary>Computes the Kernel Principal Component Analysis algorithm.</summary>
public override void Compute()
{
int rows = Source.GetLength(0);
// Center (adjust) the source matrix
sourceCentered = Adjust(Source, Overwrite);
// Create the Gram (Kernel) Matrix
double[,] K = new double[rows, rows];
for (int i = 0; i < rows; i++)
{
for (int j = i; j < rows; j++)
{
double k = kernel.Function(sourceCentered.GetRow(i), sourceCentered.GetRow(j));
K[i, j] = k; // Kernel matrix is symmetric
K[j, i] = k;
}
}
// Center the Gram (Kernel) Matrix
if (centerFeatureSpace)
{
K = centerKernel(K);
}
// Perform the Eigenvalue Decomposition (EVD) of the Kernel matrix
EigenvalueDecomposition evd = new EigenvalueDecomposition(K, true);
// Gets the eigenvalues and corresponding eigenvectors
double[] evals = evd.RealEigenvalues;
double[,] eigs = evd.Eigenvectors;
// Sort eigenvalues and vectors in descending order
eigs = Matrix.Sort(evals, eigs, new GeneralComparer(ComparerDirection.Descending, true));
if (threshold > 0)
{
// Calculate proportions in advance
double sum = 0.0;
for (int i = 0; i < evals.Length; i++)
{
sum += System.Math.Abs(evals[i]);
}
if (sum > 0)
{
sum = 1.0 / sum;
// Discard less important eigenvectors to conserve memory
int keep = 0; while (keep < evals.Length &&
System.Math.Abs(evals[keep]) * sum > threshold)
{
keep++;
}
eigs = eigs.Submatrix(0, evals.Length - 1, 0, keep - 1);
evals = evals.Submatrix(0, keep - 1);
}
}
// Normalize eigenvectors
if (centerFeatureSpace)
{
for (int j = 0; j < evals.Length; j++)
{
double eig = System.Math.Sqrt(System.Math.Abs(evals[j]));
for (int i = 0; i < eigs.GetLength(0); i++)
{
eigs[i, j] = eigs[i, j] / eig;
}
}
}
// Set analysis properties
this.SingularValues = new double[evals.Length];
this.Eigenvalues = evals;
this.ComponentMatrix = eigs;
// Project the original data into principal component space
this.Result = K.Multiply(eigs);
// Computes additional information about the analysis and creates the
// object-oriented structure to hold the principal components found.
CreateComponents();
}
/// <summary>
/// Estimates the <see cref="Complexity">complexity parameter C</see>
/// for a given kernel and a given data set.
/// </summary>
///
/// <param name="kernel">The kernel function.</param>
/// <param name="inputs">The input samples.</param>
///
/// <returns>A suitable value for C.</returns>
///
public static double EstimateComplexity(IKernel kernel, double[][] inputs)
{
// Compute initial value for C as the number of examples
// divided by the trace of the input sample kernel matrix.
double sum = 0.0;
for (int i = 0; i < inputs.Length; i++)
sum += kernel.Function(inputs[i], inputs[i]);
return inputs.Length / sum;
}
/// <summary>
/// Аналитически решается задача оптимизации двух множителей Лагранжа
/// </summary>
private bool TakeStep(int i1, int i2)
{
if (i1 == i2)
{
return(false);
}
double[] p1 = inputs[i1];
double alph1 = alpha[i1];
double y1 = outputs[i1];
double e1 = (alph1 > 0 && alph1 < c) ? errors[i1] : Compute(p1) - y1;
double[] p2 = inputs[i2];
double alph2 = alpha[i2];
double y2 = outputs[i2];
double e2 = (alph2 > 0 && alph2 < c) ? errors[i2] : Compute(p2) - y2;
double s = y1 * y2;
double L, H;
if (y1 != y2)
{
L = Math.Max(0, alph2 - alph1);
H = Math.Min(c, c + alph2 - alph1);
}
else
{
L = Math.Max(0, alph2 + alph1 - c);
H = Math.Min(c, alph2 + alph1);
}
if (L == H)
{
return(false);
}
double k11, k22, k12, eta;
k11 = kernel.Function(p1, p1);
k12 = kernel.Function(p1, p2);
k22 = kernel.Function(p2, p2);
eta = k11 + k22 - 2.0 * k12;
double a1, a2;
if (eta > 0)
{
a2 = alph2 - y2 * (e2 - e1) / eta;
if (a2 < L)
{
a2 = L;
}
else if (a2 > H)
{
a2 = H;
}
}
else
{
double L1 = alph1 + s * (alph2 - L);
double H1 = alph1 + s * (alph2 - H);
double f1 = y1 * (e1 + bias) - alph1 * k11 - s * alph2 * k12;
double f2 = y2 * (e2 + bias) - alph2 * k22 - s * alph1 * k12;
double Lobj = -0.5 * L1 * L1 * k11 - 0.5 * L * L * k22 - s * L * L1 * k12 - L1 * f1 - L * f2;
double Hobj = -0.5 * H1 * H1 * k11 - 0.5 * H * H * k22 - s * H * H1 * k12 - H1 * f1 - H * f2;
if (Lobj > Hobj + epsilon)
{
a2 = L;
}
else if (Lobj < Hobj - epsilon)
{
a2 = H;
}
else
{
a2 = alph2;
}
}
if (Math.Abs(a2 - alph2) < epsilon * (a2 + alph2 + epsilon))
{
return(false);
}
a1 = alph1 + s * (alph2 - a2);
if (a1 < 0)
{
a2 += s * a1;
a1 = 0;
}
else if (a1 > c)
{
double d = a1 - c;
a2 += s * d;
a1 = c;
}
double b1 = 0, b2 = 0;
double new_b = 0, delta_b;
if (a1 > 0 && a1 < c)
{
new_b = e1 + y1 * (a1 - alph1) * k11 + y2 * (a2 - alph2) * k12 + bias;
}
else
{
if (a2 > 0 && a2 < c)
{
new_b = e2 + y1 * (a1 - alph1) * k12 + y2 * (a2 - alph2) * k22 + bias;
}
else
{
b1 = e1 + y1 * (a1 - alph1) * k11 + y2 * (a2 - alph2) * k12 + bias;
b2 = e2 + y1 * (a1 - alph1) * k12 + y2 * (a2 - alph2) * k22 + bias;
new_b = (b1 + b2) / 2;
}
}
delta_b = new_b - bias;
bias = new_b;
double t1 = y1 * (a1 - alph1);
double t2 = y2 * (a2 - alph2);
for (int i = 0; i < inputs.Length; i++)
{
if (0 < alpha[i] && alpha[i] < c)
{
double[] point = inputs[i];
errors[i] +=
t1 * kernel.Function(p1, point) +
t2 * kernel.Function(p2, point) -
delta_b;
}
}
errors[i1] = 0f;
errors[i2] = 0f;
alpha[i1] = a1;
alpha[i2] = a2;
return(true);
}
