[Test()] public void TestMultiply()
{
var matrix = new Matrix<float>(5, 5);
float[] row = { 1, 2, 3, 4, 5 };
for (int i = 0; i < 5; i++)
matrix.SetRow(i, row);
matrix.Multiply(2.5f);
float[] testrow = { 2.5f, 5f, 7.5f, 10f, 12.5f };
Assert.AreEqual(testrow, matrix.GetRow(3));
}
public void ExceptionInterfaceIncompatibleMatrices()
{
IMathematicalMatrix matrix1 = new Matrix(2, 3);
matrix1[0, 0] = 1;
matrix1[0, 1] = 2;
matrix1[0, 2] = -4;
matrix1[1, 0] = 0;
matrix1[1, 1] = 3;
matrix1[1, 2] = -1;
IMathematicalMatrix matrix2 = MatrixTest.GetTestMatrix();
matrix1.Multiply(matrix2);
}
public void MultiplyRaiseException()
{
Matrix matrix1 = new Matrix(new double[][] { new double[] { 1.0, 2.0 }, new double[] { 3.0, 4.0 } });
Matrix matrix2 = new Matrix(new double[][] { new double[] { 5.0, 6.0 }, new double[] { 7.0, 8.0 }, new double[] { 9.0, 10.0 } });
try
{
matrix1.Multiply(matrix2);
Assert.Fail();
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(InvalidOperationException));
Assert.AreEqual("Matrices cannot be multiplied", ex.Message);
}
}
public void MultiplyByNumber()
{
Matrix matrix1 = new Matrix(new double[][] { new double[] { 1.0, 2.0 }, new double[] { 3.0, 4.0 } });
Matrix matrix2 = new Matrix(new double[][] { new double[] { 1.0 * 3.0, 2.0 * 3.0 }, new double[] { 3.0 * 3.0, 4.0 * 3.0 } });
Matrix matrix = matrix1.Multiply(3.0);
Assert.AreEqual(matrix2, matrix);
}
public void Multiply()
{
Matrix matrix1 = new Matrix(new double[][] { new double[] { 1.0, 2.0 }, new double[] { 3.0, 4.0 } });
Matrix matrix2 = new Matrix(new double[][] { new double[] { 5.0, 6.0 }, new double[] { 7.0, 8.0 } });
Matrix matrix = matrix1.Multiply(matrix2);
Assert.AreEqual(4, matrix.Size);
var elements = matrix.Elements;
Assert.AreEqual((1.0 * 5.0) + (2.0 * 7.0), elements[0][0]);
Assert.AreEqual((1.0 * 6.0) + (2.0 * 8.0), elements[0][1]);
Assert.AreEqual((3.0 * 5.0) + (4.0 * 7.0), elements[1][0]);
Assert.AreEqual((3.0 * 6.0) + (4.0 * 8.0), elements[1][1]);
}
private static double Gradient(RealVector A, RealVector grad, double[,] data, double[] classes, int dimensions, int neighborSamples, double regularization) {
var instances = data.GetLength(0);
var attributes = data.GetLength(1);
var AMatrix = new Matrix(A, A.Length / dimensions, dimensions);
alglib.sparsematrix probabilities;
alglib.sparsecreate(instances, instances, out probabilities);
var transformedDistances = new Dictionary<int, double>(instances);
for (int i = 0; i < instances; i++) {
var iVector = new Matrix(GetRow(data, i), data.GetLength(1));
for (int k = 0; k < instances; k++) {
if (k == i) {
transformedDistances.Remove(k);
continue;
}
var kVector = new Matrix(GetRow(data, k));
transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SumOfSquares());
}
var normalization = transformedDistances.Sum(x => x.Value);
if (normalization <= 0) continue;
foreach (var s in transformedDistances.Where(x => x.Value > 0).OrderByDescending(x => x.Value).Take(neighborSamples)) {
alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
}
}
alglib.sparseconverttocrs(probabilities); // needed to enumerate in order (top-down and left-right)
int t0 = 0, t1 = 0, r, c;
double val;
var pi = new double[instances];
while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
if (classes[r].IsAlmost(classes[c])) {
pi[r] += val;
}
}
var innerSum = new double[attributes, attributes];
while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
var vector = new Matrix(GetRow(data, r)).Subtract(new Matrix(GetRow(data, c)));
vector.OuterProduct(vector).Multiply(val * pi[r]).AddTo(innerSum);
if (classes[r].IsAlmost(classes[c])) {
vector.OuterProduct(vector).Multiply(-val).AddTo(innerSum);
}
}
var func = -pi.Sum() + regularization * AMatrix.SumOfSquares();
r = 0;
var newGrad = AMatrix.Multiply(-2.0).Transpose().Multiply(new Matrix(innerSum)).Transpose();
foreach (var g in newGrad) {
grad[r] = g + regularization * 2 * A[r];
r++;
}
return func;
}
/// <summary>
/// Solves the matrix equation Ax = b, where A is the coefficient matrix, b is the
/// solution vector and x is the unknown vector.
/// </summary>
/// <param name="matrix">The coefficient matrix, <c>A</c>.</param>
/// <param name="input">The solution vector, <c>b</c></param>
/// <param name="result">The result vector, <c>x</c></param>
public void Solve(Matrix matrix, Vector input, Vector result)
{
// If we were stopped before, we are no longer
// We're doing this at the start of the method to ensure
// that we can use these fields immediately.
_hasBeenStopped = false;
// Error checks
if (matrix == null)
{
throw new ArgumentNullException("matrix");
}
if (matrix.RowCount != matrix.ColumnCount)
{
throw new ArgumentException(Resources.ArgumentMatrixSquare, "matrix");
}
if (input == null)
{
throw new ArgumentNullException("input");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
if (result.Count != input.Count)
{
throw new ArgumentException(Resources.ArgumentVectorsSameLength);
}
if (input.Count != matrix.RowCount)
{
throw Matrix.DimensionsDontMatch<ArgumentException>(input, matrix);
}
// Initialize the solver fields
// Set the convergence monitor
if (_iterator == null)
{
_iterator = Iterator.CreateDefault();
}
if (_preconditioner == null)
{
_preconditioner = new UnitPreconditioner();
}
_preconditioner.Initialize(matrix);
var d = new DenseVector(input.Count);
var r = new DenseVector(input);
var uodd = new DenseVector(input.Count);
var ueven = new DenseVector(input.Count);
var v = new DenseVector(input.Count);
var pseudoResiduals = new DenseVector(input);
var x = new DenseVector(input.Count);
var yodd = new DenseVector(input.Count);
var yeven = new DenseVector(input);
// Temp vectors
var temp = new DenseVector(input.Count);
var temp1 = new DenseVector(input.Count);
var temp2 = new DenseVector(input.Count);
// Initialize
var startNorm = input.Norm(2);
// Define the scalars
double alpha = 0;
double eta = 0;
double theta = 0;
var tau = startNorm;
var rho = tau * tau;
// Calculate the initial values for v
// M temp = yEven
_preconditioner.Approximate(yeven, temp);
// v = A temp
matrix.Multiply(temp, v);
// Set uOdd
v.CopyTo(ueven);
// Start the iteration
var iterationNumber = 0;
while (ShouldContinue(iterationNumber, result, input, pseudoResiduals))
{
// First part of the step, the even bit
if (IsEven(iterationNumber))
{
// sigma = (v, r)
var sigma = v.DotProduct(r);
if (sigma.AlmostEqual(0, 1))
{
// FAIL HERE
_iterator.IterationCancelled();
break;
}
// alpha = rho / sigma
alpha = rho / sigma;
// yOdd = yEven - alpha * v
v.Multiply(-alpha, temp1);
yeven.Add(temp1, yodd);
// Solve M temp = yOdd
_preconditioner.Approximate(yodd, temp);
// uOdd = A temp
matrix.Multiply(temp, uodd);
}
// The intermediate step which is equal for both even and
// odd iteration steps.
// Select the correct vector
var uinternal = IsEven(iterationNumber) ? ueven : uodd;
var yinternal = IsEven(iterationNumber) ? yeven : yodd;
// pseudoResiduals = pseudoResiduals - alpha * uOdd
uinternal.Multiply(-alpha, temp1);
pseudoResiduals.Add(temp1, temp2);
temp2.CopyTo(pseudoResiduals);
// d = yOdd + theta * theta * eta / alpha * d
d.Multiply(theta * theta * eta / alpha, temp);
yinternal.Add(temp, d);
// theta = ||pseudoResiduals||_2 / tau
theta = pseudoResiduals.Norm(2) / tau;
var c = 1 / Math.Sqrt(1 + (theta * theta));
// tau = tau * theta * c
tau *= theta * c;
// eta = c^2 * alpha
eta = c * c * alpha;
// x = x + eta * d
d.Multiply(eta, temp1);
x.Add(temp1, temp2);
temp2.CopyTo(x);
// Check convergence and see if we can bail
if (!ShouldContinue(iterationNumber, result, input, pseudoResiduals))
{
// Calculate the real values
_preconditioner.Approximate(x, result);
// Calculate the true residual. Use the temp vector for that
// so that we don't pollute the pseudoResidual vector for no
// good reason.
CalculateTrueResidual(matrix, temp, result, input);
// Now recheck the convergence
if (!ShouldContinue(iterationNumber, result, input, temp))
{
// We're all good now.
return;
}
}
// The odd step
if (!IsEven(iterationNumber))
{
if (rho.AlmostEqual(0, 1))
{
// FAIL HERE
_iterator.IterationCancelled();
break;
}
var rhoNew = pseudoResiduals.DotProduct(r);
var beta = rhoNew / rho;
// Update rho for the next loop
rho = rhoNew;
// yOdd = pseudoResiduals + beta * yOdd
yodd.Multiply(beta, temp1);
pseudoResiduals.Add(temp1, yeven);
// Solve M temp = yOdd
_preconditioner.Approximate(yeven, temp);
// uOdd = A temp
matrix.Multiply(temp, ueven);
// v = uEven + beta * (uOdd + beta * v)
v.Multiply(beta, temp1);
uodd.Add(temp1, temp);
temp.Multiply(beta, temp1);
ueven.Add(temp1, v);
}
// Calculate the real values
_preconditioner.Approximate(x, result);
iterationNumber++;
}
}
/// <summary>
/// return a * b
/// </summary>
/// <param name="a"></param>
/// <param name="b"></param>
/// <returns></returns>
public static Matrix Multiply(Matrix a, Matrix b) {
a.ShouldNotBeNull("a");
b.ShouldNotBeNull("b");
Guard.Assert(a.Cols == b.Rows, "Matrix dimension is not match. a.Cols equals b.Rows for Multipling.");
var result = new Matrix(a.Rows, b.Cols);
result.Multiply(a, b);
return result;
}
/// <summary>
/// Solves the matrix equation Ax = b, where A is the coefficient matrix, b is the
/// solution vector and x is the unknown vector.
/// </summary>
/// <param name="matrix">The coefficient <see cref="Matrix"/>, <c>A</c>.</param>
/// <param name="input">The solution <see cref="Vector"/>, <c>b</c>.</param>
/// <param name="result">The result <see cref="Vector"/>, <c>x</c>.</param>
public void Solve(Matrix matrix, Vector input, Vector result)
{
// If we were stopped before, we are no longer
// We're doing this at the start of the method to ensure
// that we can use these fields immediately.
_hasBeenStopped = false;
// Parameters checks
if (matrix == null)
{
throw new ArgumentNullException("matrix");
}
if (matrix.RowCount != matrix.ColumnCount)
{
throw new ArgumentException(Resources.ArgumentMatrixSquare, "matrix");
}
if (input == null)
{
throw new ArgumentNullException("input");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
if (result.Count != input.Count)
{
throw new ArgumentException(Resources.ArgumentVectorsSameLength);
}
if (input.Count != matrix.RowCount)
{
throw Matrix.DimensionsDontMatch<ArgumentException>(input, result);
}
// Initialize the solver fields
// Set the convergence monitor
if (_iterator == null)
{
_iterator = Iterator.CreateDefault();
}
if (_preconditioner == null)
{
_preconditioner = new UnitPreconditioner();
}
_preconditioner.Initialize(matrix);
// Compute r_0 = b - Ax_0 for some initial guess x_0
// In this case we take x_0 = vector
// This is basically a SAXPY so it could be made a lot faster
Vector residuals = new DenseVector(matrix.RowCount);
CalculateTrueResidual(matrix, residuals, result, input);
// Choose r~ (for example, r~ = r_0)
var tempResiduals = residuals.Clone();
// create seven temporary vectors needed to hold temporary
// coefficients. All vectors are mangled in each iteration.
// These are defined here to prevent stressing the garbage collector
Vector vecP = new DenseVector(residuals.Count);
Vector vecPdash = new DenseVector(residuals.Count);
Vector nu = new DenseVector(residuals.Count);
Vector vecS = new DenseVector(residuals.Count);
Vector vecSdash = new DenseVector(residuals.Count);
Vector temp = new DenseVector(residuals.Count);
Vector temp2 = new DenseVector(residuals.Count);
// create some temporary double variables that are needed
// to hold values in between iterations
Complex currentRho = 0;
Complex alpha = 0;
Complex omega = 0;
var iterationNumber = 0;
while (ShouldContinue(iterationNumber, result, input, residuals))
{
// rho_(i-1) = r~^T r_(i-1) // dotproduct r~ and r_(i-1)
var oldRho = currentRho;
currentRho = tempResiduals.DotProduct(residuals);
// if (rho_(i-1) == 0) // METHOD FAILS
// If rho is only 1 ULP from zero then we fail.
if (currentRho.Real.AlmostEqual(0, 1) && currentRho.Imaginary.AlmostEqual(0, 1))
{
// Rho-type breakdown
throw new Exception("Iterative solver experience a numerical break down");
}
if (iterationNumber != 0)
{
// beta_(i-1) = (rho_(i-1)/rho_(i-2))(alpha_(i-1)/omega(i-1))
var beta = (currentRho / oldRho) * (alpha / omega);
// p_i = r_(i-1) + beta_(i-1)(p_(i-1) - omega_(i-1) * nu_(i-1))
nu.Multiply(-omega, temp);
vecP.Add(temp, temp2);
temp2.CopyTo(vecP);
vecP.Multiply(beta, vecP);
vecP.Add(residuals, temp2);
temp2.CopyTo(vecP);
}
else
{
// p_i = r_(i-1)
residuals.CopyTo(vecP);
}
// SOLVE Mp~ = p_i // M = preconditioner
_preconditioner.Approximate(vecP, vecPdash);
// nu_i = Ap~
matrix.Multiply(vecPdash, nu);
// alpha_i = rho_(i-1)/ (r~^T nu_i) = rho / dotproduct(r~ and nu_i)
alpha = currentRho * 1 / tempResiduals.DotProduct(nu);
// s = r_(i-1) - alpha_i nu_i
nu.Multiply(-alpha, temp);
residuals.Add(temp, vecS);
// Check if we're converged. If so then stop. Otherwise continue;
// Calculate the temporary result.
// Be careful not to change any of the temp vectors, except for
// temp. Others will be used in the calculation later on.
// x_i = x_(i-1) + alpha_i * p^_i + s^_i
vecPdash.Multiply(alpha, temp);
temp.Add(vecSdash, temp2);
temp2.CopyTo(temp);
temp.Add(result, temp2);
temp2.CopyTo(temp);
// Check convergence and stop if we are converged.
if (!ShouldContinue(iterationNumber, temp, input, vecS))
{
temp.CopyTo(result);
// Calculate the true residual
CalculateTrueResidual(matrix, residuals, result, input);
// Now recheck the convergence
if (!ShouldContinue(iterationNumber, result, input, residuals))
{
// We're all good now.
return;
}
// Continue the calculation
iterationNumber++;
continue;
}
// SOLVE Ms~ = s
_preconditioner.Approximate(vecS, vecSdash);
// temp = As~
matrix.Multiply(vecSdash, temp);
// omega_i = temp^T s / temp^T temp
omega = temp.DotProduct(vecS) / temp.DotProduct(temp);
// x_i = x_(i-1) + alpha_i p^ + omega_i s^
temp.Multiply(-omega, residuals);
residuals.Add(vecS, temp2);
temp2.CopyTo(residuals);
vecSdash.Multiply(omega, temp);
result.Add(temp, temp2);
temp2.CopyTo(result);
vecPdash.Multiply(alpha, temp);
result.Add(temp, temp2);
temp2.CopyTo(result);
// for continuation it is necessary that omega_i != 0.0
// If omega is only 1 ULP from zero then we fail.
if (omega.Real.AlmostEqual(0, 1) && omega.Imaginary.AlmostEqual(0, 1))
{
// Omega-type breakdown
throw new Exception("Iterative solver experience a numerical break down");
}
if (!ShouldContinue(iterationNumber, result, input, residuals))
{
// Recalculate the residuals and go round again. This is done to ensure that
// we have the proper residuals.
// The residual calculation based on omega_i * s can be off by a factor 10. So here
// we calculate the real residual (which can be expensive) but we only do it if we're
// sufficiently close to the finish.
CalculateTrueResidual(matrix, residuals, result, input);
}
iterationNumber++;
}
}
private void IterateBatch()
{
SetupLoss();
SparseBooleanMatrix user_reverse_connections = (SparseBooleanMatrix) user_connections.Transpose();
// I. compute gradients
var user_factors_gradient = new Matrix<float>(user_factors.dim1, user_factors.dim2);
var item_factors_gradient = new Matrix<float>(item_factors.dim1, item_factors.dim2);
var user_bias_gradient = new float[user_factors.dim1];
var item_bias_gradient = new float[item_factors.dim1];
// I.1 prediction error
for (int index = 0; index < ratings.Count; index++)
{
int user_id = ratings.Users[index];
int item_id = ratings.Items[index];
// prediction
float score = global_bias + user_bias[user_id] + item_bias[item_id];
score += DataType.MatrixExtensions.RowScalarProduct(user_factors, user_id, item_factors, item_id);
double sig_score = 1 / (1 + Math.Exp(-score));
float prediction = (float) (MinRating + sig_score * rating_range_size);
float error = prediction - ratings[index];
float gradient_common = compute_gradient_common(sig_score, error);
user_bias_gradient[user_id] += gradient_common;
item_bias_gradient[item_id] += gradient_common;
for (int f = 0; f < NumFactors; f++)
{
float u_f = user_factors[user_id, f];
float i_f = item_factors[item_id, f];
user_factors_gradient.Inc(user_id, f, gradient_common * i_f);
item_factors_gradient.Inc(item_id, f, gradient_common * u_f);
}
}
// I.2 L2 regularization
// biases
for (int u = 0; u < user_bias_gradient.Length; u++)
user_bias_gradient[u] += user_bias[u] * RegU * BiasReg;
for (int i = 0; i < item_bias_gradient.Length; i++)
item_bias_gradient[i] += item_bias[i] * RegI * BiasReg;
// latent factors
for (int u = 0; u < user_factors_gradient.dim1; u++)
for (int f = 0; f < user_factors_gradient.dim2; f++)
user_factors_gradient.Inc(u, f, user_factors[u, f] * RegU);
for (int i = 0; i < item_factors_gradient.dim1; i++)
for (int f = 0; f < item_factors_gradient.dim2; f++)
item_factors_gradient.Inc(i, f, item_factors[i, f] * RegI);
// I.3 social network regularization -- see eq. (13) in the paper
if (SocialRegularization != 0)
for (int u = 0; u < user_factors_gradient.dim1; u++)
{
var sum_connections = new float[NumFactors];
float bias_sum_connections = 0;
int num_connections = user_connections[u].Count;
foreach (int v in user_connections[u])
{
bias_sum_connections += user_bias[v];
for (int f = 0; f < sum_connections.Length; f++)
sum_connections[f] += user_factors[v, f];
}
if (num_connections != 0)
{
user_bias_gradient[u] += social_regularization * (user_bias[u] - bias_sum_connections / num_connections);
for (int f = 0; f < user_factors_gradient.dim2; f++)
user_factors_gradient.Inc(u, f, social_regularization * (user_factors[u, f] - sum_connections[f] / num_connections));
}
foreach (int v in user_reverse_connections[u])
{
float trust_v = (float) 1 / user_connections[v].Count;
float neg_trust_times_reg = -social_regularization * trust_v;
float bias_diff = 0;
var factor_diffs = new float[NumFactors];
foreach (int w in user_connections[v])
{
bias_diff -= user_bias[w];
for (int f = 0; f < factor_diffs.Length; f++)
factor_diffs[f] -= user_factors[w, f];
}
bias_diff *= trust_v; // normalize
bias_diff += user_bias[v];
user_bias_gradient[u] += neg_trust_times_reg * bias_diff;
for (int f = 0; f < factor_diffs.Length; f++)
{
factor_diffs[f] *= trust_v; // normalize
factor_diffs[f] += user_factors[v, f];
user_factors_gradient.Inc(u, f, neg_trust_times_reg * factor_diffs[f]);
}
}
}
// II. apply gradient descent step
for (int user_id = 0; user_id < user_factors_gradient.dim1; user_id++)
user_bias[user_id] -= user_bias_gradient[user_id] * LearnRate * BiasLearnRate;
for (int item_id = 0; item_id < item_factors_gradient.dim1; item_id++)
item_bias[item_id] -= item_bias_gradient[item_id] * LearnRate * BiasLearnRate;
user_factors_gradient.Multiply(-LearnRate);
user_factors.Inc(user_factors_gradient);
item_factors_gradient.Multiply(-LearnRate);
item_factors.Inc(item_factors_gradient);
}
private void Matrix4fSetRotationFromMatrix3f(ref Matrix transform, Matrix matrix)
{
float scale = transform.TempSVD();
transform.FromOtherMatrix(matrix, 3, 3);
transform.Multiply(scale, 3, 3);
}
public void TestMultiply()
{
var matrix = new Matrix<double>(5, 5);
double[] row = { 1, 2, 3, 4, 5 };
for (int i = 0; i < 5; i++)
matrix.SetRow(i, row);
matrix.Multiply(2.5);
double[] testrow = { 2.5, 5, 7.5, 10, 12.5 };
Assert.AreEqual(testrow, matrix.GetRow(3));
}
/// <summary>
/// After the forward and backward pass, the weights must be updated. This is done in this function.
/// </summary>
/// <param name="pat">input pattern</param>
/// <param name="dw">delta w, from last updates. Used to update weights1 after recalculation.</param>
/// <param name="dv">delta v, from last updates. Used to update weights2 after recalculation.</param>
private void weightUpdate(Matrix<float> pat, Matrix<float> dw, Matrix<float> dv, Matrix<float> targets)
{
/*
% weight update, MATLAB code
dw = (dw .* alpha) - (delta_h * pat') .* (1-alpha);
dv = (dv .* alpha) - (delta_o * hout') .* (1-alpha);
w = w + dw .* eta .* (1 + rand(1,1)/1000)';
v = v + dv .* eta .* (1 + rand(1,1)/1000)';
error(epoch) = sum(sum(abs(sign(out) - targets)./2));
*/
float alpha = 0.9f;
float eta = 0.1f;
dw = (dw.Multiply(alpha)).Subtract((net_deltah.Multiply(pat.Transpose())).Multiply(1 - alpha));
dv = (dv.Multiply(alpha)).Subtract((net_deltao.Multiply(net_hout.Transpose())).Multiply(1 - alpha));
weights1 += dw.Multiply(eta).Multiply(1 + DataManipulation.rand(1, 1)[0, 0] / 1000f);
weights2 += dv.Multiply(eta).Multiply(1 + DataManipulation.rand(1, 1)[0, 0] / 1000f);
Matrix<float> e1 = (MatrixSign(net_out) - targets).Multiply(0.5f).Multiply(new DenseMatrix(net_out.ColumnCount, 1, 1.0f)).Transpose().Multiply(new DenseMatrix(net_out.RowCount, 1, 1.0f));
double e = e1[0, 0];
e = e;
}
/// <summary>
/// return a * s
/// </summary>
/// <param name="a"></param>
/// <param name="s"></param>
/// <returns></returns>
public static Matrix Multiply(Matrix a, double s) {
a.ShouldNotBeNull("a");
var result = new Matrix(a.Rows, a.Cols);
result.Multiply(a, s);
return result;
}
/// <summary>
/// Solves the matrix equation Ax = b, where A is the coefficient matrix, b is the
/// solution vector and x is the unknown vector.
/// </summary>
/// <param name="matrix">The coefficient matrix, <c>A</c>.</param>
/// <param name="input">The solution vector, <c>b</c></param>
/// <param name="result">The result vector, <c>x</c></param>
public void Solve(Matrix matrix, Vector input, Vector result)
{
// If we were stopped before, we are no longer
// We're doing this at the start of the method to ensure
// that we can use these fields immediately.
_hasBeenStopped = false;
// Error checks
if (matrix == null)
{
throw new ArgumentNullException("matrix");
}
if (matrix.RowCount != matrix.ColumnCount)
{
throw new ArgumentException(Resources.ArgumentMatrixSquare, "matrix");
}
if (input == null)
{
throw new ArgumentNullException("input");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
if (result.Count != input.Count)
{
throw new ArgumentException(Resources.ArgumentVectorsSameLength);
}
if (input.Count != matrix.RowCount)
{
throw Matrix.DimensionsDontMatch<ArgumentException>(input, matrix);
}
// Initialize the solver fields
// Set the convergence monitor
if (_iterator == null)
{
_iterator = Iterator.CreateDefault();
}
if (_preconditioner == null)
{
_preconditioner = new UnitPreconditioner();
}
_preconditioner.Initialize(matrix);
// x_0 is initial guess
// Take x_0 = 0
Vector xtemp = new DenseVector(input.Count);
// r_0 = b - Ax_0
// This is basically a SAXPY so it could be made a lot faster
Vector residuals = new DenseVector(matrix.RowCount);
CalculateTrueResidual(matrix, residuals, xtemp, input);
// Define the temporary scalars
float beta = 0;
float sigma;
// Define the temporary vectors
// rDash_0 = r_0
Vector rdash = new DenseVector(residuals);
// t_-1 = 0
Vector t = new DenseVector(residuals.Count);
Vector t0 = new DenseVector(residuals.Count);
// w_-1 = 0
Vector w = new DenseVector(residuals.Count);
// Define the remaining temporary vectors
Vector c = new DenseVector(residuals.Count);
Vector p = new DenseVector(residuals.Count);
Vector s = new DenseVector(residuals.Count);
Vector u = new DenseVector(residuals.Count);
Vector y = new DenseVector(residuals.Count);
Vector z = new DenseVector(residuals.Count);
Vector temp = new DenseVector(residuals.Count);
Vector temp2 = new DenseVector(residuals.Count);
Vector temp3 = new DenseVector(residuals.Count);
// for (k = 0, 1, .... )
var iterationNumber = 0;
while (ShouldContinue(iterationNumber, xtemp, input, residuals))
{
// p_k = r_k + beta_(k-1) * (p_(k-1) - u_(k-1))
p.Subtract(u, temp);
temp.Multiply(beta, temp2);
residuals.Add(temp2, p);
// Solve M b_k = p_k
_preconditioner.Approximate(p, temp);
// s_k = A b_k
matrix.Multiply(temp, s);
// alpha_k = (r*_0 * r_k) / (r*_0 * s_k)
var alpha = rdash.DotProduct(residuals) / rdash.DotProduct(s);
// y_k = t_(k-1) - r_k - alpha_k * w_(k-1) + alpha_k s_k
s.Subtract(w, temp);
t.Subtract(residuals, y);
temp.Multiply(alpha, temp2);
y.Add(temp2, temp3);
temp3.CopyTo(y);
// Store the old value of t in t0
t.CopyTo(t0);
// t_k = r_k - alpha_k s_k
s.Multiply(-alpha, temp2);
residuals.Add(temp2, t);
// Solve M d_k = t_k
_preconditioner.Approximate(t, temp);
// c_k = A d_k
matrix.Multiply(temp, c);
var cdot = c.DotProduct(c);
// cDot can only be zero if c is a zero vector
// We'll set cDot to 1 if it is zero to prevent NaN's
// Note that the calculation should continue fine because
// c.DotProduct(t) will be zero and so will c.DotProduct(y)
if (cdot.AlmostEqual(0, 1))
{
cdot = 1.0f;
}
// Even if we don't want to do any BiCGStab steps we'll still have
// to do at least one at the start to initialize the
// system, but we'll only have to take special measures
// if we don't do any so ...
var ctdot = c.DotProduct(t);
float eta;
if (((_numberOfBiCgStabSteps == 0) && (iterationNumber == 0)) || ShouldRunBiCgStabSteps(iterationNumber))
{
// sigma_k = (c_k * t_k) / (c_k * c_k)
sigma = ctdot / cdot;
// eta_k = 0
eta = 0;
}
else
{
var ydot = y.DotProduct(y);
// yDot can only be zero if y is a zero vector
// We'll set yDot to 1 if it is zero to prevent NaN's
// Note that the calculation should continue fine because
// y.DotProduct(t) will be zero and so will c.DotProduct(y)
if (ydot.AlmostEqual(0, 1))
{
ydot = 1.0f;
}
var ytdot = y.DotProduct(t);
var cydot = c.DotProduct(y);
var denom = (cdot * ydot) - (cydot * cydot);
// sigma_k = ((y_k * y_k)(c_k * t_k) - (y_k * t_k)(c_k * y_k)) / ((c_k * c_k)(y_k * y_k) - (y_k * c_k)(c_k * y_k))
sigma = ((ydot * ctdot) - (ytdot * cydot)) / denom;
// eta_k = ((c_k * c_k)(y_k * t_k) - (y_k * c_k)(c_k * t_k)) / ((c_k * c_k)(y_k * y_k) - (y_k * c_k)(c_k * y_k))
eta = ((cdot * ytdot) - (cydot * ctdot)) / denom;
}
// u_k = sigma_k s_k + eta_k (t_(k-1) - r_k + beta_(k-1) u_(k-1))
u.Multiply(beta, temp2);
t0.Add(temp2, temp);
temp.Subtract(residuals, temp3);
temp3.CopyTo(temp);
temp.Multiply(eta, temp);
s.Multiply(sigma, temp2);
temp.Add(temp2, u);
// z_k = sigma_k r_k +_ eta_k z_(k-1) - alpha_k u_k
z.Multiply(eta, z);
u.Multiply(-alpha, temp2);
z.Add(temp2, temp3);
temp3.CopyTo(z);
residuals.Multiply(sigma, temp2);
z.Add(temp2, temp3);
temp3.CopyTo(z);
// x_(k+1) = x_k + alpha_k p_k + z_k
p.Multiply(alpha, temp2);
xtemp.Add(temp2, temp3);
temp3.CopyTo(xtemp);
xtemp.Add(z, temp3);
temp3.CopyTo(xtemp);
// r_(k+1) = t_k - eta_k y_k - sigma_k c_k
// Copy the old residuals to a temp vector because we'll
// need those in the next step
residuals.CopyTo(t0);
y.Multiply(-eta, temp2);
t.Add(temp2, residuals);
c.Multiply(-sigma, temp2);
residuals.Add(temp2, temp3);
temp3.CopyTo(residuals);
// beta_k = alpha_k / sigma_k * (r*_0 * r_(k+1)) / (r*_0 * r_k)
// But first we check if there is a possible NaN. If so just reset beta to zero.
beta = (!sigma.AlmostEqual(0, 1)) ? alpha / sigma * rdash.DotProduct(residuals) / rdash.DotProduct(t0) : 0;
// w_k = c_k + beta_k s_k
s.Multiply(beta, temp2);
c.Add(temp2, w);
// Get the real value
_preconditioner.Approximate(xtemp, result);
// Now check for convergence
if (!ShouldContinue(iterationNumber, result, input, residuals))
{
// Recalculate the residuals and go round again. This is done to ensure that
// we have the proper residuals.
CalculateTrueResidual(matrix, residuals, result, input);
}
// Next iteration.
iterationNumber++;
}
}
/// <summary>
/// Calculates the <c>true</c> residual of the matrix equation Ax = b according to: residual = b - Ax
/// </summary>
/// <param name="matrix">Instance of the <see cref="Matrix"/> A.</param>
/// <param name="residual">Residual values in <see cref="Vector"/>.</param>
/// <param name="x">Instance of the <see cref="Vector"/> x.</param>
/// <param name="b">Instance of the <see cref="Vector"/> b.</param>
private static void CalculateTrueResidual(Matrix matrix, Vector residual, Vector x, Vector b)
{
// -Ax = residual
matrix.Multiply(x, residual);
residual.Multiply(-1, residual);
// residual + b
residual.Add(b, residual);
}
/// <summary>
/// Solves the matrix equation Ax = b, where A is the coefficient matrix, b is the
/// solution vector and x is the unknown vector.
/// </summary>
/// <param name="matrix">The coefficient matrix, <c>A</c>.</param>
/// <param name="input">The solution vector, <c>b</c></param>
/// <param name="result">The result vector, <c>x</c></param>
public void Solve(Matrix matrix, Vector input, Vector result)
{
// If we were stopped before, we are no longer
// We're doing this at the start of the method to ensure
// that we can use these fields immediately.
_hasBeenStopped = false;
// Error checks
if (matrix == null)
{
throw new ArgumentNullException("matrix");
}
if (matrix.RowCount != matrix.ColumnCount)
{
throw new ArgumentException(Resources.ArgumentMatrixSquare, "matrix");
}
if (input == null)
{
throw new ArgumentNullException("input");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
if (result.Count != input.Count)
{
throw new ArgumentException(Resources.ArgumentVectorsSameLength);
}
if (input.Count != matrix.RowCount)
{
throw new ArgumentException(Resources.ArgumentMatrixDimensions);
}
// Initialize the solver fields
// Set the convergence monitor
if (_iterator == null)
{
_iterator = Iterator.CreateDefault();
}
if (_preconditioner == null)
{
_preconditioner = new UnitPreconditioner();
}
_preconditioner.Initialize(matrix);
// Choose an initial guess x_0
// Take x_0 = 0
Vector xtemp = new DenseVector(input.Count);
// Choose k vectors q_1, q_2, ..., q_k
// Build a new set if:
// a) the stored set doesn't exist (i.e. == null)
// b) Is of an incorrect length (i.e. too long)
// c) The vectors are of an incorrect length (i.e. too long or too short)
var useOld = false;
if (_startingVectors != null)
{
// We don't accept collections with zero starting vectors so ...
if (_startingVectors.Count <= NumberOfStartingVectorsToCreate(_numberOfStartingVectors, input.Count))
{
// Only check the first vector for sizing. If that matches we assume the
// other vectors match too. If they don't the process will crash
if (_startingVectors[0].Count == input.Count)
{
useOld = true;
}
}
}
_startingVectors = useOld ? _startingVectors : CreateStartingVectors(_numberOfStartingVectors, input.Count);
// Store the number of starting vectors. Not really necessary but easier to type :)
var k = _startingVectors.Count;
// r_0 = b - Ax_0
// This is basically a SAXPY so it could be made a lot faster
Vector residuals = new DenseVector(matrix.RowCount);
CalculateTrueResidual(matrix, residuals, xtemp, input);
// Define the temporary values
var c = new Complex[k];
// Define the temporary vectors
Vector gtemp = new DenseVector(residuals.Count);
Vector u = new DenseVector(residuals.Count);
Vector utemp = new DenseVector(residuals.Count);
Vector temp = new DenseVector(residuals.Count);
Vector temp1 = new DenseVector(residuals.Count);
Vector temp2 = new DenseVector(residuals.Count);
Vector zd = new DenseVector(residuals.Count);
Vector zg = new DenseVector(residuals.Count);
Vector zw = new DenseVector(residuals.Count);
var d = CreateVectorArray(_startingVectors.Count, residuals.Count);
// g_0 = r_0
var g = CreateVectorArray(_startingVectors.Count, residuals.Count);
residuals.CopyTo(g[k - 1]);
var w = CreateVectorArray(_startingVectors.Count, residuals.Count);
// FOR (j = 0, 1, 2 ....)
var iterationNumber = 0;
while (ShouldContinue(iterationNumber, xtemp, input, residuals))
{
// SOLVE M g~_((j-1)k+k) = g_((j-1)k+k)
_preconditioner.Approximate(g[k - 1], gtemp);
// w_((j-1)k+k) = A g~_((j-1)k+k)
matrix.Multiply(gtemp, w[k - 1]);
// c_((j-1)k+k) = q^T_1 w_((j-1)k+k)
c[k - 1] = _startingVectors[0].DotProduct(w[k - 1]);
if (c[k - 1].Real.AlmostEqual(0, 1) && c[k - 1].Imaginary.AlmostEqual(0, 1))
{
throw new Exception("Iterative solver experience a numerical break down");
}
// alpha_(jk+1) = q^T_1 r_((j-1)k+k) / c_((j-1)k+k)
var alpha = _startingVectors[0].DotProduct(residuals) / c[k - 1];
// u_(jk+1) = r_((j-1)k+k) - alpha_(jk+1) w_((j-1)k+k)
w[k - 1].Multiply(-alpha, temp);
residuals.Add(temp, u);
// SOLVE M u~_(jk+1) = u_(jk+1)
_preconditioner.Approximate(u, temp1);
temp1.CopyTo(utemp);
// rho_(j+1) = -u^t_(jk+1) A u~_(jk+1) / ||A u~_(jk+1)||^2
matrix.Multiply(temp1, temp);
var rho = temp.DotProduct(temp);
// If rho is zero then temp is a zero vector and we're probably
// about to have zero residuals (i.e. an exact solution).
// So set rho to 1.0 because in the next step it will turn to zero.
if (rho.Real.AlmostEqual(0, 1) && rho.Imaginary.AlmostEqual(0, 1))
{
rho = 1.0;
}
rho = -u.DotProduct(temp) / rho;
// r_(jk+1) = rho_(j+1) A u~_(jk+1) + u_(jk+1)
u.CopyTo(residuals);
// Reuse temp
temp.Multiply(rho, temp);
residuals.Add(temp, temp2);
temp2.CopyTo(residuals);
// x_(jk+1) = x_((j-1)k_k) - rho_(j+1) u~_(jk+1) + alpha_(jk+1) g~_((j-1)k+k)
utemp.Multiply(-rho, temp);
xtemp.Add(temp, temp2);
temp2.CopyTo(xtemp);
gtemp.Multiply(alpha, gtemp);
xtemp.Add(gtemp, temp2);
temp2.CopyTo(xtemp);
// Check convergence and stop if we are converged.
if (!ShouldContinue(iterationNumber, xtemp, input, residuals))
{
// Calculate the true residual
CalculateTrueResidual(matrix, residuals, xtemp, input);
// Now recheck the convergence
if (!ShouldContinue(iterationNumber, xtemp, input, residuals))
{
// We're all good now.
// Exit from the while loop.
break;
}
}
// FOR (i = 1,2, ...., k)
for (var i = 0; i < k; i++)
{
// z_d = u_(jk+1)
u.CopyTo(zd);
// z_g = r_(jk+i)
residuals.CopyTo(zg);
// z_w = 0
zw.Clear();
// FOR (s = i, ...., k-1) AND j >= 1
Complex beta;
if (iterationNumber >= 1)
{
for (var s = i; s < k - 1; s++)
{
// beta^(jk+i)_((j-1)k+s) = -q^t_(s+1) z_d / c_((j-1)k+s)
beta = -_startingVectors[s + 1].DotProduct(zd) / c[s];
// z_d = z_d + beta^(jk+i)_((j-1)k+s) d_((j-1)k+s)
d[s].Multiply(beta, temp);
zd.Add(temp, temp2);
temp2.CopyTo(zd);
// z_g = z_g + beta^(jk+i)_((j-1)k+s) g_((j-1)k+s)
g[s].Multiply(beta, temp);
zg.Add(temp, temp2);
temp2.CopyTo(zg);
// z_w = z_w + beta^(jk+i)_((j-1)k+s) w_((j-1)k+s)
w[s].Multiply(beta, temp);
zw.Add(temp, temp2);
temp2.CopyTo(zw);
}
}
beta = rho * c[k - 1];
if (beta.Real.AlmostEqual(0, 1) && beta.Imaginary.AlmostEqual(0, 1))
{
throw new Exception("Iterative solver experience a numerical break down");
}
// beta^(jk+i)_((j-1)k+k) = -(q^T_1 (r_(jk+1) + rho_(j+1) z_w)) / (rho_(j+1) c_((j-1)k+k))
zw.Multiply(rho, temp2);
residuals.Add(temp2, temp);
beta = -_startingVectors[0].DotProduct(temp) / beta;
// z_g = z_g + beta^(jk+i)_((j-1)k+k) g_((j-1)k+k)
g[k - 1].Multiply(beta, temp);
zg.Add(temp, temp2);
temp2.CopyTo(zg);
// z_w = rho_(j+1) (z_w + beta^(jk+i)_((j-1)k+k) w_((j-1)k+k))
w[k - 1].Multiply(beta, temp);
zw.Add(temp, temp2);
temp2.CopyTo(zw);
zw.Multiply(rho, zw);
// z_d = r_(jk+i) + z_w
residuals.Add(zw, zd);
// FOR (s = 1, ... i - 1)
for (var s = 0; s < i - 1; s++)
{
// beta^(jk+i)_(jk+s) = -q^T_s+1 z_d / c_(jk+s)
beta = -_startingVectors[s + 1].DotProduct(zd) / c[s];
// z_d = z_d + beta^(jk+i)_(jk+s) * d_(jk+s)
d[s].Multiply(beta, temp);
zd.Add(temp, temp2);
temp2.CopyTo(zd);
// z_g = z_g + beta^(jk+i)_(jk+s) * g_(jk+s)
g[s].Multiply(beta, temp);
zg.Add(temp, temp2);
temp2.CopyTo(zg);
}
// d_(jk+i) = z_d - u_(jk+i)
zd.Subtract(u, d[i]);
// g_(jk+i) = z_g + z_w
zg.Add(zw, g[i]);
// IF (i < k - 1)
if (i < k - 1)
{
// c_(jk+1) = q^T_i+1 d_(jk+i)
c[i] = _startingVectors[i + 1].DotProduct(d[i]);
if (c[i].Real.AlmostEqual(0, 1) && c[i].Imaginary.AlmostEqual(0, 1))
{
throw new Exception("Iterative solver experience a numerical break down");
}
// alpha_(jk+i+1) = q^T_(i+1) u_(jk+i) / c_(jk+i)
alpha = _startingVectors[i + 1].DotProduct(u) / c[i];
// u_(jk+i+1) = u_(jk+i) - alpha_(jk+i+1) d_(jk+i)
d[i].Multiply(-alpha, temp);
u.Add(temp, temp2);
temp2.CopyTo(u);
// SOLVE M g~_(jk+i) = g_(jk+i)
_preconditioner.Approximate(g[i], gtemp);
// x_(jk+i+1) = x_(jk+i) + rho_(j+1) alpha_(jk+i+1) g~_(jk+i)
gtemp.Multiply(rho * alpha, temp);
xtemp.Add(temp, temp2);
temp2.CopyTo(xtemp);
// w_(jk+i) = A g~_(jk+i)
matrix.Multiply(gtemp, w[i]);
// r_(jk+i+1) = r_(jk+i) - rho_(j+1) alpha_(jk+i+1) w_(jk+i)
w[i].Multiply(-rho * alpha, temp);
residuals.Add(temp, temp2);
temp2.CopyTo(residuals);
// We can check the residuals here if they're close
if (!ShouldContinue(iterationNumber, xtemp, input, residuals))
{
// Recalculate the residuals and go round again. This is done to ensure that
// we have the proper residuals.
CalculateTrueResidual(matrix, residuals, xtemp, input);
}
}
} // END ITERATION OVER i
iterationNumber++;
}
// copy the temporary result to the real result vector
xtemp.CopyTo(result);
}
/// <summary>
/// Calculates the <c>true</c> residual of the matrix equation Ax = b according to: residual = b - Ax
/// </summary>
/// <param name="matrix">Instance of the <see cref="Matrix"/> A.</param>
/// <param name="residual">Residual values in <see cref="Vector"/>.</param>
/// <param name="x">Instance of the <see cref="Vector"/> x.</param>
/// <param name="b">Instance of the <see cref="Vector"/> b.</param>
private static void CalculateTrueResidual(Matrix matrix, Vector residual, Vector x, Vector b)
{
// -Ax = residual
matrix.Multiply(x, residual);
// Do not use residual = residual.Negate() because it creates another object
residual.Multiply(-1, residual);
// residual + b
residual.Add(b, residual);
}
Vector mul_matrix(Matrix a, Vector b)
{
return a.Multiply(b);
}