protected tp.Matrix GetJacobiMatrix()
{
var j = new tp.Matrix(JacobiHeight, JacobiWidth);
for (int row = 0; row < JacobiHeight; row++)
for (int col = 0; col < JacobiWidth; col++)
j[row, col] = this[row, col];
return j;
}
private void LM()
{
Action step = null;
// Step by Hessian:
step = () =>
{
Debug.WriteLine(lambda);
SaveWeightsAndError();
var jacobi = GetJacobiMatrix();
var jacobiT = GetJacobiMatrix().Transpose();
var jTj = jacobiT * jacobi;
var matrix2 = new tp.Matrix(jTj.Rows, jTj.Rows);
for (int j = 0; j < matrix2.Rows; j++) matrix2[j, j] = lambda;
var matrix3 = jTj + matrix2;
bool inv = true;
tp.Matrix hessInv = new ThirdParty.Matrix();
try
{
hessInv = matrix3.Inverse();
}
catch
{
inv = false;
}
var a = LearningConnections.ItemArray;
for (int idx = 0; idx < a.Length; idx++)
{
var lc = a[idx];
var bc = (IBackwardConnection)lc.Connection;
var rule = (LMRule)lc.Rule;
double gradient = bc.BackwardValues.AvgGradient;
double nv = 0.0;
int ccount = inv ? hessInv.Columns : 0;
double vDelta = 0.0;
if (inv)
{
for (int col = 0; col < ccount; col++)
{
nv += hessInv[idx, col];
}
vDelta = nv * gradient;
}
else
{
//bc.Weight += gradient * 0.01;
//next = step;
return;
}
if (vDelta < -5.0) vDelta = -5.0; else if (vDelta > 5.0) vDelta = 5.0;
bc.Weight += vDelta;
}
CalculateNewError(() =>
{
double derror = error - lastError;
if (error <= lastError || Math.Abs(derror) < 0.00000)
{
// dec lambda
lambda /= delta;
if (lambda < double.Epsilon) lambda = double.Epsilon;
//Console.WriteLine("OK");
// GOTO 10
step();
return;
}
else
{
RestoreWeights();
//if (error > 0.01)
//{
// var ac = LearningConnections.ItemArray;
// for (int idx = 0; idx < ac.Length; idx++)
// {
// var lc = ac[idx];
// var bc = (IBackwardConnection)lc.Connection;
// bc.Weight += bc.BackwardValues.AvgGradient * 0.01;
// }
//}
lambda *= delta;
if (lambda > 100) lambda = 100;
// Retry:
next = step;
}
});
};
step();
}