protected tp.Matrix GetJacobiMatrix() { var j = new tp.Matrix(JacobiHeight, JacobiWidth); for (int row = 0; row < JacobiHeight; row++) for (int col = 0; col < JacobiWidth; col++) j[row, col] = this[row, col]; return j; }
private void LM() { Action step = null; // Step by Hessian: step = () => { Debug.WriteLine(lambda); SaveWeightsAndError(); var jacobi = GetJacobiMatrix(); var jacobiT = GetJacobiMatrix().Transpose(); var jTj = jacobiT * jacobi; var matrix2 = new tp.Matrix(jTj.Rows, jTj.Rows); for (int j = 0; j < matrix2.Rows; j++) matrix2[j, j] = lambda; var matrix3 = jTj + matrix2; bool inv = true; tp.Matrix hessInv = new ThirdParty.Matrix(); try { hessInv = matrix3.Inverse(); } catch { inv = false; } var a = LearningConnections.ItemArray; for (int idx = 0; idx < a.Length; idx++) { var lc = a[idx]; var bc = (IBackwardConnection)lc.Connection; var rule = (LMRule)lc.Rule; double gradient = bc.BackwardValues.AvgGradient; double nv = 0.0; int ccount = inv ? hessInv.Columns : 0; double vDelta = 0.0; if (inv) { for (int col = 0; col < ccount; col++) { nv += hessInv[idx, col]; } vDelta = nv * gradient; } else { //bc.Weight += gradient * 0.01; //next = step; return; } if (vDelta < -5.0) vDelta = -5.0; else if (vDelta > 5.0) vDelta = 5.0; bc.Weight += vDelta; } CalculateNewError(() => { double derror = error - lastError; if (error <= lastError || Math.Abs(derror) < 0.00000) { // dec lambda lambda /= delta; if (lambda < double.Epsilon) lambda = double.Epsilon; //Console.WriteLine("OK"); // GOTO 10 step(); return; } else { RestoreWeights(); //if (error > 0.01) //{ // var ac = LearningConnections.ItemArray; // for (int idx = 0; idx < ac.Length; idx++) // { // var lc = ac[idx]; // var bc = (IBackwardConnection)lc.Connection; // bc.Weight += bc.BackwardValues.AvgGradient * 0.01; // } //} lambda *= delta; if (lambda > 100) lambda = 100; // Retry: next = step; } }); }; step(); }