A * X = B
/// <summary> /// Perform one iteration. /// </summary> /// public override void Iteration() { LUDecomposition decomposition = null; PreIteration(); _weights = NetworkCODEC.NetworkToArray(_network); IComputeJacobian j = new JacobianChainRule(_network, _indexableTraining); double sumOfSquaredErrors = j.Calculate(_weights); double sumOfSquaredWeights = CalculateSumOfSquaredWeights(); // this.setError(j.getError()); CalculateHessian(j.Jacobian, j.RowErrors); // Define the objective function // bayesian regularization objective function double objective = _beta*sumOfSquaredErrors + _alpha *sumOfSquaredWeights; double current = objective + 1.0d; // Start the main Levenberg-Macquardt method _lambda /= ScaleLambda; // We'll try to find a direction with less error // (or where the objective function is smaller) while ((current >= objective) && (_lambda < LambdaMax)) { _lambda *= ScaleLambda; // Update diagonal (Levenberg-Marquardt formula) for (int i = 0; i < _parametersLength; i++) { _hessian[i][i] = _diagonal[i] + (_lambda + _alpha); } // Decompose to solve the linear system decomposition = new LUDecomposition(_hessianMatrix); // Check if the Jacobian has become non-invertible if (!decomposition.IsNonsingular) { continue; } // Solve using LU (or SVD) decomposition _deltas = decomposition.Solve(_gradient); // Update weights using the calculated deltas sumOfSquaredWeights = UpdateWeights(); // Calculate the new error sumOfSquaredErrors = 0.0d; for (int i = 0; i < _trainingLength; i++) { _indexableTraining.GetRecord(i, _pair); IMLData actual = _network .Compute(_pair.Input); double e = _pair.Ideal[0] - actual[0]; sumOfSquaredErrors += e*e; } sumOfSquaredErrors /= 2.0d; // Update the objective function current = _beta*sumOfSquaredErrors + _alpha *sumOfSquaredWeights; // If the object function is bigger than before, the method // is tried again using a greater dumping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. _lambda /= ScaleLambda; if (_useBayesianRegularization && (decomposition != null)) { // Compute the trace for the inverse Hessian double trace = Trace(decomposition.Inverse()); // Poland update's formula: _gamma = _parametersLength - (_alpha*trace); _alpha = _parametersLength /(2.0d*sumOfSquaredWeights + trace); _beta = Math.Abs((_trainingLength - _gamma) /(2.0d*sumOfSquaredErrors)); } Error = sumOfSquaredErrors; PostIteration(); }