/// <summary> /// Perform one iteration. /// </summary> /// public override void Iteration() { LUDecomposition decomposition = null; PreIteration(); _weights = NetworkCODEC.NetworkToArray(_network); IComputeJacobian j = new JacobianChainRule(_network, _indexableTraining); double sumOfSquaredErrors = j.Calculate(_weights); double sumOfSquaredWeights = CalculateSumOfSquaredWeights(); // this.setError(j.getError()); CalculateHessian(j.Jacobian, j.RowErrors); // Define the objective function // bayesian regularization objective function double objective = _beta*sumOfSquaredErrors + _alpha *sumOfSquaredWeights; double current = objective + 1.0d; // Start the main Levenberg-Macquardt method _lambda /= ScaleLambda; // We'll try to find a direction with less error // (or where the objective function is smaller) while ((current >= objective) && (_lambda < LambdaMax)) { _lambda *= ScaleLambda; // Update diagonal (Levenberg-Marquardt formula) for (int i = 0; i < _parametersLength; i++) { _hessian[i][i] = _diagonal[i] + (_lambda + _alpha); } // Decompose to solve the linear system decomposition = new LUDecomposition(_hessianMatrix); // Check if the Jacobian has become non-invertible if (!decomposition.IsNonsingular) { continue; } // Solve using LU (or SVD) decomposition _deltas = decomposition.Solve(_gradient); // Update weights using the calculated deltas sumOfSquaredWeights = UpdateWeights(); // Calculate the new error sumOfSquaredErrors = 0.0d; for (int i = 0; i < _trainingLength; i++) { _indexableTraining.GetRecord(i, _pair); IMLData actual = _network .Compute(_pair.Input); double e = _pair.Ideal[0] - actual[0]; sumOfSquaredErrors += e*e; } sumOfSquaredErrors /= 2.0d; // Update the objective function current = _beta*sumOfSquaredErrors + _alpha *sumOfSquaredWeights; // If the object function is bigger than before, the method // is tried again using a greater dumping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. _lambda /= ScaleLambda; if (_useBayesianRegularization && (decomposition != null)) { // Compute the trace for the inverse Hessian double trace = Trace(decomposition.Inverse()); // Poland update's formula: _gamma = _parametersLength - (_alpha*trace); _alpha = _parametersLength /(2.0d*sumOfSquaredWeights + trace); _beta = Math.Abs((_trainingLength - _gamma) /(2.0d*sumOfSquaredErrors)); } Error = sumOfSquaredErrors; PostIteration(); }
public override void Iteration() { LUDecomposition decomposition = null; double num; double num2; double num3; double num4; int num5; int num6; IMLData data; double num7; double num8; if ((((uint) num4) | 15) != 0) { goto Label_0399; } goto Label_01C7; Label_0022: base.PostIteration(); if ((((uint) num5) & 0) == 0) { return; } goto Label_01C7; Label_0044: this._xd7d571ecee49d1e4 = Math.Abs((double) ((this._x8557b7ee760663f3 - this._xc7c4e9c099884228) / (2.0 * num))); Label_0073: this.Error = num; goto Label_0022; Label_00F1: if ((num4 >= num3) && (this._x3271cefb1a159639 < 1E+25)) { this._x3271cefb1a159639 *= 10.0; num5 = 0; Label_01EF: if (num5 < this._xe2982b936ae423cd) { this._xc410e3804222557a[num5][num5] = this._x3cb63876dda4b74a[num5] + (this._x3271cefb1a159639 + this._x6ad505c7ef981b0e); num5++; if ((((uint) num7) + ((uint) num3)) >= 0) { goto Label_01EF; } } else { if ((((uint) num7) + ((uint) num8)) > uint.MaxValue) { return; } decomposition = new LUDecomposition(this._x05fb16197e552de6); if (!decomposition.IsNonsingular) { goto Label_00F1; } this._xdadd8f92d75a3aba = decomposition.Solve(this._x878c4eb3cef19a5a); } if (((uint) num6) >= 0) { if (0 != 0) { goto Label_0340; } goto Label_01C7; } goto Label_029A; } this._x3271cefb1a159639 /= 10.0; if (this._x6c57e14b737e51f6 && (decomposition != null)) { num8 = Trace(decomposition.Inverse()); this._xc7c4e9c099884228 = this._xe2982b936ae423cd - (this._x6ad505c7ef981b0e * num8); if (((uint) num7) < 0) { goto Label_0044; } this._x6ad505c7ef981b0e = ((double) this._xe2982b936ae423cd) / ((2.0 * num2) + num8); if ((((uint) num4) | 0xfffffffe) != 0) { goto Label_0044; } goto Label_0022; } goto Label_0073; Label_0137: num7 = this._x61830ac74d65acc3.Ideal[0] - data[0]; num += num7 * num7; num6++; Label_0161: if (num6 < this._x8557b7ee760663f3) { this._xb12276308f0fa6d9.GetRecord((long) num6, this._x61830ac74d65acc3); data = this._x87a7fc6a72741c2e.Compute(this._x61830ac74d65acc3.Input); goto Label_0137; } num /= 2.0; num4 = (this._xd7d571ecee49d1e4 * num) + (this._x6ad505c7ef981b0e * num2); goto Label_00F1; Label_01C7: num2 = this.UpdateWeights(); num = 0.0; num6 = 0; goto Label_0161; Label_029A: if ((((uint) num7) | 0xff) == 0) { goto Label_0137; } this._x3271cefb1a159639 /= 10.0; goto Label_00F1; Label_0340: num4 = num3 + 1.0; if ((((uint) num4) - ((uint) num6)) >= 0) { goto Label_029A; } Label_0399: base.PreIteration(); this._x2f33d779e5a20b28 = NetworkCODEC.NetworkToArray(this._x87a7fc6a72741c2e); if ((((uint) num8) | 4) == 0) { goto Label_0022; } IComputeJacobian jacobian = new JacobianChainRule(this._x87a7fc6a72741c2e, this._xb12276308f0fa6d9); num = jacobian.Calculate(this._x2f33d779e5a20b28); num2 = this.x01818299df58497e(); this.CalculateHessian(jacobian.Jacobian, jacobian.RowErrors); num3 = (this._xd7d571ecee49d1e4 * num) + (this._x6ad505c7ef981b0e * num2); goto Label_0340; }
/// <summary> /// Perform one iteration. /// </summary> /// public override void Iteration() { LUDecomposition decomposition = null; PreIteration(); _weights = NetworkCODEC.NetworkToArray(_network); IComputeJacobian j = new JacobianChainRule(_network, _indexableTraining); double sumOfSquaredErrors = j.Calculate(_weights); double sumOfSquaredWeights = CalculateSumOfSquaredWeights(); // this.setError(j.getError()); CalculateHessian(j.Jacobian, j.RowErrors); // Define the objective function // bayesian regularization objective function double objective = _beta * sumOfSquaredErrors + _alpha * sumOfSquaredWeights; double current = objective + 1.0d; // Start the main Levenberg-Macquardt method _lambda /= ScaleLambda; // We'll try to find a direction with less error // (or where the objective function is smaller) while ((current >= objective) && (_lambda < LambdaMax)) { _lambda *= ScaleLambda; // Update diagonal (Levenberg-Marquardt formula) for (int i = 0; i < _parametersLength; i++) { _hessian[i][i] = _diagonal[i] + (_lambda + _alpha); } // Decompose to solve the linear system decomposition = new LUDecomposition(_hessianMatrix); // Check if the Jacobian has become non-invertible if (!decomposition.IsNonsingular) { continue; } // Solve using LU (or SVD) decomposition _deltas = decomposition.Solve(_gradient); // Update weights using the calculated deltas sumOfSquaredWeights = UpdateWeights(); // Calculate the new error sumOfSquaredErrors = 0.0d; for (int i = 0; i < _trainingLength; i++) { _indexableTraining.GetRecord(i, _pair); IMLData actual = _network .Compute(_pair.Input); double e = _pair.Ideal[0] - actual[0]; sumOfSquaredErrors += e * e; } sumOfSquaredErrors /= 2.0d; // Update the objective function current = _beta * sumOfSquaredErrors + _alpha * sumOfSquaredWeights; // If the object function is bigger than before, the method // is tried again using a greater dumping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. _lambda /= ScaleLambda; if (_useBayesianRegularization && (decomposition != null)) { // Compute the trace for the inverse Hessian double trace = Trace(decomposition.Inverse()); // Poland update's formula: _gamma = _parametersLength - (_alpha * trace); _alpha = _parametersLength / (2.0d * sumOfSquaredWeights + trace); _beta = Math.Abs((_trainingLength - _gamma) / (2.0d * sumOfSquaredErrors)); } Error = sumOfSquaredErrors; PostIteration(); }