private IVector MakeStep(IVector direction, INet net, double currentError)
{
const int maximumNumberOfTry = 50;
var numberOfTry = 0;
double error;
var step = direction.Mul(_alpha); ;
var oldWeights = _weights;
//can change alpha by minimizing it in f(xk + alpha*direction)
do
{
if (numberOfTry > maximumNumberOfTry || _alpha < 0.000000000001)
break;
_weights = oldWeights.Sum(step); //x(k+1) = xk + sk
net.SetWeights(_weights); //content of _weights now shared between net and _weights vector
error = GetBatchError(net);
_alpha /= 2.1;
step = direction.Mul(_alpha);
numberOfTry++;
} while (error > currentError);
if (numberOfTry > maximumNumberOfTry || _alpha < 0.000000000001)
{
Console.WriteLine("Simple step was performed. Too little alpha: {0:0.#############}.", _alpha);
//step = direction.Mul(_alpha);
_weights = oldWeights.Sum(direction.Mul(0.1));
net.SetWeights(_weights);
//AddLittleCorrectionToWeights(net.Layers);
//_gradient = null;
_alpha = 1.0;
return null;
}
_alpha = 1.0;
return step;
}
private IVector MakeStepGoldstein(IVector direction, INet net, double currentError, IVector gradient)
{
const int maximumNumberOfTry = 50;
var numberOfTry = 0;
double error;
var step = direction.Mul(_alpha); ;
var c = 0.1;
var p = _random.NextDouble()*0.3 + 0.3;
var oldWeights = _weights;
//can change alpha by minimizing it in f(xk + alpha*direction)
var threshold = 0.0;
do
{
if (numberOfTry > maximumNumberOfTry || _alpha < 0.00001)
break;
_weights = oldWeights.Sum(step); //x(k+1) = xk + sk
net.SetWeights(_weights); //content of _weights now shared between net and _weights vector
error = GetBatchError(net);
threshold = gradient.Mul(step).Mul(_alpha).Mul(c).GetMod().X;
_alpha *= p;
step = direction.Mul(_alpha);
numberOfTry++;
} while (error > currentError + threshold);
if (numberOfTry > maximumNumberOfTry || _alpha < 0.00001)
{
_weights = oldWeights;
net.SetWeights(_weights);
//will make gradient descent on next step
_gradient = null;
_b = Matrix.CreateI(net.WeightsCount, net.WeightsCount, () => new RealNumber(1), () => new RealNumber(0)); //b0
_alpha = 1.0;
Console.WriteLine("Reset matrix");
return null;
}
_alpha = 1.0;
return step;
}
private void ChangeAndSetWeights(IVector delta, INet net)
{
_weights = _weights.Sum(delta);
net.SetWeights(_weights);
}