public void Minimize(IHypothesis hypothesis, Vector[] inputs, double[] outputs) { if (iterationCount < 1) return; // first iteration var prevGrad = hypothesis.BatchGradient(inputs, outputs); hypothesis.Weights = hypothesis.Weights - learningRate * prevGrad; // other iterations for (var dummy = 1; dummy < iterationCount; ++dummy) { // find gradient of cost(loss) function var grad = hypothesis.BatchGradient(inputs, outputs); // find Euclidean norm of delta of last two gradients var dGrad = (grad - prevGrad); var dGradNorm = dGrad.EuclideanNorm; // adopt adaptive learning rate var rate = learningRate * dGradNorm; // do gradient step hypothesis.Weights = hypothesis.Weights - (learningRate * dGradNorm) * grad; // store gradient thus we can easily find delta of gradient prevGrad = grad; } }
public void Minimize(IHypothesis hypothesis, Vector[] inputs, double[] outputs) { if (inputs.Length == 0) return; var inputCount = inputs.Length; var inputSize = inputs[0].Size; // calculate invert avg multiplier var contribution = 1.0d / (double)inputSize; // prevent to passing small inputs away without weigths have been changed var currentRefreshRate = System.Math.Min(refreshRate, inputCount); for (var dummy = 0; dummy < iterationCount; ++dummy) { var gradAcc = new Vector(inputSize); for (var i = 0; i < inputCount; ++i) { // find gradient of cost(loss) function gradAcc += hypothesis.Gradient(inputs[i], outputs[i]); if ((i % currentRefreshRate) == 0) { // do gradient step hypothesis.Weights = hypothesis.Weights - learningRate * contribution * gradAcc; // reset accumulator gradAcc.Transform(x => 0.0d); } } hypothesis.Weights = hypothesis.Weights - learningRate * contribution * gradAcc; } }