public override double[][][][] Regularize(double[][][][] matrix, int numExamples)
{
double multiplicator = regularizationParameter / numExamples;
double[][][][] resMatrix = MatOp.MultiplyByConst(matrix, multiplicator);
return(resMatrix);
}
public override void UpdateWeights(ILearnable learnableLayer, int iteration = 1)
{
// update all weights by stochastic gradient descent
double[][][][] weights = learnableLayer.Weights;
double[][][][] dWeights = learnableLayer.Dweights;
int lx0 = weights.Length;
int lx1 = weights[0].Length;
int lx2 = weights[0][0].Length;
int lx3 = weights[0][0][0].Length;
int index = ((AbstractLayer)learnableLayer).Index;
// if it is first iteration we will init fields
if (!m.ContainsKey(index))
{
m.Add(index, Utils.Init4dArr(lx0, lx1, lx2, lx3));
}
if (!r.ContainsKey(index))
{
r.Add(index, Utils.Init4dArr(lx0, lx1, lx2, lx3));
}
m[index] = MatOp.Add(MatOp.MultiplyByConst(dWeights, 1d - beta1), MatOp.MultiplyByConst(m[index], beta1));
r[index] = MatOp.Add(MatOp.MultiplyByConst(MatOp.Cwise(dWeights, dWeights), 1d - beta2), MatOp.MultiplyByConst(r[index], beta2));
double[][][][] mExt = MatOp.DivideByConst(m[index], 1d - Math.Pow(beta1, iteration));
double[][][][] rExt = MatOp.DivideByConst(r[index], 1d - Math.Pow(beta2, iteration));
var a = MatOp.MultiplyByConst(mExt, learningRate);
var b = MatOp.AddConst(MatOp.Sqrt(rExt), epsilon);
var c = MatOp.Mwise(a, b);
learnableLayer.Weights = MatOp.Substract(weights, c);
// update biases
double[] biases = learnableLayer.Biases;
double[] dBiases = learnableLayer.Dbiases;
for (int i = 0; i < biases.Length; i++)
{
biases[i] -= learningRate * dBiases[i];
}
}
