public override double[][][][] BackwardPropagation(double[][][][] input, int startIndex = 0, int endIndex = 0)
{
// set indexes for propagating batch
int numSamples = 0;
if (startIndex == DEFAULT_INDEX && endIndex == startIndex)
{
startIndex = 0;
endIndex = numSamples;
}
else
{
numSamples = endIndex - startIndex;
}
int n = input.Length;
dWeights = MatOp.Dot(MatOp.Transpose(lastInput, startIndex, endIndex), input, divisor: n);
// regularization
if (regularizer != null)
{
// lambda/m * W
double[][][][] regMatrix = regularizer.Regularize(weights, numSamples);
// add regularization coef
dWeights = MatOp.Add(dWeights, regMatrix);
}
dBiases = MatOp.Mean(input);
double[][][][] result = MatOp.Dot(input, MatOp.Transpose(weights));
return(result);
}
public override double[][][][] Regularize(double[][][][] matrix, int numExamples)
{
double multiplicator = regularizationParameter / numExamples;
double[][][][] resMatrix = MatOp.MultiplyByConst(matrix, multiplicator);
return(resMatrix);
}
public override double[][][][] BackwardPropagation(double[][][][] input, int startIndex = 0, int endIndex = 0)
{
// set indexes for propagating batch
int numSamples = 0;
if (startIndex == DEFAULT_INDEX && endIndex == startIndex)
{
startIndex = 0;
endIndex = numSamples;
}
else
{
numSamples = endIndex - startIndex;
}
// softmax has prettier derivation so we need one step less
if (activationFunction.GetType() == typeof(Softmax))
{
return(input);
}
// value activations is overriden so be careful about it
double[][][][] currGrad = MatOp.Cwise(input, activationFunction.Derivate(activations));
return(currGrad);
}
public override double[][][][] BackwardPropagation(double[][][][] input, int startIndex, int endIndex)
{
int currImageCount = input.Length;
int currWidth = input[0][0].Length;
int currHeight = input[0][0][0].Length;
int channels = filters[0].Length;
int filtersCount = filters.Length;
int filterWidth = filters[0][0].Length;
int filterHeight = filters[0][0][0].Length;
int middleWidthOfFilter = filterWidth / 2;
int middleHeightOfFilter = filterHeight / 2;
double normValue = 1 / currImageCount;
double gradientValue;
int yOffsetMin, yOffsetMax;
double[][][][] imageGradient = Utils.Init4dArr(inputDimension.imageCount, inputDimension.depth,
inputDimension.width, inputDimension.height);
dWeights = Utils.Init4dArr(filtersCount, channels, filterWidth, filterHeight);
for (int i = 0; i < currImageCount; i++)
{
for (int convChannel = 0; convChannel < filtersCount; convChannel++)
{
for (int y = 0; y < currWidth; y++)
{
yOffsetMin = Math.Max(-y, -middleWidthOfFilter);
yOffsetMax = Math.Min(currWidth - y, middleWidthOfFilter + 1);
gradientValue = input[i][convChannel][0][y];
for (int y_off = yOffsetMin; y_off < yOffsetMax; y_off++)
{
for (int c_imgs = 0; c_imgs < channels; c_imgs++)
{
//imageGradient[i][c_imgs][imgY][imgX] += filters[c_imgs][convChannel][subY][subX] * gradientValue;
dWeights[convChannel][c_imgs][0][y] += (lastInput[i][c_imgs][0][y_off] * gradientValue) * normValue;
// regularization
if (regularizer != null)
{
// lambda/m * W
double[][][][] regMatrix = regularizer.Regularize(lastInput, currImageCount);
// add regularization coef
dWeights = MatOp.Add(dWeights, MatOp.Transpose(regMatrix));
}
}
}
}
}
}
dBiases = SumBiases(input, biases.Length, currImageCount);
dWeights = MatOp.Substract(dWeights, filters);
return(imageGradient);
}
public override void UpdateWeights(ILearnable learnableLayer, int iteration = 1)
{
// update all weights by stochastic gradient descent
double[][][][] weights = learnableLayer.Weights;
double[][][][] dWeights = learnableLayer.Dweights;
int lx0 = weights.Length;
int lx1 = weights[0].Length;
int lx2 = weights[0][0].Length;
int lx3 = weights[0][0][0].Length;
int index = ((AbstractLayer)learnableLayer).Index;
// if it is first iteration we will init fields
if (!m.ContainsKey(index))
{
m.Add(index, Utils.Init4dArr(lx0, lx1, lx2, lx3));
}
if (!r.ContainsKey(index))
{
r.Add(index, Utils.Init4dArr(lx0, lx1, lx2, lx3));
}
m[index] = MatOp.Add(MatOp.MultiplyByConst(dWeights, 1d - beta1), MatOp.MultiplyByConst(m[index], beta1));
r[index] = MatOp.Add(MatOp.MultiplyByConst(MatOp.Cwise(dWeights, dWeights), 1d - beta2), MatOp.MultiplyByConst(r[index], beta2));
double[][][][] mExt = MatOp.DivideByConst(m[index], 1d - Math.Pow(beta1, iteration));
double[][][][] rExt = MatOp.DivideByConst(r[index], 1d - Math.Pow(beta2, iteration));
var a = MatOp.MultiplyByConst(mExt, learningRate);
var b = MatOp.AddConst(MatOp.Sqrt(rExt), epsilon);
var c = MatOp.Mwise(a, b);
learnableLayer.Weights = MatOp.Substract(weights, c);
// update biases
double[] biases = learnableLayer.Biases;
double[] dBiases = learnableLayer.Dbiases;
for (int i = 0; i < biases.Length; i++)
{
biases[i] -= learningRate * dBiases[i];
}
}
public override double[][][][] BackwardPropagation(double[][][][] input, int startIndex, int endIndex)
{
int currImageCount = input.Length;
int currWidth = input[0][0].Length;
int currHeight = input[0][0][0].Length;
int filtersChannels = filters[0].Length;
int filtersCount = filters.Length;
int filterWidth = filters[0][0].Length;
int filterHeight = filters[0][0][0].Length;
int middleWidthOfFilter = filterWidth / 2;
int middleHeightOfFilter = filterHeight / 2;
double normValue = 1 / currImageCount;
double gradientValue;
int yOffsetMin, yOffsetMax, xOffsetMin, xOffsetMax, imgY, imgX, subY, subX;
double[][][][] imageGradient = Utils.Init4dArr(inputDimension.imageCount, inputDimension.depth,
inputDimension.width, inputDimension.height);
dWeights = Utils.Init4dArr(filtersCount, filtersChannels, filterWidth, filterHeight);
for (int i = 0; i < currImageCount; i++)
{
for (int convChannel = 0; convChannel < filtersChannels; convChannel++)
{
for (int y = 0; y < currWidth; y++)
{
yOffsetMin = Math.Max(-y, -middleWidthOfFilter);
yOffsetMax = Math.Min(currWidth - y, middleWidthOfFilter + 1);
for (int x = 0; x < currHeight; x++)
{
gradientValue = input[i][convChannel][x][y];
gradientValue = input[i][convChannel][x][y];
xOffsetMin = Math.Max(-x, -middleHeightOfFilter);
xOffsetMax = Math.Min(currHeight - x, middleHeightOfFilter + 1);
for (int verticalOffset = yOffsetMin; verticalOffset < yOffsetMax; verticalOffset++)
{
for (int horizontalOffset = xOffsetMin; horizontalOffset < xOffsetMax; horizontalOffset++)
{
imgY = y + verticalOffset;
imgX = x + horizontalOffset;
subY = middleHeightOfFilter + verticalOffset;
subX = middleWidthOfFilter + horizontalOffset;
for (int imageChannel = 0; imageChannel < filtersChannels; imageChannel++)
{
imageGradient[i][imageChannel][imgY][imgX] += filters[imageChannel][convChannel][subY][subX] * gradientValue;
dWeights[imageChannel][convChannel][subY][subX] += (lastInput[i][imageChannel][imgY][imgX] * gradientValue) * normValue;
//regularization
//http://cs231n.github.io/neural-networks-2/#reg それは必要ないですけれどもそれを読んで下さい
if (regularizer != null)
{
//lambda/m * W
//clean version with some method calls, both add the regularization coef
double[][][][] regMatrix = regularizer.Regularize(lastInput, currImageCount);
dWeights = MatOp.Add(dWeights, MatOp.Transpose(regMatrix));
/*
* //if you're going to use this version instead, transpose the regMatrix, also comment the regmatrix above
* regMatrix = MatOp.Transpose(regularizer.Regularize(lastInput, currImageCount));
* for (int dL0 = 0; dL0 < dWeights.Length; dL0++)
* {
* for (int dL1 = 0; dL1 < dWeights[0].Length; dL1++)
* {
* for (int dL2 = 0; dL2 < dWeights[0][0].Length; dL2++)
* {
* for (int dL3 = 0; dL3 < dWeights[0][0][0].Length; dL3++)
* {
* dWeights[dL0][dL1][dL2][dL3] = dWeights[dL0][dL1][dL2][dL3] + regMatrix[dL0][dL1][dL2][dL3];
* }
* }
* }
* }
* //*/
}
}
}
}
}
}
}
}
dBiases = SumBiases(input, biases.Length, currImageCount);
dWeights = MatOp.Substract(dWeights, filters);
return(imageGradient);
}
