public void WeightRenewal(double momentum, double learningRate, double decayRate)
{
base.ErrorCalculate_Sigmoid(momentum);
HideLayerList[CurrentTick].LayerErrorMatrix = (Matrix <double>)LayerErrorMatrix.Clone();
for (int i = CurrentTick - 1; i >= 0; i--)
{
HideLayerList[i].ErrorCalculate_Sigmoid(momentum); //Boden (2001)
}
Matrix <double> hideWeightDeltaMatrix = DenseMatrix.Create(UnitCount, UnitCount, 0);
Matrix <double> hideBiasDeltaMatrix = DenseMatrix.Create(1, UnitCount, 0);
foreach (Layer layer in HideLayerList)
{
hideBiasDeltaMatrix += layer.LayerErrorMatrix.ColumnSums().ToRowMatrix() * learningRate;
}
foreach (Connection conneciton in HideConnectionList)
{
hideWeightDeltaMatrix += conneciton.SendLayer.LayerActivationMatrix.Transpose() * conneciton.ReceiveLayer.LayerErrorMatrix * learningRate;
}
//for (int layerIndex = 0; layerIndex < CurrentTick - 1; layerIndex++)
//{
// hideWeightDeltaMatrix += HideLayerList[layerIndex].LayerActivationMatrix.Transpose() * HideLayerList[layerIndex + 1].LayerErrorMatrix * learningRate;
//}
BaseHideBiasMatrix += hideBiasDeltaMatrix / CurrentTick;
BaseHideWeightMatrix += hideWeightDeltaMatrix / CurrentTick;
if (decayRate > 0)
{
Parallel.For(0, BaseHideBiasMatrix.ColumnCount, columnIndex =>
{
if (BaseHideBiasMatrix[0, columnIndex] > 0)
{
BaseHideBiasMatrix[0, columnIndex] -= decayRate;
}
else if (BaseHideBiasMatrix[0, columnIndex] < 0)
{
BaseHideBiasMatrix[0, columnIndex] += decayRate;
}
});
BaseHideBiasMatrix.CoerceZero(decayRate);
Parallel.For(0, BaseHideWeightMatrix.RowCount, rowIndex =>
{
for (int columnIndex = 0; columnIndex < BaseHideWeightMatrix.ColumnCount; columnIndex++)
{
if (BaseHideWeightMatrix[rowIndex, columnIndex] > 0)
{
BaseHideWeightMatrix[rowIndex, columnIndex] -= decayRate;
}
else if (BaseHideWeightMatrix[rowIndex, columnIndex] < 0)
{
BaseHideWeightMatrix[rowIndex, columnIndex] += decayRate;
}
}
});
BaseHideWeightMatrix.CoerceZero(decayRate);
}
//BPTT Layer에 Sending하는 Layer 및 Connection의 값을 구하기 위함
Matrix <double> hideLayerErrorMatrixSum = DenseMatrix.Create(HideLayerList[0].LayerErrorMatrix.RowCount, UnitCount, 0);
for (int i = CurrentTick; i >= 0; i--)
{
hideLayerErrorMatrixSum += HideLayerList[i].LayerErrorMatrix;
}
LayerErrorMatrix = hideLayerErrorMatrixSum / CurrentTick;
}
