public virtual void Duplicate(Layer cloneLayer) { cloneLayer.LayerStroageMatrix = LayerStroageMatrix.Clone(); cloneLayer.LayerActivationMatrix = LayerActivationMatrix.Clone(); cloneLayer.LayerErrorMatrix = LayerErrorMatrix.Clone(); cloneLayer.BiasMatrix = BiasMatrix.Clone(); cloneLayer.InterConnectionMatrix = InterConnectionMatrix.Clone(); }
public virtual void BiasRenewal(double learningRate, double decayRate) { BiasMatrix += LayerErrorMatrix.ColumnSums().ToRowMatrix() * learningRate; if (decayRate > 0) { Parallel.For(0, BiasMatrix.ColumnCount, columnIndex => { if (BiasMatrix[0, columnIndex] > 0) { BiasMatrix[0, columnIndex] -= decayRate; } else if (BiasMatrix[0, columnIndex] < 0) { BiasMatrix[0, columnIndex] += decayRate; } }); BiasMatrix.CoerceZero(decayRate); } }
public void WeightRenewal(double momentum, double learningRate, double decayRate) { base.ErrorCalculate_Sigmoid(momentum); HideLayerList[CurrentTick].LayerErrorMatrix = (Matrix <double>)LayerErrorMatrix.Clone(); for (int i = CurrentTick - 1; i >= 0; i--) { HideLayerList[i].ErrorCalculate_Sigmoid(momentum); //Boden (2001) } Matrix <double> hideWeightDeltaMatrix = DenseMatrix.Create(UnitCount, UnitCount, 0); Matrix <double> hideBiasDeltaMatrix = DenseMatrix.Create(1, UnitCount, 0); foreach (Layer layer in HideLayerList) { hideBiasDeltaMatrix += layer.LayerErrorMatrix.ColumnSums().ToRowMatrix() * learningRate; } foreach (Connection conneciton in HideConnectionList) { hideWeightDeltaMatrix += conneciton.SendLayer.LayerActivationMatrix.Transpose() * conneciton.ReceiveLayer.LayerErrorMatrix * learningRate; } //for (int layerIndex = 0; layerIndex < CurrentTick - 1; layerIndex++) //{ // hideWeightDeltaMatrix += HideLayerList[layerIndex].LayerActivationMatrix.Transpose() * HideLayerList[layerIndex + 1].LayerErrorMatrix * learningRate; //} BaseHideBiasMatrix += hideBiasDeltaMatrix / CurrentTick; BaseHideWeightMatrix += hideWeightDeltaMatrix / CurrentTick; if (decayRate > 0) { Parallel.For(0, BaseHideBiasMatrix.ColumnCount, columnIndex => { if (BaseHideBiasMatrix[0, columnIndex] > 0) { BaseHideBiasMatrix[0, columnIndex] -= decayRate; } else if (BaseHideBiasMatrix[0, columnIndex] < 0) { BaseHideBiasMatrix[0, columnIndex] += decayRate; } }); BaseHideBiasMatrix.CoerceZero(decayRate); Parallel.For(0, BaseHideWeightMatrix.RowCount, rowIndex => { for (int columnIndex = 0; columnIndex < BaseHideWeightMatrix.ColumnCount; columnIndex++) { if (BaseHideWeightMatrix[rowIndex, columnIndex] > 0) { BaseHideWeightMatrix[rowIndex, columnIndex] -= decayRate; } else if (BaseHideWeightMatrix[rowIndex, columnIndex] < 0) { BaseHideWeightMatrix[rowIndex, columnIndex] += decayRate; } } }); BaseHideWeightMatrix.CoerceZero(decayRate); } //BPTT Layer에 Sending하는 Layer 및 Connection의 값을 구하기 위함 Matrix <double> hideLayerErrorMatrixSum = DenseMatrix.Create(HideLayerList[0].LayerErrorMatrix.RowCount, UnitCount, 0); for (int i = CurrentTick; i >= 0; i--) { hideLayerErrorMatrixSum += HideLayerList[i].LayerErrorMatrix; } LayerErrorMatrix = hideLayerErrorMatrixSum / CurrentTick; }