public override void Duplicate(Layer cloneLayer)
{
if (cloneLayer.LayerType != LayerType.BPTTLayer)
{
throw new Exception();
}
else if (((BPTTLayer)cloneLayer).Tick != Tick)
{
throw new Exception();
}
else
{
base.Duplicate(cloneLayer);
for (int hideLayerIndex = 0; hideLayerIndex < HideLayerList.Count; hideLayerIndex++)
{
((BPTTLayer)cloneLayer).HideLayerList[hideLayerIndex].LayerStroageMatrix = HideLayerList[hideLayerIndex].LayerStroageMatrix.Clone();
((BPTTLayer)cloneLayer).HideLayerList[hideLayerIndex].LayerActivationMatrix = HideLayerList[hideLayerIndex].LayerActivationMatrix.Clone();
((BPTTLayer)cloneLayer).HideLayerList[hideLayerIndex].LayerErrorMatrix = HideLayerList[hideLayerIndex].LayerErrorMatrix.Clone();
((BPTTLayer)cloneLayer).HideLayerList[hideLayerIndex].BiasMatrix = HideLayerList[hideLayerIndex].BiasMatrix.Clone();
((BPTTLayer)cloneLayer).HideLayerList[hideLayerIndex].InterConnectionMatrix = HideLayerList[hideLayerIndex].InterConnectionMatrix.Clone();
}
for (int hideConnectionIndex = 0; hideConnectionIndex < HideConnectionList.Count; hideConnectionIndex++)
{
((BPTTLayer)cloneLayer).HideConnectionList[hideConnectionIndex].WeightMatrix = HideConnectionList[hideConnectionIndex].WeightMatrix;
}
((BPTTLayer)cloneLayer).BaseHideWeightMatrix = BaseHideWeightMatrix.Clone();
((BPTTLayer)cloneLayer).BaseHideBiasMatrix = BaseHideBiasMatrix.Clone();
}
}
public override void Initialize(LayerState layerState, double damagedSD, int rowCount)
{
base.Initialize(layerState, damagedSD, rowCount);
CurrentTick = 0;
HideLayerList = new List <Layer>();
for (int layerIndex = 0; layerIndex <= Tick; layerIndex++)
{
Layer newLayer = new Layer(random, Name + "-Hide" + layerIndex, UnitCount, 0, 0);
newLayer.BiasMatrix = (Matrix <double>)BaseHideBiasMatrix.Clone();
newLayer.Initialize(layerState, damagedSD, rowCount);
HideLayerList.Add(newLayer);
}
HideConnectionList = new List <Connection>();
for (int connectionIndex = 0; connectionIndex < Tick; connectionIndex++)
{
Connection newConnection = new Connection(random, Name + "-Hide" + connectionIndex, HideLayerList[connectionIndex], HideLayerList[connectionIndex + 1], 0);
newConnection.WeightMatrix = (Matrix <double>)BaseHideWeightMatrix.Clone();
newConnection.Initialize(ConnectionState.On, 0);
HideConnectionList.Add(newConnection);
}
}
public override void Initialize(LayerState layerState, double damagedSD, int rowCount)
{
base.Initialize(layerState, damagedSD, rowCount);
CurrentTick = 0;
HideLayerList = new List <Layer>();
for (int layerIndex = 0; layerIndex <= Tick; layerIndex++)
{
Layer newLayer = new Layer(random, Name + "-Hide" + layerIndex, UnitCount, 0, 0);
newLayer.BiasMatrix = (Matrix <double>)BaseHideBiasMatrix.Clone();
newLayer.Initialize(layerState, damagedSD, rowCount);
HideLayerList.Add(newLayer);
}
HideBundleList = new List <Bundle>();
for (int bundleIndex = 0; bundleIndex < Tick; bundleIndex++)
{
Bundle newBundle = new Bundle(random, Name + "-Hide" + bundleIndex, HideLayerList[bundleIndex], HideLayerList[bundleIndex + 1], 0);
newBundle.WeightMatrix = (Matrix <double>)BaseHideWeightMatrix.Clone();
newBundle.Initialize(BundleState.On, 0);
HideBundleList.Add(newBundle);
}
}
public void WeightRenewal(double momentum, double learningRate, double decayRate)
{
base.ErrorCalculate_Sigmoid(momentum);
HideLayerList[CurrentTick].LayerErrorMatrix = (Matrix <double>)LayerErrorMatrix.Clone();
for (int i = CurrentTick - 1; i >= 0; i--)
{
HideLayerList[i].ErrorCalculate_Sigmoid(momentum); //Boden (2001)
}
Matrix <double> hideWeightDeltaMatrix = DenseMatrix.Create(UnitCount, UnitCount, 0);
Matrix <double> hideBiasDeltaMatrix = DenseMatrix.Create(1, UnitCount, 0);
foreach (Layer layer in HideLayerList)
{
hideBiasDeltaMatrix += layer.LayerErrorMatrix.ColumnSums().ToRowMatrix() * learningRate;
}
foreach (Connection conneciton in HideConnectionList)
{
hideWeightDeltaMatrix += conneciton.SendLayer.LayerActivationMatrix.Transpose() * conneciton.ReceiveLayer.LayerErrorMatrix * learningRate;
}
//for (int layerIndex = 0; layerIndex < CurrentTick - 1; layerIndex++)
//{
// hideWeightDeltaMatrix += HideLayerList[layerIndex].LayerActivationMatrix.Transpose() * HideLayerList[layerIndex + 1].LayerErrorMatrix * learningRate;
//}
BaseHideBiasMatrix += hideBiasDeltaMatrix / CurrentTick;
BaseHideWeightMatrix += hideWeightDeltaMatrix / CurrentTick;
if (decayRate > 0)
{
Parallel.For(0, BaseHideBiasMatrix.ColumnCount, columnIndex =>
{
if (BaseHideBiasMatrix[0, columnIndex] > 0)
{
BaseHideBiasMatrix[0, columnIndex] -= decayRate;
}
else if (BaseHideBiasMatrix[0, columnIndex] < 0)
{
BaseHideBiasMatrix[0, columnIndex] += decayRate;
}
});
BaseHideBiasMatrix.CoerceZero(decayRate);
Parallel.For(0, BaseHideWeightMatrix.RowCount, rowIndex =>
{
for (int columnIndex = 0; columnIndex < BaseHideWeightMatrix.ColumnCount; columnIndex++)
{
if (BaseHideWeightMatrix[rowIndex, columnIndex] > 0)
{
BaseHideWeightMatrix[rowIndex, columnIndex] -= decayRate;
}
else if (BaseHideWeightMatrix[rowIndex, columnIndex] < 0)
{
BaseHideWeightMatrix[rowIndex, columnIndex] += decayRate;
}
}
});
BaseHideWeightMatrix.CoerceZero(decayRate);
}
//BPTT Layer에 Sending하는 Layer 및 Connection의 값을 구하기 위함
Matrix <double> hideLayerErrorMatrixSum = DenseMatrix.Create(HideLayerList[0].LayerErrorMatrix.RowCount, UnitCount, 0);
for (int i = CurrentTick; i >= 0; i--)
{
hideLayerErrorMatrixSum += HideLayerList[i].LayerErrorMatrix;
}
LayerErrorMatrix = hideLayerErrorMatrixSum / CurrentTick;
}
