private void InitializeLSTMCell(LSTMCell c, LSTMCellWeight cw, LSTMCellWeightDeri deri)
{
c.cellState = 0;
//partial derivatives
deri.dSWPeepholeIn = 0;
deri.dSWPeepholeForget = 0;
deri.dSWCellIn = 0;
deri.dSWCellForget = 0;
deri.dSWCellState = 0;
}
private void InitializeCellWeights(BinaryReader br)
{
CellWeights = new LSTMCellWeight[LayerSize];
CellWeightsDeri = new LSTMCellWeightDeri[LayerSize];
if (br != null)
{
//Load weight from input file
for (var i = 0; i < LayerSize; i++)
{
CellWeights[i] = new LSTMCellWeight();
CellWeights[i].wPeepholeIn = br.ReadDouble();
CellWeights[i].wPeepholeForget = br.ReadDouble();
CellWeights[i].wPeepholeOut = br.ReadDouble();
CellWeights[i].wCellIn = br.ReadDouble();
CellWeights[i].wCellForget = br.ReadDouble();
CellWeights[i].wCellState = br.ReadDouble();
CellWeights[i].wCellOut = br.ReadDouble();
CellWeightsDeri[i] = new LSTMCellWeightDeri();
}
}
else
{
//Initialize weight by random number
for (var i = 0; i < LayerSize; i++)
{
CellWeights[i] = new LSTMCellWeight();
//internal weights, also important
CellWeights[i].wPeepholeIn = RNNHelper.RandInitWeight();
CellWeights[i].wPeepholeForget = RNNHelper.RandInitWeight();
CellWeights[i].wPeepholeOut = RNNHelper.RandInitWeight();
CellWeights[i].wCellIn = RNNHelper.RandInitWeight();
CellWeights[i].wCellForget = RNNHelper.RandInitWeight();
CellWeights[i].wCellState = RNNHelper.RandInitWeight();
CellWeights[i].wCellOut = RNNHelper.RandInitWeight();
CellWeightsDeri[i] = new LSTMCellWeightDeri();
}
}
}
public override void InitializeInternalTrainingParameters()
{
if (SparseFeatureSize > 0)
{
sparseFeatureToHiddenDeri = new Dictionary <int, Vector3> [LayerSize];
for (var i = 0; i < LayerSize; i++)
{
sparseFeatureToHiddenDeri[i] = new Dictionary <int, Vector3>();
}
if (sparseFeatureLearningRate == null)
{
sparseFeatureLearningRate = new Vector4[LayerSize][];
sparseFeatureWeightsDelta = new Vector4[LayerSize][];
for (var i = 0; i < LayerSize; i++)
{
sparseFeatureLearningRate[i] = new Vector4[SparseFeatureSize];
sparseFeatureWeightsDelta[i] = new Vector4[SparseFeatureSize];
}
}
}
if (DenseFeatureSize > 0)
{
wDenseInputGate.InitInternaTrainingParameters();
wDenseForgetGate.InitInternaTrainingParameters();
wDenseCellGate.InitInternaTrainingParameters();
wDenseOutputGate.InitInternaTrainingParameters(false);
}
cellDelta = new Vector4[LayerSize];
peepholeDelta = new Vector3[LayerSize];
CellWeightsDeri = new LSTMCellWeightDeri[LayerSize];
for (var i = 0; i < LayerSize; i++)
{
CellWeightsDeri[i] = new LSTMCellWeightDeri();
}
}
public override void InitializeInternalTrainingParameters()
{
if (SparseFeatureSize > 0)
{
sparseFeatureToHiddenDeri = new Dictionary <int, Vector3> [LayerSize];
for (var i = 0; i < LayerSize; i++)
{
sparseFeatureToHiddenDeri[i] = new Dictionary <int, Vector3>();
}
}
if (sparseFeatureToHiddenLearningRate == null)
{
if (SparseFeatureSize > 0)
{
sparseFeatureToHiddenLearningRate = new Vector4[LayerSize][];
for (var i = 0; i < LayerSize; i++)
{
sparseFeatureToHiddenLearningRate[i] = new Vector4[SparseFeatureSize];
}
}
}
if (DenseFeatureSize > 0)
{
wDenseInputGate.InitInternaTrainingParameters();
wDenseForgetGate.InitInternaTrainingParameters();
wDenseCellGate.InitInternaTrainingParameters();
wDenseOutputGate.InitInternaTrainingParameters(false);
}
CellWeightsDeri = new LSTMCellWeightDeri[LayerSize];
for (var i = 0; i < LayerSize; i++)
{
CellWeightsDeri[i] = new LSTMCellWeightDeri();
}
if (lockerDenseFeature == null)
{
lockerDenseFeature = new object[LayerSize];
for (int i = 0; i < LayerSize; i++)
{
lockerDenseFeature[i] = new object();
}
}
if (lockerSparseFeature == null)
{
lockerSparseFeature = new object[LayerSize];
for (int i = 0; i < LayerSize; i++)
{
lockerSparseFeature[i] = new object();
}
}
if (cellLockers == null)
{
cellLockers = new object[LayerSize];
for (int i = 0; i < LayerSize; i++)
{
cellLockers[i] = new object();
}
}
}