/// <summary>
/// Update weights
/// </summary>
public void UpdateWeights()
{
Vector <float> vecMiniBatchSize = new Vector <float>(RNNHelper.MiniBatchSize);
for (var i = 0; i < layerSize; i++)
{
var j = 0;
var weights_i = weights[i];
var weightsDelta_i = weightsDelta[i];
var learningrate_i = learningRate[i];
var moreItems = (denseFeatureSize % Vector <float> .Count);
while (j < denseFeatureSize - moreItems)
{
//Vectorize weights delta
Vector <float> vecDelta = new Vector <float>(weightsDelta_i, j);
Vector <float> .Zero.CopyTo(weightsDelta_i, j);
//Normalize weights delta
vecDelta = vecDelta / vecMiniBatchSize;
vecDelta = RNNHelper.NormalizeGradient(vecDelta);
//Get learning rate dymanticly
var wlr_i = new Vector <float>(learningrate_i, j);
var vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref wlr_i);
wlr_i.CopyTo(learningrate_i, j);
//Update weights
Vector <float> vecWeights = new Vector <float>(weights_i, j);
vecWeights += vecLearningRate * vecDelta;
vecWeights.CopyTo(weights_i, j);
j += Vector <float> .Count;
}
while (j < denseFeatureSize)
{
var delta = weightsDelta_i[j];
weightsDelta_i[j] = 0;
delta = delta / RNNHelper.MiniBatchSize;
delta = RNNHelper.NormalizeGradient(delta);
var wlr_i = learningrate_i[j];
var learningRate = ComputeLearningRate(delta, ref wlr_i);
learningrate_i[j] = wlr_i;
weights_i[j] += learningRate * delta;
j++;
}
}
}
private void learnBptt()
{
for (int step = 0; step < bptt + bptt_block - 2; step++)
{
if (null == bptt_inputs[step] && null == bptt_fea[step])
{
break;
}
var sparse = bptt_inputs[step];
var bptt_fea_step = bptt_fea[step];
var last_bptt_hidden = bptt_hidden[step + 1];
var last_last_bptt_hidden = bptt_hidden[step + 2];
Parallel.For(0, LayerSize, parallelOption, a =>
{
// compute hidden layer gradient
er[a] *= cellOutput[a] * (1 - cellOutput[a]);
//dense weight update fea->0
double[] vector_a = null;
double er2 = er[a];
Vector <double> vecErr = new Vector <double>(er2);
int i = 0;
if (DenseFeatureSize > 0)
{
vector_a = DenseWeightsDelta[a];
i = 0;
while (i < DenseFeatureSize - Vector <double> .Count)
{
Vector <double> v1 = new Vector <double>(bptt_fea_step, i);
Vector <double> v2 = new Vector <double>(vector_a, i);
v2 += vecErr * v1;
v2.CopyTo(vector_a, i);
i += Vector <double> .Count;
}
while (i < DenseFeatureSize)
{
vector_a[i] += er2 * bptt_fea_step[i];
i++;
}
}
if (SparseFeatureSize > 0)
{
//sparse weight update hidden->input
vector_a = SparseWeightsDelta[a];
for (i = 0; i < sparse.Count; i++)
{
var entry = sparse.GetEntry(i);
vector_a[entry.Key] += er2 * entry.Value;
}
}
//bptt weight update
vector_a = BpttWeightsDelta[a];
i = 0;
while (i < LayerSize - Vector <double> .Count)
{
Vector <double> v1 = new Vector <double>(previousCellOutput, i);
Vector <double> v2 = new Vector <double>(vector_a, i);
v2 += vecErr * v1;
v2.CopyTo(vector_a, i);
i += Vector <double> .Count;
}
while (i < LayerSize)
{
vector_a[i] += er2 * previousCellOutput[i];
i++;
}
});
//propagates errors hidden->input to the recurrent part
double[] previousHiddenErr = new double[LayerSize];
RNNHelper.matrixXvectorADDErr(previousHiddenErr, er, BpttWeights, LayerSize, LayerSize);
for (int a = 0; a < LayerSize; a++)
{
//propagate error from time T-n to T-n-1
er[a] = previousHiddenErr[a] + last_bptt_hidden.er[a];
}
if (step < bptt + bptt_block - 3)
{
for (int a = 0; a < LayerSize; a++)
{
cellOutput[a] = last_bptt_hidden.cellOutput[a];
previousCellOutput[a] = last_last_bptt_hidden.cellOutput[a];
}
}
}
//restore hidden layer after bptt
bptt_hidden[0].cellOutput.CopyTo(cellOutput, 0);
Parallel.For(0, LayerSize, parallelOption, b =>
{
double[] vector_b = null;
double[] vector_bf = null;
double[] vector_lr = null;
//Update bptt feature weights
vector_b = BpttWeights[b];
vector_bf = BpttWeightsDelta[b];
vector_lr = BpttWeightsLearningRate[b];
int i = 0;
while (i < LayerSize - Vector <double> .Count)
{
Vector <double> vecDelta = new Vector <double>(vector_bf, i);
Vector <double> vecLearningRateWeights = new Vector <double>(vector_lr, i);
Vector <double> vecB = new Vector <double>(vector_b, i);
//Normalize delta
vecDelta = RNNHelper.NormalizeGradient(vecDelta);
//Computing learning rate and update its weights
Vector <double> vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref vecLearningRateWeights);
vecLearningRateWeights.CopyTo(vector_lr, i);
//Update weights
vecB += vecLearningRate * vecDelta;
vecB.CopyTo(vector_b, i);
//Clean weights
Vector <double> .Zero.CopyTo(vector_bf, i);
i += Vector <double> .Count;
}
while (i < LayerSize)
{
double delta = RNNHelper.NormalizeGradient(vector_bf[i]);
double newLearningRate = RNNHelper.UpdateLearningRate(BpttWeightsLearningRate, b, i, delta);
vector_b[i] += newLearningRate * delta;
//Clean bptt weight error
vector_bf[i] = 0;
i++;
}
//Update dense feature weights
if (DenseFeatureSize > 0)
{
vector_b = DenseWeights[b];
vector_bf = DenseWeightsDelta[b];
vector_lr = DenseWeightsLearningRate[b];
i = 0;
while (i < DenseFeatureSize - Vector <double> .Count)
{
Vector <double> vecDelta = new Vector <double>(vector_bf, i);
Vector <double> vecLearningRateWeights = new Vector <double>(vector_lr, i);
Vector <double> vecB = new Vector <double>(vector_b, i);
//Normalize delta
vecDelta = RNNHelper.NormalizeGradient(vecDelta);
//Computing learning rate and update its weights
Vector <double> vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref vecLearningRateWeights);
vecLearningRateWeights.CopyTo(vector_lr, i);
//Update weights
vecB += vecLearningRate * vecDelta;
vecB.CopyTo(vector_b, i);
//Clean weights
vecDelta = Vector <double> .Zero;
vecDelta.CopyTo(vector_bf, i);
i += Vector <double> .Count;
}
while (i < DenseFeatureSize)
{
double delta = RNNHelper.NormalizeGradient(vector_bf[i]);
double newLearningRate = RNNHelper.UpdateLearningRate(DenseWeightsLearningRate, b, i, delta);
vector_b[i] += newLearningRate * delta;
//Clean dense feature weights error
vector_bf[i] = 0;
i++;
}
}
if (SparseFeatureSize > 0)
{
//Update sparse feature weights
vector_b = SparseWeights[b];
vector_bf = SparseWeightsDelta[b];
for (int step = 0; step < bptt + bptt_block - 2; step++)
{
var sparse = bptt_inputs[step];
if (sparse == null)
{
break;
}
for (i = 0; i < sparse.Count; i++)
{
int pos = sparse.GetEntry(i).Key;
double delta = RNNHelper.NormalizeGradient(vector_bf[pos]);
double newLearningRate = RNNHelper.UpdateLearningRate(SparseWeightsLearningRate, b, pos, delta);
vector_b[pos] += newLearningRate * delta;
//Clean sparse feature weight error
vector_bf[pos] = 0;
}
}
}
});
}
public virtual void UpdateWeights()
{
Vector <float> vecMiniBatchSize = new Vector <float>(RNNHelper.MiniBatchSize);
for (var i = 0; i < LayerSize; i++)
{
if (SparseFeatureSize > 0)
{
var sparseWeights_i = SparseWeights[i];
var sparseDelta_i = SparseWeightsDelta[i];
var sparseLearningRate_i = SparseWeightsLearningRate[i];
var j = 0;
var moreItems = (SparseFeatureSize % Vector <float> .Count);
while (j < SparseFeatureSize - moreItems)
{
Vector <float> vecDelta = new Vector <float>(sparseDelta_i, j);
Vector <float> .Zero.CopyTo(sparseDelta_i, j);
if (vecDelta != Vector <float> .Zero)
{
vecDelta = vecDelta / vecMiniBatchSize;
vecDelta = RNNHelper.NormalizeGradient(vecDelta);
var wlr_i = new Vector <float>(sparseLearningRate_i, j);
var vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref wlr_i);
wlr_i.CopyTo(sparseLearningRate_i, j);
vecDelta = vecLearningRate * vecDelta;
Vector <float> vecWeights = new Vector <float>(sparseWeights_i, j);
vecWeights += vecDelta;
vecWeights.CopyTo(sparseWeights_i, j);
}
j += Vector <float> .Count;
}
while (j < SparseFeatureSize)
{
var delta = sparseDelta_i[j];
sparseDelta_i[j] = 0;
delta = delta / RNNHelper.MiniBatchSize;
delta = RNNHelper.NormalizeGradient(delta);
var newLearningRate = RNNHelper.ComputeLearningRate(SparseWeightsLearningRate, i, j, delta);
sparseWeights_i[j] += newLearningRate * delta;
j++;
}
}
if (DenseFeatureSize > 0)
{
var denseWeights_i = DenseWeights[i];
var denseDelta_i = DenseWeightsDelta[i];
var denseLearningRate_i = DenseWeightsLearningRate[i];
var j = 0;
var moreItems = (DenseFeatureSize % Vector <float> .Count);
while (j < DenseFeatureSize - moreItems)
{
Vector <float> vecDelta = new Vector <float>(denseDelta_i, j);
Vector <float> .Zero.CopyTo(denseDelta_i, j);
vecDelta = vecDelta / vecMiniBatchSize;
vecDelta = RNNHelper.NormalizeGradient(vecDelta);
var wlr_i = new Vector <float>(denseLearningRate_i, j);
var vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref wlr_i);
wlr_i.CopyTo(denseLearningRate_i, j);
vecDelta = vecLearningRate * vecDelta;
Vector <float> vecWeights = new Vector <float>(denseWeights_i, j);
vecWeights += vecDelta;
vecWeights.CopyTo(denseWeights_i, j);
j += Vector <float> .Count;
}
while (j < DenseFeatureSize)
{
var delta = denseDelta_i[j];
denseDelta_i[j] = 0;
delta = delta / RNNHelper.MiniBatchSize;
delta = RNNHelper.NormalizeGradient(delta);
var newLearningRate = RNNHelper.ComputeLearningRate(DenseWeightsLearningRate, i, j, delta);
denseWeights_i[j] += newLearningRate * delta;
j++;
}
}
}
}