public virtual void ComputeLayerErr(SimpleLayer nextLayer, float[] destErrLayer, float[] srcErrLayer)
{
var sampledSoftmaxLayer = nextLayer as SampledSoftmaxLayer;
if (sampledSoftmaxLayer != null)
{
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, sampledSoftmaxLayer.DenseWeights, LayerSize,
sampledSoftmaxLayer.negativeSampleWordList);
}
else
{
var lstmLayer = nextLayer as LSTMLayer;
if (lstmLayer != null)
{
for (var i = 0; i < LayerSize; i++)
{
var err = 0.0f;
for (var k = 0; k < nextLayer.LayerSize; k++)
{
err += srcErrLayer[k] * lstmLayer.wDenseOutputGate.weights[k][i];
}
destErrLayer[i] = err; // RNNHelper.NormalizeGradient(err);
}
}
else
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize,
nextLayer.LayerSize);
}
}
}
public override void ComputeLayerErr(SimpleLayer nextLayer, double[] destErrLayer, double[] srcErrLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
for (int i = 0; i < LayerSize; i++)
{
if (mask[i] == true)
{
destErrLayer[i] = 0;
}
}
}
public override void ComputeLayerErr(SimpleLayer nextLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(er, nextLayer.er, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
//Apply drop out on error in hidden layer
for (int i = 0; i < LayerSize; i++)
{
if (mask[i] == true)
{
er[i] = 0;
}
}
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer)
{
NCEOutputLayer largeOutputLayer = nextLayer as NCEOutputLayer;
if (largeOutputLayer != null)
{
RNNHelper.matrixXvectorADDErr(er, largeOutputLayer.er, largeOutputLayer.DenseWeights, LayerSize, largeOutputLayer.negativeSampleWordList);
}
else
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(er, nextLayer.er, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
}
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer, float[] destErrLayer, float[] srcErrLayer)
{
var sampledSoftmaxLayer = nextLayer as SampledSoftmaxLayer;
if (sampledSoftmaxLayer != null)
{
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, sampledSoftmaxLayer.DenseWeights, LayerSize,
sampledSoftmaxLayer.negativeSampleWordList);
}
else
{
var lstmLayer = nextLayer as LSTMLayer;
if (lstmLayer != null)
{
Array.Clear(destErrLayer, 0, destErrLayer.Length);
for (var k = 0; k < nextLayer.LayerSize; k++)
{
int i = 0;
float[] weights = lstmLayer.wDenseOutputGate.weights[k];
float err = srcErrLayer[k];
var moreItems = (LayerSize % Vector <float> .Count);
while (i < LayerSize - moreItems)
{
Vector <float> vecWeights = new Vector <float>(weights, i);
Vector <float> vecErrs = new Vector <float>(destErrLayer, i);
vecErrs += err * vecWeights;
vecErrs.CopyTo(destErrLayer, i);
i += Vector <float> .Count;
}
while (i < LayerSize)
{
destErrLayer[i] += err * weights[i];
i++;
}
}
}
else
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize,
nextLayer.LayerSize);
}
}
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer, double[] destErrLayer, double[] srcErrLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
if (Dropout > 0)
{
//Apply drop out on error in hidden layer
for (int i = 0; i < LayerSize; i++)
{
if (mask[i] == true)
{
destErrLayer[i] = 0;
}
}
}
}
private void learnBptt()
{
for (int step = 0; step < bptt + bptt_block - 2; step++)
{
if (null == bptt_inputs[step] && null == bptt_fea[step])
{
break;
}
var sparse = bptt_inputs[step];
var bptt_fea_step = bptt_fea[step];
var last_bptt_hidden = bptt_hidden[step + 1];
var last_last_bptt_hidden = bptt_hidden[step + 2];
Parallel.For(0, LayerSize, parallelOption, a =>
{
// compute hidden layer gradient
er[a] *= cellOutput[a] * (1 - cellOutput[a]);
//dense weight update fea->0
double[] vector_a = null;
double er2 = er[a];
Vector <double> vecErr = new Vector <double>(er2);
int i = 0;
if (DenseFeatureSize > 0)
{
vector_a = DenseWeightsDelta[a];
i = 0;
while (i < DenseFeatureSize - Vector <double> .Count)
{
Vector <double> v1 = new Vector <double>(bptt_fea_step, i);
Vector <double> v2 = new Vector <double>(vector_a, i);
v2 += vecErr * v1;
v2.CopyTo(vector_a, i);
i += Vector <double> .Count;
}
while (i < DenseFeatureSize)
{
vector_a[i] += er2 * bptt_fea_step[i];
i++;
}
}
if (SparseFeatureSize > 0)
{
//sparse weight update hidden->input
vector_a = SparseWeightsDelta[a];
for (i = 0; i < sparse.Count; i++)
{
var entry = sparse.GetEntry(i);
vector_a[entry.Key] += er2 * entry.Value;
}
}
//bptt weight update
vector_a = BpttWeightsDelta[a];
i = 0;
while (i < LayerSize - Vector <double> .Count)
{
Vector <double> v1 = new Vector <double>(previousCellOutput, i);
Vector <double> v2 = new Vector <double>(vector_a, i);
v2 += vecErr * v1;
v2.CopyTo(vector_a, i);
i += Vector <double> .Count;
}
while (i < LayerSize)
{
vector_a[i] += er2 * previousCellOutput[i];
i++;
}
});
//propagates errors hidden->input to the recurrent part
double[] previousHiddenErr = new double[LayerSize];
RNNHelper.matrixXvectorADDErr(previousHiddenErr, er, BpttWeights, LayerSize, LayerSize);
for (int a = 0; a < LayerSize; a++)
{
//propagate error from time T-n to T-n-1
er[a] = previousHiddenErr[a] + last_bptt_hidden.er[a];
}
if (step < bptt + bptt_block - 3)
{
for (int a = 0; a < LayerSize; a++)
{
cellOutput[a] = last_bptt_hidden.cellOutput[a];
previousCellOutput[a] = last_last_bptt_hidden.cellOutput[a];
}
}
}
//restore hidden layer after bptt
bptt_hidden[0].cellOutput.CopyTo(cellOutput, 0);
Parallel.For(0, LayerSize, parallelOption, b =>
{
double[] vector_b = null;
double[] vector_bf = null;
double[] vector_lr = null;
//Update bptt feature weights
vector_b = BpttWeights[b];
vector_bf = BpttWeightsDelta[b];
vector_lr = BpttWeightsLearningRate[b];
int i = 0;
while (i < LayerSize - Vector <double> .Count)
{
Vector <double> vecDelta = new Vector <double>(vector_bf, i);
Vector <double> vecLearningRateWeights = new Vector <double>(vector_lr, i);
Vector <double> vecB = new Vector <double>(vector_b, i);
//Normalize delta
vecDelta = RNNHelper.NormalizeGradient(vecDelta);
//Computing learning rate and update its weights
Vector <double> vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref vecLearningRateWeights);
vecLearningRateWeights.CopyTo(vector_lr, i);
//Update weights
vecB += vecLearningRate * vecDelta;
vecB.CopyTo(vector_b, i);
//Clean weights
Vector <double> .Zero.CopyTo(vector_bf, i);
i += Vector <double> .Count;
}
while (i < LayerSize)
{
double delta = RNNHelper.NormalizeGradient(vector_bf[i]);
double newLearningRate = RNNHelper.UpdateLearningRate(BpttWeightsLearningRate, b, i, delta);
vector_b[i] += newLearningRate * delta;
//Clean bptt weight error
vector_bf[i] = 0;
i++;
}
//Update dense feature weights
if (DenseFeatureSize > 0)
{
vector_b = DenseWeights[b];
vector_bf = DenseWeightsDelta[b];
vector_lr = DenseWeightsLearningRate[b];
i = 0;
while (i < DenseFeatureSize - Vector <double> .Count)
{
Vector <double> vecDelta = new Vector <double>(vector_bf, i);
Vector <double> vecLearningRateWeights = new Vector <double>(vector_lr, i);
Vector <double> vecB = new Vector <double>(vector_b, i);
//Normalize delta
vecDelta = RNNHelper.NormalizeGradient(vecDelta);
//Computing learning rate and update its weights
Vector <double> vecLearningRate = RNNHelper.ComputeLearningRate(vecDelta, ref vecLearningRateWeights);
vecLearningRateWeights.CopyTo(vector_lr, i);
//Update weights
vecB += vecLearningRate * vecDelta;
vecB.CopyTo(vector_b, i);
//Clean weights
vecDelta = Vector <double> .Zero;
vecDelta.CopyTo(vector_bf, i);
i += Vector <double> .Count;
}
while (i < DenseFeatureSize)
{
double delta = RNNHelper.NormalizeGradient(vector_bf[i]);
double newLearningRate = RNNHelper.UpdateLearningRate(DenseWeightsLearningRate, b, i, delta);
vector_b[i] += newLearningRate * delta;
//Clean dense feature weights error
vector_bf[i] = 0;
i++;
}
}
if (SparseFeatureSize > 0)
{
//Update sparse feature weights
vector_b = SparseWeights[b];
vector_bf = SparseWeightsDelta[b];
for (int step = 0; step < bptt + bptt_block - 2; step++)
{
var sparse = bptt_inputs[step];
if (sparse == null)
{
break;
}
for (i = 0; i < sparse.Count; i++)
{
int pos = sparse.GetEntry(i).Key;
double delta = RNNHelper.NormalizeGradient(vector_bf[pos]);
double newLearningRate = RNNHelper.UpdateLearningRate(SparseWeightsLearningRate, b, pos, delta);
vector_b[pos] += newLearningRate * delta;
//Clean sparse feature weight error
vector_bf[pos] = 0;
}
}
}
});
}
public override void ComputeLayerErr(SimpleLayer nextLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(er, nextLayer.er, nextLayer.DenseWeights, negativeSampleWordList, nextLayer.LayerSize);
}
public virtual void ComputeLayerErr(List <float[]> destErrsList, bool cleanDest = true)
{
RNNHelper.matrixXvectorADDErr(destErrsList, Errs, DenseWeights, cleanDest);
}
public override void ComputeLayerErr(float[] destErrs, bool cleanDest = true)
{
RNNHelper.matrixXvectorADDErr(destErrs, Errs, DenseWeights, negativeSampleWordList, cleanDest);
}
