public SimpleLayer CloneHiddenLayer()
{
SimpleLayer m = new SimpleLayer(LayerSize);
int j = 0;
while (j < LayerSize - Vector <double> .Count)
{
Vector <double> vCellOutput = new Vector <double>(cellOutput, j);
vCellOutput.CopyTo(m.cellOutput, j);
Vector <double> vEr = new Vector <double>(er, j);
vEr.CopyTo(m.er, j);
j += Vector <double> .Count;
}
while (j < LayerSize)
{
m.cellOutput[j] = cellOutput[j];
m.er[j] = er[j];
j++;
}
return(m);
}
public static SimpleLayer Load(BinaryReader br, LayerType layerType)
{
LayerConfig config = new LayerConfig();
config.LayerSize = br.ReadInt32();
config.LayerType = layerType;
SimpleLayer layer = new SimpleLayer(config);
layer.SparseFeatureSize = br.ReadInt32();
layer.DenseFeatureSize = br.ReadInt32();
if (layer.SparseFeatureSize > 0)
{
Logger.WriteLine("Loading sparse feature weights...");
layer.SparseWeights = RNNHelper.LoadMatrix(br);
}
if (layer.DenseFeatureSize > 0)
{
Logger.WriteLine("Loading dense feature weights...");
layer.DenseWeights = RNNHelper.LoadMatrix(br);
}
return(layer);
}
public ForwardRNN(List <SimpleLayer> hiddenLayerList, int outputLayerSize)
{
HiddenLayerList = hiddenLayerList;
OutputLayer = new SimpleLayer(outputLayerSize);
OutputLayer.InitializeWeights(0, HiddenLayerList[HiddenLayerList.Count - 1].LayerSize);
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer, float[] destErrLayer, float[] srcErrLayer)
{
var sampledSoftmaxLayer = nextLayer as SampledSoftmaxLayer;
if (sampledSoftmaxLayer != null)
{
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, sampledSoftmaxLayer.DenseWeights, LayerSize,
sampledSoftmaxLayer.negativeSampleWordList);
}
else
{
var lstmLayer = nextLayer as LSTMLayer;
if (lstmLayer != null)
{
for (var i = 0; i < LayerSize; i++)
{
var err = 0.0f;
for (var k = 0; k < nextLayer.LayerSize; k++)
{
err += srcErrLayer[k] * lstmLayer.wDenseOutputGate.weights[k][i];
}
destErrLayer[i] = err; // RNNHelper.NormalizeGradient(err);
}
}
else
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize,
nextLayer.LayerSize);
}
}
}
public virtual SimpleLayer CreateLayerSharedWegiths()
{
SimpleLayer layer = new SimpleLayer(LayerConfig);
ShallowCopyWeightTo(layer);
return(layer);
}
public void matrixXvectorADD(SimpleLayer dest, SimpleCell[] srcvec, Matrix <double> srcmatrix, int DestSize, int SrcSize, int type)
{
if (type == 0)
{
//ac mod
Parallel.For(0, DestSize, parallelOption, i =>
{
double[] vector_i = srcmatrix[i];
double cellOutput = 0;
for (int j = 0; j < SrcSize; j++)
{
cellOutput += srcvec[j].cellOutput * vector_i[j];
}
dest.cellOutput[i] = cellOutput;
});
}
else
{
Parallel.For(0, DestSize, parallelOption, i =>
{
double er = 0;
for (int j = 0; j < SrcSize; j++)
{
er += srcvec[j].er * srcmatrix[j][i];
}
dest.er[i] = NormalizeGradient(er);
});
}
}
private SimpleLayer[] ComputeTopLayer(SimpleLayer[] lastLayer, out Matrix <double> rawOutputLayer, bool isTrain)
{
int numStates = lastLayer.Length;
//Calculate output layer
Matrix <double> tmp_rawOutputLayer = new Matrix <double>(numStates, OutputLayer.LayerSize);
SimpleLayer[] seqFinalOutput = new SimpleLayer[numStates];
Parallel.For(0, numStates, parallelOption, curState =>
{
seqFinalOutput[curState] = new SimpleLayer(OutputLayer.LayerSize);
SimpleLayer outputCells = seqFinalOutput[curState];
outputCells.DenseWeights = OutputLayer.DenseWeights;
outputCells.DenseWeightsLearningRate = OutputLayer.DenseWeightsLearningRate;
outputCells.DenseFeatureSize = OutputLayer.DenseFeatureSize;
outputCells.computeLayer(null, lastLayer[curState].cellOutput, isTrain);
outputCells.cellOutput.CopyTo(tmp_rawOutputLayer[curState], 0);
Softmax(outputCells);
});
rawOutputLayer = tmp_rawOutputLayer;
return(seqFinalOutput);
}
public static DropoutLayer Load(BinaryReader br, LayerType layerType)
{
DropoutLayer dropoutLayer;
DropoutLayerConfig config = new DropoutLayerConfig();
SimpleLayer simpleLayer = SimpleLayer.Load(br, layerType);
config.DropoutRatio = br.ReadSingle();
config.LayerSize = simpleLayer.LayerSize;
dropoutLayer = new DropoutLayer(config);
dropoutLayer.SparseFeatureSize = simpleLayer.SparseFeatureSize;
dropoutLayer.DenseFeatureSize = simpleLayer.DenseFeatureSize;
if (dropoutLayer.SparseFeatureSize > 0)
{
dropoutLayer.SparseWeights = simpleLayer.SparseWeights;
}
if (dropoutLayer.DenseFeatureSize > 0)
{
dropoutLayer.DenseWeights = simpleLayer.DenseWeights;
}
return(dropoutLayer);
}
public SimpleLayer CloneHiddenLayer()
{
var m = new SimpleLayer(LayerConfig);
var j = 0;
while (j < LayerSize - Vector <float> .Count)
{
var vCellOutput = new Vector <float>(Cell, j);
vCellOutput.CopyTo(m.Cell, j);
var vEr = new Vector <float>(Err, j);
vEr.CopyTo(m.Err, j);
j += Vector <float> .Count;
}
while (j < LayerSize)
{
m.Cell[j] = Cell[j];
m.Err[j] = Err[j];
j++;
}
return(m);
}
public BiRNN(List <SimpleLayer> s_forwardRNN, List <SimpleLayer> s_backwardRNN, SimpleLayer outputLayer)
{
forwardHiddenLayers = s_forwardRNN;
backwardHiddenLayers = s_backwardRNN;
//Initialize output layer
OutputLayer = outputLayer;
}
public BiRNN(List<SimpleLayer> s_forwardRNN, List<SimpleLayer> s_backwardRNN, int outputLayerSize)
{
forwardHiddenLayers = s_forwardRNN;
backwardHiddenLayers = s_backwardRNN;
//Initialize output layer
OutputLayer = new SimpleLayer(outputLayerSize);
OutputLayer.InitializeWeights(0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
}
public BiRNN(List <SimpleLayer> s_forwardRNN, List <SimpleLayer> s_backwardRNN, int outputLayerSize)
{
forwardHiddenLayers = s_forwardRNN;
backwardHiddenLayers = s_backwardRNN;
//Initialize output layer
OutputLayer = new SimpleLayer(outputLayerSize);
OutputLayer.InitializeWeights(0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
}
private SimpleLayer[] ComputeTopLayer(Sequence pSequence, SimpleLayer[] lastLayer, out Matrix <double> rawOutputLayer, bool isTraining, bool outputRawScore, out int[] seqBestOutput)
{
int numStates = lastLayer.Length;
seqBestOutput = new int[numStates];
//Calculate output layer
Matrix <double> tmp_rawOutputLayer = null;
if (outputRawScore == true)
{
tmp_rawOutputLayer = new Matrix <double>(numStates, OutputLayer.LayerSize);
}
List <int> labelSet = new List <int>();
foreach (State state in pSequence.States)
{
labelSet.Add(state.Label);
}
//Initialize output layer or reallocate it
if (seqFinalOutput == null || seqFinalOutput.Length < numStates)
{
seqFinalOutput = Array.CreateInstance(OutputLayer.GetType(), numStates);
for (int i = 0; i < numStates; i++)
{
seqFinalOutput.SetValue(Activator.CreateInstance(OutputLayer.GetType(), OutputLayer.LayerSize), i);
OutputLayer.ShallowCopyWeightTo((SimpleLayer)seqFinalOutput.GetValue(i));
}
}
Parallel.For(0, numStates, parallelOption, curState =>
{
State state = pSequence.States[curState];
var outputCells = (SimpleLayer)seqFinalOutput.GetValue(curState);
outputCells.LabelShortList = labelSet;
outputCells.computeLayer(state.SparseFeature, lastLayer[curState].cellOutput, isTraining);
if (outputRawScore == true)
{
outputCells.cellOutput.CopyTo(tmp_rawOutputLayer[curState], 0);
}
outputCells.Softmax(isTraining);
});
SimpleLayer[] tmpSeqFinalOutput = new SimpleLayer[numStates];
for (int i = 0; i < numStates; i++)
{
tmpSeqFinalOutput[i] = (SimpleLayer)seqFinalOutput.GetValue(i);
seqBestOutput[i] = tmpSeqFinalOutput[i].GetBestOutputIndex(isTraining);
}
rawOutputLayer = tmp_rawOutputLayer;
return(tmpSeqFinalOutput);
}
public virtual void ShallowCopyWeightTo(SimpleLayer destLayer)
{
destLayer.DenseWeights = DenseWeights;
destLayer.DenseWeightsLearningRate = DenseWeightsLearningRate;
destLayer.DenseFeatureSize = DenseFeatureSize;
destLayer.SparseWeights = SparseWeights;
destLayer.SparseWeightsLearningRate = SparseWeightsLearningRate;
destLayer.SparseFeatureSize = SparseFeatureSize;
}
public override void ShallowCopyWeightTo(SimpleLayer destLayer)
{
NCEOutputLayer layer = destLayer as NCEOutputLayer;
layer.accFreqTable = accFreqTable;
layer.accTagIdTable = accTagIdTable;
layer.vocab_size = vocab_size;
layer.accTotalFreq = accTotalFreq;
base.ShallowCopyWeightTo(layer);
}
private SimpleLayer[] ComputeTopLayer(Sequence pSequence, SimpleLayer[] lastLayer,
out Matrix <float> rawOutputLayer, bool isTraining, bool outputRawScore, out int[] seqBestOutput)
{
var numStates = lastLayer.Length;
seqBestOutput = new int[numStates];
//Calculate output layer
Matrix <float> tmp_rawOutputLayer = null;
if (outputRawScore)
{
tmp_rawOutputLayer = new Matrix <float>(numStates, OutputLayer.LayerSize);
}
var labelSet = pSequence.States.Select(state => state.Label).ToList();
//Initialize output layer or reallocate it
if (seqFinalOutput == null || seqFinalOutput.Length < numStates)
{
seqFinalOutput = Array.CreateInstance(OutputLayer.GetType(), numStates);
for (var i = 0; i < numStates; i++)
{
seqFinalOutput.SetValue(Activator.CreateInstance(OutputLayer.GetType(), OutputLayer.LayerConfig), i);
OutputLayer.ShallowCopyWeightTo((SimpleLayer)seqFinalOutput.GetValue(i));
}
}
Parallel.For(0, numStates, parallelOption, curState =>
{
var state = pSequence.States[curState];
var outputCells = (SimpleLayer)seqFinalOutput.GetValue(curState);
outputCells.LabelShortList = labelSet;
outputCells.ForwardPass(state.SparseFeature, lastLayer[curState].Cell, isTraining);
if (outputRawScore)
{
outputCells.Cell.CopyTo(tmp_rawOutputLayer[curState], 0);
}
outputCells.Softmax(isTraining);
});
var tmpSeqFinalOutput = new SimpleLayer[numStates];
for (var i = 0; i < numStates; i++)
{
tmpSeqFinalOutput[i] = (SimpleLayer)seqFinalOutput.GetValue(i);
seqBestOutput[i] = tmpSeqFinalOutput[i].GetBestOutputIndex(isTraining);
}
rawOutputLayer = tmp_rawOutputLayer;
return(tmpSeqFinalOutput);
}
private void ResetBpttMem()
{
bptt_inputs = new SparseVector[MAX_RNN_HIST];
bptt_hidden = new SimpleLayer[bptt + bptt_block + 1];
for (var i = 0; i < bptt + bptt_block + 1; i++)
{
bptt_hidden[i] = new SimpleLayer(LayerConfig);
}
bptt_fea = new float[bptt + bptt_block + 2][];
}
public void resetBpttMem()
{
bptt_inputs = new SparseVector[MAX_RNN_HIST];
bptt_hidden = new SimpleLayer[bptt + bptt_block + 1];
for (int i = 0; i < bptt + bptt_block + 1; i++)
{
bptt_hidden[i] = new SimpleLayer(LayerSize);
}
bptt_fea = new double[bptt + bptt_block + 2][];
}
public override void LoadModel(string filename)
{
Logger.WriteLine("Loading SimpleRNN model: {0}", filename);
StreamReader sr = new StreamReader(filename);
BinaryReader br = new BinaryReader(sr.BaseStream);
int modelType = br.ReadInt32();
ModelDirection = (MODELDIRECTION)br.ReadInt32();
int iflag = br.ReadInt32();
if (iflag == 1)
{
IsCRFTraining = true;
}
else
{
IsCRFTraining = false;
}
//Create cells of each layer
int layerSize = br.ReadInt32();
HiddenLayerList = new List <SimpleLayer>();
for (int i = 0; i < layerSize; i++)
{
SimpleLayer layer = null;
if (modelType == 0)
{
layer = new BPTTLayer();
}
else
{
layer = new LSTMLayer();
}
layer.Load(br);
HiddenLayerList.Add(layer);
}
OutputLayer = new SimpleLayer();
OutputLayer.Load(br);
if (iflag == 1)
{
Logger.WriteLine("Loading CRF tag trans weights...");
CRFTagTransWeights = RNNHelper.LoadMatrix(br);
}
sr.Close();
}
public SimpleLayer GetHiddenLayer()
{
SimpleLayer m = new SimpleLayer(LayerSize);
for (int i = 0; i < LayerSize; i++)
{
m.cellOutput[i] = cellOutput[i];
m.er[i] = er[i];
}
return(m);
}
public override void ComputeLayerErr(SimpleLayer nextLayer)
{
base.ComputeLayerErr(nextLayer);
//Apply drop out on error in hidden layer
for (var i = 0; i < LayerSize; i++)
{
if (mask[i])
{
Errs[i] = 0;
}
}
}
public override void ComputeLayerErr(SimpleLayer nextLayer, double[] destErrLayer, double[] srcErrLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
for (int i = 0; i < LayerSize; i++)
{
if (mask[i] == true)
{
destErrLayer[i] = 0;
}
}
}
public override SimpleLayer GetHiddenLayer()
{
SimpleLayer m = new SimpleLayer(L1);
for (int i = 0; i < L1; i++)
{
m.cellOutput[i] = neuHidden[i].cellOutput;
m.er[i] = neuHidden[i].er;
m.mask[i] = neuHidden[i].mask;
}
return(m);
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer)
{
NCEOutputLayer largeOutputLayer = nextLayer as NCEOutputLayer;
if (largeOutputLayer != null)
{
RNNHelper.matrixXvectorADDErr(er, largeOutputLayer.er, largeOutputLayer.DenseWeights, LayerSize, largeOutputLayer.negativeSampleWordList);
}
else
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(er, nextLayer.er, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
}
}
public virtual void ShallowCopyWeightTo(SimpleLayer destLayer)
{
destLayer.DenseWeights = DenseWeights;
destLayer.DenseWeightsDelta = DenseWeightsDelta;
destLayer.DenseWeightsLearningRate = DenseWeightsLearningRate;
destLayer.DenseFeatureSize = DenseFeatureSize;
destLayer.SparseWeights = SparseWeights;
destLayer.SparseWeightsDelta = SparseWeightsDelta;
destLayer.SparseWeightsLearningRate = SparseWeightsLearningRate;
destLayer.SparseFeatureSize = SparseFeatureSize;
destLayer.InitializeInternalTrainingParameters();
}
public override void ComputeLayerErr(SimpleLayer nextLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(er, nextLayer.er, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
//Apply drop out on error in hidden layer
for (int i = 0; i < LayerSize; i++)
{
if (mask[i] == true)
{
er[i] = 0;
}
}
}
public override int[] PredictSentenceCRF(Sequence pSequence, RunningMode runningMode)
{
//Reset the network
int numStates = pSequence.States.Length;
//Predict output
SimpleLayer[] mergedHiddenLayer = null;
Matrix <double> rawOutputLayer = null;
SimpleLayer[] seqOutput = InnerDecode(pSequence, out mergedHiddenLayer, out rawOutputLayer);
ForwardBackward(numStates, rawOutputLayer);
if (runningMode != RunningMode.Test)
{
//Get the best result
for (int i = 0; i < numStates; i++)
{
logp += Math.Log10(CRFSeqOutput[i][pSequence.States[i].Label] + 0.0001);
}
}
int[] predict = Viterbi(rawOutputLayer, numStates);
if (runningMode == RunningMode.Train)
{
UpdateBigramTransition(pSequence);
//Update hidden-output layer weights
for (int curState = 0; curState < numStates; curState++)
{
int label = pSequence.States[curState].Label;
SimpleLayer layer = seqOutput[curState];
double[] CRFOutputLayer = CRFSeqOutput[curState];
//For standard RNN
for (int c = 0; c < L2; c++)
{
layer.er[c] = -CRFOutputLayer[c];
}
layer.er[label] = 1 - CRFOutputLayer[label];
}
LearnTwoRNN(pSequence, mergedHiddenLayer, seqOutput);
}
return(predict);
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer, float[] destErrLayer, float[] srcErrLayer)
{
var sampledSoftmaxLayer = nextLayer as SampledSoftmaxLayer;
if (sampledSoftmaxLayer != null)
{
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, sampledSoftmaxLayer.DenseWeights, LayerSize,
sampledSoftmaxLayer.negativeSampleWordList);
}
else
{
var lstmLayer = nextLayer as LSTMLayer;
if (lstmLayer != null)
{
Array.Clear(destErrLayer, 0, destErrLayer.Length);
for (var k = 0; k < nextLayer.LayerSize; k++)
{
int i = 0;
float[] weights = lstmLayer.wDenseOutputGate.weights[k];
float err = srcErrLayer[k];
var moreItems = (LayerSize % Vector <float> .Count);
while (i < LayerSize - moreItems)
{
Vector <float> vecWeights = new Vector <float>(weights, i);
Vector <float> vecErrs = new Vector <float>(destErrLayer, i);
vecErrs += err * vecWeights;
vecErrs.CopyTo(destErrLayer, i);
i += Vector <float> .Count;
}
while (i < LayerSize)
{
destErrLayer[i] += err * weights[i];
i++;
}
}
}
else
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize,
nextLayer.LayerSize);
}
}
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer, double[] destErrLayer, double[] srcErrLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(destErrLayer, srcErrLayer, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
if (Dropout > 0)
{
//Apply drop out on error in hidden layer
for (int i = 0; i < LayerSize; i++)
{
if (mask[i] == true)
{
destErrLayer[i] = 0;
}
}
}
}
public override void LearnNet(State state, int numStates, int curState)
{
int maxBptt = 0;
for (maxBptt = 0; maxBptt < bptt + bptt_block - 1; maxBptt++)
{
if (bptt_inputs[maxBptt] == null)
{
break;
}
}
//Shift memory needed for bptt to next time step,
//and save current hidden and feature layer nodes values for bptt
SimpleLayer last_bptt_hidden = bptt_hidden[maxBptt];
double[] last_bptt_fea = bptt_fea[maxBptt];
for (int a = maxBptt; a > 0; a--)
{
bptt_inputs[a] = bptt_inputs[a - 1];
bptt_hidden[a] = bptt_hidden[a - 1];
bptt_fea[a] = bptt_fea[a - 1];
}
bptt_inputs[0] = state.SparseData;
bptt_hidden[0] = last_bptt_hidden;
bptt_fea[0] = last_bptt_fea;
for (int i = 0; i < L1; i++)
{
last_bptt_hidden.cellOutput[i] = neuHidden.cellOutput[i];
last_bptt_hidden.er[i] = neuHidden.er[i];
last_bptt_hidden.mask[i] = neuHidden.mask[i];
}
for (int i = 0; i < DenseFeatureSize; i++)
{
last_bptt_fea[i] = neuFeatures[i];
}
// time to learn bptt
if (curState > 0 && ((curState % bptt_block) == 0 || curState == (numStates - 1)))
{
learnBptt(state);
}
}
// forward process. output layer consists of tag value
public override void computeHiddenLayer(State state, bool isTrain = true)
{
//keep last hidden layer and erase activations
neuLastHidden = neuHidden;
//hidden(t-1) -> hidden(t)
neuHidden = new SimpleLayer(L1);
matrixXvectorADD(neuHidden, neuLastHidden, HiddenBpttWeights, L1, L1, 0);
//Apply feature values on hidden layer
var sparse = state.SparseData;
int n = sparse.Count;
Parallel.For(0, L1, parallelOption, b =>
{
//Sparse features:
//inputs(t) -> hidden(t)
//Get sparse feature and apply it into hidden layer
double[] vector_b = Input2HiddenWeights[b];
double cellOutput = 0;
for (int i = 0; i < n; i++)
{
var entry = sparse.GetEntry(i);
cellOutput += entry.Value * vector_b[entry.Key];
}
//Dense features:
//fea(t) -> hidden(t)
if (DenseFeatureSize > 0)
{
vector_b = Feature2HiddenWeights[b];
for (int j = 0; j < DenseFeatureSize; j++)
{
cellOutput += neuFeatures[j] * vector_b[j];
}
}
neuHidden.cellOutput[b] += cellOutput;
});
//activate 1 --sigmoid
computeHiddenActivity(isTrain);
}
public SimpleLayer GetHiddenLayer()
{
SimpleLayer m = new SimpleLayer(LayerSize);
for (int i = 0; i < LayerSize; i++)
{
m.cellOutput[i] = cellOutput[i];
m.er[i] = er[i];
m.mask[i] = mask[i];
}
return m;
}
public override void ComputeLayerErr(SimpleLayer nextLayer)
{
LSTMLayer layer = nextLayer as LSTMLayer;
if (layer != null)
{
Parallel.For(0, LayerSize, parallelOption, i =>
{
er[i] = 0.0;
if (mask[i] == false)
{
for (int k = 0; k < nextLayer.LayerSize; k++)
{
er[i] += layer.er[k] * layer.feature2hidden[k][i].W;
}
}
});
}
else
{
base.ComputeLayerErr(nextLayer);
}
}
public void Softmax(SimpleLayer outputLayer)
{
double sum = 0;
for (int c = 0; c < outputLayer.LayerSize; c++)
{
double cellOutput = outputLayer.cellOutput[c];
if (cellOutput > 50) cellOutput = 50;
if (cellOutput < -50) cellOutput = -50;
double val = Math.Exp(cellOutput);
sum += val;
outputLayer.cellOutput[c] = val;
}
int i = 0;
Vector<double> vecSum = new Vector<double>(sum);
while (i < outputLayer.LayerSize - Vector<double>.Count)
{
Vector<double> v = new Vector<double>(outputLayer.cellOutput, i);
v /= vecSum;
v.CopyTo(outputLayer.cellOutput, i);
i += Vector<double>.Count;
}
while (i < outputLayer.LayerSize)
{
outputLayer.cellOutput[i] /= sum;
i++;
}
}
/// <summary>
/// Compute the output of bottom layer
/// </summary>
/// <param name="pSequence"></param>
/// <param name="forwardLayer"></param>
/// <param name="backwardLayer"></param>
/// <returns></returns>
private SimpleLayer[] ComputeBottomLayer(Sequence pSequence, SimpleLayer forwardLayer, SimpleLayer backwardLayer)
{
int numStates = pSequence.States.Length;
SimpleLayer[] mForward = null;
SimpleLayer[] mBackward = null;
Parallel.Invoke(() =>
{
//Computing forward RNN
forwardLayer.netReset(false);
mForward = new SimpleLayer[numStates];
for (int curState = 0; curState < numStates; curState++)
{
State state = pSequence.States[curState];
SetInputLayer(state, curState, numStates, null);
forwardLayer.computeLayer(state.SparseData, state.DenseData.CopyTo());
mForward[curState] = forwardLayer.GetHiddenLayer();
}
},
() =>
{
//Computing backward RNN
backwardLayer.netReset(false);
mBackward = new SimpleLayer[numStates];
for (int curState = numStates - 1; curState >= 0; curState--)
{
State state = pSequence.States[curState];
SetInputLayer(state, curState, numStates, null, false);
backwardLayer.computeLayer(state.SparseData, state.DenseData.CopyTo()); //compute probability distribution
mBackward[curState] = backwardLayer.GetHiddenLayer();
}
});
SimpleLayer[] mergedLayer = new SimpleLayer[numStates];
Parallel.For(0, numStates, parallelOption, curState =>
{
State state = pSequence.States[curState];
mergedLayer[curState] = new SimpleLayer(forwardLayer.LayerSize);
mergedLayer[curState].SparseFeature = state.SparseData;
mergedLayer[curState].DenseFeature = state.DenseData.CopyTo();
SimpleLayer forwardCells = mForward[curState];
SimpleLayer backwardCells = mBackward[curState];
int i = 0;
while (i < forwardLayer.LayerSize - Vector<double>.Count)
{
Vector<double> v1 = new Vector<double>(forwardCells.cellOutput, i);
Vector<double> v2 = new Vector<double>(backwardCells.cellOutput, i);
Vector<double> v = (v1 + v2) / vecConst2;
v.CopyTo(mergedLayer[curState].cellOutput, i);
i += Vector<float>.Count;
}
while (i < forwardLayer.LayerSize)
{
mergedLayer[curState].cellOutput[i] = (forwardCells.cellOutput[i] + backwardCells.cellOutput[i]) / 2.0;
i++;
}
});
return mergedLayer;
}
/// <summary>
/// Pass error from the last layer to the first layer
/// </summary>
/// <param name="pSequence"></param>
/// <param name="seqFinalOutput"></param>
/// <param name="isCRF"></param>
/// <returns></returns>
private void ComputeDeepErr(Sequence pSequence, SimpleLayer[] seqFinalOutput, out List<double[][]> fErrLayers, out List<double[][]> bErrLayers, bool isCRF = false)
{
int numStates = pSequence.States.Length;
int numLayers = forwardHiddenLayers.Count;
//Calculate output layer error
for (int curState = 0; curState < numStates; curState++)
{
int label = pSequence.States[curState].Label;
SimpleLayer layer = seqFinalOutput[curState];
if (isCRF == false)
{
for (int c = 0; c < layer.LayerSize; c++)
{
layer.er[c] = -layer.cellOutput[c];
}
layer.er[label] = 1.0 - layer.cellOutput[label];
}
else
{
double[] CRFOutputLayer = CRFSeqOutput[curState];
for (int c = 0; c < layer.LayerSize; c++)
{
layer.er[c] = -CRFOutputLayer[c];
}
layer.er[label] = 1 - CRFOutputLayer[label];
}
}
//Now we already have err in output layer, let's pass them back to other layers
fErrLayers = new List<double[][]>();
bErrLayers = new List<double[][]>();
for (int i = 0; i < numLayers; i++)
{
fErrLayers.Add(null);
bErrLayers.Add(null);
}
//Pass error from i+1 to i layer
SimpleLayer forwardLayer = forwardHiddenLayers[numLayers - 1];
SimpleLayer backwardLayer = backwardHiddenLayers[numLayers - 1];
double[][] errLayer = new double[numStates][];
Parallel.For(0, numStates, parallelOption, curState =>
{
errLayer[curState] = new double[forwardLayer.LayerSize];
forwardLayer.ComputeLayerErr(seqFinalOutput[curState], errLayer[curState], seqFinalOutput[curState].er);
});
fErrLayers[numLayers - 1] = errLayer;
bErrLayers[numLayers - 1] = errLayer;
// Forward
for (int i = numLayers - 2; i >= 0; i--)
{
forwardLayer = forwardHiddenLayers[i];
errLayer = new double[numStates][];
double[][] srcErrLayer = fErrLayers[i + 1];
Parallel.For(0, numStates, parallelOption, curState =>
{
int curState2 = numStates - curState - 1;
errLayer[curState2] = new double[forwardLayer.LayerSize];
forwardLayer.ComputeLayerErr(forwardHiddenLayers[i + 1], errLayer[curState2], srcErrLayer[curState2]);
});
fErrLayers[i] = errLayer;
}
// Backward
for (int i = numLayers - 2; i >= 0; i--)
{
backwardLayer = backwardHiddenLayers[i];
errLayer = new double[numStates][];
double[][] srcErrLayer = bErrLayers[i + 1];
Parallel.For(0, numStates, parallelOption, curState =>
{
errLayer[curState] = new double[backwardLayer.LayerSize];
backwardLayer.ComputeLayerErr(backwardHiddenLayers[i + 1], errLayer[curState], srcErrLayer[curState]);
});
bErrLayers[i] = errLayer;
}
}
private SimpleLayer[] ComputeMiddleLayers(SimpleLayer[] lastLayers, SimpleLayer forwardLayer, SimpleLayer backwardLayer)
{
int numStates = lastLayers.Length;
SimpleLayer[] mForward = null;
SimpleLayer[] mBackward = null;
Parallel.Invoke(() =>
{
//Computing forward RNN
forwardLayer.netReset(false);
mForward = new SimpleLayer[lastLayers.Length];
for (int curState = 0; curState < lastLayers.Length; curState++)
{
forwardLayer.computeLayer(null, lastLayers[curState].cellOutput);
mForward[curState] = forwardLayer.GetHiddenLayer();
}
},
() =>
{
//Computing backward RNN
backwardLayer.netReset(false);
mBackward = new SimpleLayer[lastLayers.Length];
for (int curState = lastLayers.Length - 1; curState >= 0; curState--)
{
backwardLayer.computeLayer(null, lastLayers[curState].cellOutput);
mBackward[curState] = backwardLayer.GetHiddenLayer();
}
});
//Merge forward and backward
SimpleLayer[] mergedLayer = new SimpleLayer[numStates];
Parallel.For(0, numStates, parallelOption, curState =>
{
mergedLayer[curState] = new SimpleLayer(forwardLayer.LayerSize);
mergedLayer[curState].SparseFeature = null;
mergedLayer[curState].DenseFeature = lastLayers[curState].cellOutput;
SimpleLayer forwardCells = mForward[curState];
SimpleLayer backwardCells = mBackward[curState];
int i = 0;
while (i < forwardLayer.LayerSize - Vector<double>.Count)
{
Vector<double> v1 = new Vector<double>(forwardCells.cellOutput, i);
Vector<double> v2 = new Vector<double>(backwardCells.cellOutput, i);
Vector<double> v = (v1 + v2) / vecConst2;
v.CopyTo(mergedLayer[curState].cellOutput, i);
i += Vector<float>.Count;
}
while (i < forwardLayer.LayerSize)
{
mergedLayer[curState].cellOutput[i] = (forwardCells.cellOutput[i] + backwardCells.cellOutput[i]) / 2.0;
i++;
}
});
return mergedLayer;
}
private SimpleLayer[] ComputeTopLayer(SimpleLayer[] lastLayer, out Matrix<double> rawOutputLayer, bool isTrain)
{
int numStates = lastLayer.Length;
//Calculate output layer
Matrix<double> tmp_rawOutputLayer = new Matrix<double>(numStates, OutputLayer.LayerSize);
SimpleLayer[] seqFinalOutput = new SimpleLayer[numStates];
Parallel.For(0, numStates, parallelOption, curState =>
{
seqFinalOutput[curState] = new SimpleLayer(OutputLayer.LayerSize);
SimpleLayer outputCells = seqFinalOutput[curState];
outputCells.DenseWeights = OutputLayer.DenseWeights;
outputCells.DenseWeightsLearningRate = OutputLayer.DenseWeightsLearningRate;
outputCells.DenseFeatureSize = OutputLayer.DenseFeatureSize;
outputCells.computeLayer(null, lastLayer[curState].cellOutput, isTrain);
outputCells.cellOutput.CopyTo(tmp_rawOutputLayer[curState], 0);
Softmax(outputCells);
});
rawOutputLayer = tmp_rawOutputLayer;
return seqFinalOutput;
}
private void DeepLearningNet(Sequence pSequence, SimpleLayer[] seqOutput, List<double[][]> fErrLayers,
List<double[][]> bErrLayers, List<SimpleLayer[]> layerList)
{
int numStates = pSequence.States.Length;
int numLayers = forwardHiddenLayers.Count;
//Learning output layer
Parallel.Invoke(() =>
{
for (int curState = 0; curState < numStates; curState++)
{
seqOutput[curState].LearnFeatureWeights(numStates, curState);
}
},
() =>
{
Parallel.For(0, numLayers, parallelOption, i =>
{
Parallel.Invoke(() =>
{
SimpleLayer forwardLayer = forwardHiddenLayers[i];
forwardLayer.netReset(true);
for (int curState = 0; curState < numStates; curState++)
{
forwardLayer.computeLayer(layerList[i][curState].SparseFeature, layerList[i][curState].DenseFeature, true);
forwardLayer.er = fErrLayers[i][curState];
forwardLayer.LearnFeatureWeights(numStates, curState);
}
},
() =>
{
SimpleLayer backwardLayer = backwardHiddenLayers[i];
backwardLayer.netReset(true);
for (int curState = 0; curState < numStates; curState++)
{
int curState2 = numStates - curState - 1;
backwardLayer.computeLayer(layerList[i][curState2].SparseFeature, layerList[i][curState2].DenseFeature, true);
backwardLayer.er = bErrLayers[i][curState2];
backwardLayer.LearnFeatureWeights(numStates, curState);
}
});
});
});
}
public override void LoadModel(string filename)
{
Logger.WriteLine(Logger.Level.info, "Loading bi-directional model: {0}", filename);
using (StreamReader sr = new StreamReader(filename))
{
BinaryReader br = new BinaryReader(sr.BaseStream);
int modelType = br.ReadInt32();
ModelDirection = (MODELDIRECTION)br.ReadInt32();
int iflag = br.ReadInt32();
if (iflag == 1)
{
IsCRFTraining = true;
}
else
{
IsCRFTraining = false;
}
int layerSize = br.ReadInt32();
//Load forward layers from file
forwardHiddenLayers = new List<SimpleLayer>();
for (int i = 0; i < layerSize; i++)
{
SimpleLayer layer = null;
if (modelType == 0)
{
Logger.WriteLine("Create BPTT hidden layer");
layer = new BPTTLayer();
}
else
{
Logger.WriteLine("Crate LSTM hidden layer");
layer = new LSTMLayer();
}
layer.Load(br);
forwardHiddenLayers.Add(layer);
}
//Load backward layers from file
backwardHiddenLayers = new List<SimpleLayer>();
for (int i = 0; i < layerSize; i++)
{
SimpleLayer layer = null;
if (modelType == 0)
{
Logger.WriteLine("Create BPTT hidden layer");
layer = new BPTTLayer();
}
else
{
Logger.WriteLine("Crate LSTM hidden layer");
layer = new LSTMLayer();
}
layer.Load(br);
backwardHiddenLayers.Add(layer);
}
OutputLayer = new SimpleLayer();
OutputLayer.Load(br);
if (iflag == 1)
{
Logger.WriteLine("Loading CRF tag trans weights...");
CRFTagTransWeights = RNNHelper.LoadMatrix(br);
}
}
}
public virtual void ComputeLayerErr(SimpleLayer nextLayer)
{
//error output->hidden for words from specific class
RNNHelper.matrixXvectorADDErr(er, nextLayer.er, nextLayer.DenseWeights, LayerSize, nextLayer.LayerSize);
if (Dropout > 0)
{
//Apply drop out on error in hidden layer
for (int i = 0; i < LayerSize; i++)
{
if (mask[i] == true)
{
er[i] = 0;
}
}
}
}
public void resetBpttMem()
{
bptt_inputs = new SparseVector[MAX_RNN_HIST];
bptt_hidden = new SimpleLayer[bptt + bptt_block + 1];
for (int i = 0; i < bptt + bptt_block + 1; i++)
{
bptt_hidden[i] = new SimpleLayer(LayerSize);
}
bptt_fea = new double[bptt + bptt_block + 2][];
}