/// <summary>
/// Process a given sequence by bi-directional recurrent neural network and CRF
/// </summary>
/// <param name="pSequence"></param>
/// <param name="runningMode"></param>
/// <returns></returns>
public override int[] ProcessSequenceCRF(Sequence pSequence, RunningMode runningMode)
{
//Reset the network
int numStates = pSequence.States.Length;
List<SimpleLayer[]> layerList;
Matrix<double> rawOutputLayer;
SimpleLayer[] seqOutput = ComputeLayers(pSequence, runningMode == RunningMode.Train, out layerList, out rawOutputLayer);
ForwardBackward(numStates, rawOutputLayer);
if (runningMode != RunningMode.Test)
{
//Merge forward and backward
for (int curState = 0; curState < numStates; curState++)
{
logp += Math.Log10(CRFSeqOutput[curState][pSequence.States[curState].Label] + 0.0001);
}
}
int[] predict = Viterbi(rawOutputLayer, numStates);
if (runningMode == RunningMode.Train)
{
UpdateBigramTransition(pSequence);
List<double[][]> fErrLayers;
List<double[][]> bErrLayers;
ComputeDeepErr(pSequence, seqOutput, out fErrLayers, out bErrLayers, true);
DeepLearningNet(pSequence, seqOutput, fErrLayers, bErrLayers, layerList);
}
return predict;
}
/// <summary>
/// Compute the output of bottom layer
/// </summary>
/// <param name="pSequence"></param>
/// <param name="forwardLayer"></param>
/// <param name="backwardLayer"></param>
/// <returns></returns>
private SimpleLayer[] ComputeBottomLayer(Sequence pSequence, SimpleLayer forwardLayer, SimpleLayer backwardLayer)
{
int numStates = pSequence.States.Length;
SimpleLayer[] mForward = null;
SimpleLayer[] mBackward = null;
Parallel.Invoke(() =>
{
//Computing forward RNN
forwardLayer.netReset(false);
mForward = new SimpleLayer[numStates];
for (int curState = 0; curState < numStates; curState++)
{
State state = pSequence.States[curState];
SetInputLayer(state, curState, numStates, null);
forwardLayer.computeLayer(state.SparseData, state.DenseData.CopyTo());
mForward[curState] = forwardLayer.GetHiddenLayer();
}
},
() =>
{
//Computing backward RNN
backwardLayer.netReset(false);
mBackward = new SimpleLayer[numStates];
for (int curState = numStates - 1; curState >= 0; curState--)
{
State state = pSequence.States[curState];
SetInputLayer(state, curState, numStates, null, false);
backwardLayer.computeLayer(state.SparseData, state.DenseData.CopyTo()); //compute probability distribution
mBackward[curState] = backwardLayer.GetHiddenLayer();
}
});
SimpleLayer[] mergedLayer = new SimpleLayer[numStates];
Parallel.For(0, numStates, parallelOption, curState =>
{
State state = pSequence.States[curState];
mergedLayer[curState] = new SimpleLayer(forwardLayer.LayerSize);
mergedLayer[curState].SparseFeature = state.SparseData;
mergedLayer[curState].DenseFeature = state.DenseData.CopyTo();
SimpleLayer forwardCells = mForward[curState];
SimpleLayer backwardCells = mBackward[curState];
int i = 0;
while (i < forwardLayer.LayerSize - Vector<double>.Count)
{
Vector<double> v1 = new Vector<double>(forwardCells.cellOutput, i);
Vector<double> v2 = new Vector<double>(backwardCells.cellOutput, i);
Vector<double> v = (v1 + v2) / vecConst2;
v.CopyTo(mergedLayer[curState].cellOutput, i);
i += Vector<float>.Count;
}
while (i < forwardLayer.LayerSize)
{
mergedLayer[curState].cellOutput[i] = (forwardCells.cellOutput[i] + backwardCells.cellOutput[i]) / 2.0;
i++;
}
});
return mergedLayer;
}
public override int[] PredictSentenceCRF(Sequence pSequence, RunningMode runningMode)
{
//Reset the network
int numStates = pSequence.States.Length;
//Predict output
Matrix<neuron> mergedHiddenLayer = null;
Matrix<double> rawOutputLayer = null;
neuron[][] seqOutput = InnerDecode(pSequence, out mergedHiddenLayer, out rawOutputLayer);
ForwardBackward(numStates, rawOutputLayer);
if (runningMode != RunningMode.Test)
{
//Get the best result
for (int i = 0; i < numStates; i++)
{
logp += Math.Log10(CRFSeqOutput[i][pSequence.States[i].Label]);
}
}
int[] predict = Viterbi(rawOutputLayer, numStates);
if (runningMode == RunningMode.Train)
{
UpdateBigramTransition(pSequence);
//Update hidden-output layer weights
for (int curState = 0; curState < numStates; curState++)
{
int label = pSequence.States[curState].Label;
//For standard RNN
for (int c = 0; c < L2; c++)
{
seqOutput[curState][c].er = -CRFSeqOutput[curState][c];
}
seqOutput[curState][label].er = 1 - CRFSeqOutput[curState][label];
}
LearnTwoRNN(pSequence, mergedHiddenLayer, seqOutput);
}
return predict;
}
/// <summary>
/// Process a given sequence by bi-directional recurrent neural network
/// </summary>
/// <param name="pSequence"></param>
/// <param name="runningMode"></param>
/// <returns></returns>
public override Matrix<double> ProcessSequence(Sequence pSequence, RunningMode runningMode)
{
List<SimpleLayer[]> layerList;
Matrix<double> rawOutputLayer;
//Forward process from bottom layer to top layer
SimpleLayer[] seqOutput = ComputeLayers(pSequence, runningMode == RunningMode.Train, out layerList, out rawOutputLayer);
if (runningMode != RunningMode.Test)
{
int numStates = pSequence.States.Length;
for (int curState = 0; curState < numStates; curState++)
{
logp += Math.Log10(seqOutput[curState].cellOutput[pSequence.States[curState].Label] + 0.0001);
}
}
if (runningMode == RunningMode.Train)
{
//In training mode, we calculate each layer's error and update their net weights
List<double[][]> fErrLayers;
List<double[][]> bErrLayers;
ComputeDeepErr(pSequence, seqOutput, out fErrLayers, out bErrLayers);
DeepLearningNet(pSequence, seqOutput, fErrLayers, bErrLayers, layerList);
}
return rawOutputLayer;
}
public Sequence ExtractFeatures(Sentence sentence)
{
int n = sentence.TokensList.Count;
Sequence sequence = new Sequence(n);
//For each token, get its sparse and dense feature set according configuration and training corpus
for (int i = 0; i < n; i++)
{
State state = sequence.States[i];
ExtractSparseFeature(i, n, sentence.TokensList, state);
state.DenseData = ExtractDenseFeature(i, n, sentence.TokensList);
}
return sequence;
}
public override int[] PredictSentence(Sequence pSequence)
{
//Reset the network
int numStates = pSequence.GetSize();
int[] predicted = new int[numStates];
//Predict output
Matrix m = InnerDecode(pSequence);
//Merge forward and backward
for (int curState = 0; curState < numStates; curState++)
{
State state = pSequence.Get(curState);
//activation 2 --softmax on words
double sum = 0; //sum is used for normalization: it's better to have larger precision as many numbers are summed together here
for (int c = 0; c < forwardRNN.L2; c++)
{
if (m[curState][c] > 50) m[curState][c] = 50; //for numerical stability
if (m[curState][c] < -50) m[curState][c] = -50; //for numerical stability
double val = Math.Exp(m[curState][c]);
sum += val;
m[curState][c] = val;
}
for (int c = 0; c < forwardRNN.L2; c++)
{
m[curState][c] /= sum;
}
logp += Math.Log10(m[curState][state.GetLabel()]);
counter++;
predicted[curState] = GetBestOutputIndex(m, curState);
}
netReset();
double[] output = new double[L2];
//Learn forward network
for (int curState = 0; curState < numStates; curState++)
{
// error propogation
State state = pSequence.Get(curState);
forwardRNN.setInputLayer(state, curState, numStates, predicted_fnn);
forwardRNN.computeNet(state, output); //compute probability distribution
//Copy output result to forward net work's output
for (int i = 0; i < forwardRNN.L2; i++)
{
forwardRNN.neuOutput[i].ac = m[curState][i];
}
forwardRNN.learnNet(state, curState);
forwardRNN.LearnBackTime(state, numStates, curState);
forwardRNN.copyHiddenLayerToInput();
}
for (int curState = numStates - 1; curState >= 0; curState--)
{
// error propogation
State state = pSequence.Get(curState);
backwardRNN.setInputLayer(state, curState, numStates, predicted_bnn, false);
backwardRNN.computeNet(state, output); //compute probability distribution
//Copy output result to forward net work's output
for (int i = 0; i < backwardRNN.L2; i++)
{
backwardRNN.neuOutput[i].ac = m[curState][i];
}
backwardRNN.learnNet(state, curState);
backwardRNN.LearnBackTime(state, numStates, curState);
backwardRNN.copyHiddenLayerToInput();
}
return predicted;
}
public abstract int[] ProcessSequenceCRF(Sequence pSequence, RunningMode runningMode);
private void DeepLearningNet(Sequence pSequence, SimpleLayer[] seqOutput, List<double[][]> fErrLayers,
List<double[][]> bErrLayers, List<SimpleLayer[]> layerList)
{
int numStates = pSequence.States.Length;
int numLayers = forwardHiddenLayers.Count;
//Learning output layer
Parallel.Invoke(() =>
{
for (int curState = 0; curState < numStates; curState++)
{
seqOutput[curState].LearnFeatureWeights(numStates, curState);
}
},
() =>
{
Parallel.For(0, numLayers, parallelOption, i =>
{
Parallel.Invoke(() =>
{
SimpleLayer forwardLayer = forwardHiddenLayers[i];
forwardLayer.netReset(true);
for (int curState = 0; curState < numStates; curState++)
{
forwardLayer.computeLayer(layerList[i][curState].SparseFeature, layerList[i][curState].DenseFeature, true);
forwardLayer.er = fErrLayers[i][curState];
forwardLayer.LearnFeatureWeights(numStates, curState);
}
},
() =>
{
SimpleLayer backwardLayer = backwardHiddenLayers[i];
backwardLayer.netReset(true);
for (int curState = 0; curState < numStates; curState++)
{
int curState2 = numStates - curState - 1;
backwardLayer.computeLayer(layerList[i][curState2].SparseFeature, layerList[i][curState2].DenseFeature, true);
backwardLayer.er = bErrLayers[i][curState2];
backwardLayer.LearnFeatureWeights(numStates, curState);
}
});
});
});
}
public int[] DecodeNN(Sequence seq)
{
Matrix<double> ys = ProcessSequence(seq, RunningMode.Test);
return GetBestResult(ys);
}
public abstract Matrix<double> ProcessSequence(Sequence pSequence, RunningMode runningMode);
public Sequence ExtractFeatures(Sentence sentence)
{
Sequence sequence = new Sequence();
int n = sentence.GetTokenSize();
List<string[]> features = sentence.GetFeatureSet();
//For each token, get its sparse and dense feature set according configuration and training corpus
sequence.SetSize(n);
for (int i = 0; i < n; i++)
{
State state = sequence.Get(i);
ExtractSparseFeature(i, n, features, state);
var spDenseFeature = ExtractDenseFeature(i, n, features);
state.SetDenseData(spDenseFeature);
}
return sequence;
}
private void LearnTwoRNN(Sequence pSequence, Matrix<neuron> mergedHiddenLayer, neuron[][] seqOutput)
{
netReset(true);
int numStates = pSequence.States.Length;
forwardRNN.Hidden2OutputWeight = Hidden2OutputWeight.CopyTo();
backwardRNN.Hidden2OutputWeight = Hidden2OutputWeight.CopyTo();
Parallel.Invoke(() =>
{
for (int curState = 0; curState < numStates; curState++)
{
for (int i = 0; i < Hidden2OutputWeight.GetHeight(); i++)
{
//update weights for hidden to output layer
for (int k = 0; k < Hidden2OutputWeight.GetWidth(); k++)
{
Hidden2OutputWeight[i][k] += LearningRate * mergedHiddenLayer[curState][k].cellOutput * seqOutput[curState][i].er;
}
}
}
},
()=>
{
//Learn forward network
for (int curState = 0; curState < numStates; curState++)
{
// error propogation
State state = pSequence.States[curState];
forwardRNN.setInputLayer(state, curState, numStates, null);
forwardRNN.computeNet(state, null); //compute probability distribution
//Copy output result to forward net work's output
forwardRNN.OutputLayer = seqOutput[curState];
forwardRNN.learnNet(state, curState, true);
forwardRNN.LearnBackTime(state, numStates, curState);
}
},
() =>
{
for (int curState = 0; curState < numStates; curState++)
{
int curState2 = numStates - 1 - curState;
// error propogation
State state2 = pSequence.States[curState2];
backwardRNN.setInputLayer(state2, curState2, numStates, null, false);
backwardRNN.computeNet(state2, null); //compute probability distribution
//Copy output result to forward net work's output
backwardRNN.OutputLayer = seqOutput[curState2];
backwardRNN.learnNet(state2, curState2, true);
backwardRNN.LearnBackTime(state2, numStates, curState2);
}
});
}
public override Matrix<double> PredictSentence(Sequence pSequence, RunningMode runningMode)
{
//Reset the network
int numStates = pSequence.States.Length;
//Predict output
Matrix<neuron> mergedHiddenLayer = null;
Matrix<double> rawOutputLayer = null;
neuron[][] seqOutput = InnerDecode(pSequence, out mergedHiddenLayer, out rawOutputLayer);
if (runningMode != RunningMode.Test)
{
//Merge forward and backward
for (int curState = 0; curState < numStates; curState++)
{
logp += Math.Log10(seqOutput[curState][pSequence.States[curState].Label].cellOutput);
}
}
if (runningMode == RunningMode.Train)
{
//Update hidden-output layer weights
for (int curState = 0; curState < numStates; curState++)
{
int label = pSequence.States[curState].Label;
//For standard RNN
for (int c = 0; c < L2; c++)
{
seqOutput[curState][c].er = -seqOutput[curState][c].cellOutput;
}
seqOutput[curState][label].er = 1 - seqOutput[curState][label].cellOutput;
}
LearnTwoRNN(pSequence, mergedHiddenLayer, seqOutput);
}
return rawOutputLayer;
}
/// <summary>
/// Pass error from the last layer to the first layer
/// </summary>
/// <param name="pSequence"></param>
/// <param name="seqFinalOutput"></param>
/// <param name="isCRF"></param>
/// <returns></returns>
private void ComputeDeepErr(Sequence pSequence, SimpleLayer[] seqFinalOutput, out List<double[][]> fErrLayers, out List<double[][]> bErrLayers, bool isCRF = false)
{
int numStates = pSequence.States.Length;
int numLayers = forwardHiddenLayers.Count;
//Calculate output layer error
for (int curState = 0; curState < numStates; curState++)
{
int label = pSequence.States[curState].Label;
SimpleLayer layer = seqFinalOutput[curState];
if (isCRF == false)
{
for (int c = 0; c < layer.LayerSize; c++)
{
layer.er[c] = -layer.cellOutput[c];
}
layer.er[label] = 1.0 - layer.cellOutput[label];
}
else
{
double[] CRFOutputLayer = CRFSeqOutput[curState];
for (int c = 0; c < layer.LayerSize; c++)
{
layer.er[c] = -CRFOutputLayer[c];
}
layer.er[label] = 1 - CRFOutputLayer[label];
}
}
//Now we already have err in output layer, let's pass them back to other layers
fErrLayers = new List<double[][]>();
bErrLayers = new List<double[][]>();
for (int i = 0; i < numLayers; i++)
{
fErrLayers.Add(null);
bErrLayers.Add(null);
}
//Pass error from i+1 to i layer
SimpleLayer forwardLayer = forwardHiddenLayers[numLayers - 1];
SimpleLayer backwardLayer = backwardHiddenLayers[numLayers - 1];
double[][] errLayer = new double[numStates][];
Parallel.For(0, numStates, parallelOption, curState =>
{
errLayer[curState] = new double[forwardLayer.LayerSize];
forwardLayer.ComputeLayerErr(seqFinalOutput[curState], errLayer[curState], seqFinalOutput[curState].er);
});
fErrLayers[numLayers - 1] = errLayer;
bErrLayers[numLayers - 1] = errLayer;
// Forward
for (int i = numLayers - 2; i >= 0; i--)
{
forwardLayer = forwardHiddenLayers[i];
errLayer = new double[numStates][];
double[][] srcErrLayer = fErrLayers[i + 1];
Parallel.For(0, numStates, parallelOption, curState =>
{
int curState2 = numStates - curState - 1;
errLayer[curState2] = new double[forwardLayer.LayerSize];
forwardLayer.ComputeLayerErr(forwardHiddenLayers[i + 1], errLayer[curState2], srcErrLayer[curState2]);
});
fErrLayers[i] = errLayer;
}
// Backward
for (int i = numLayers - 2; i >= 0; i--)
{
backwardLayer = backwardHiddenLayers[i];
errLayer = new double[numStates][];
double[][] srcErrLayer = bErrLayers[i + 1];
Parallel.For(0, numStates, parallelOption, curState =>
{
errLayer[curState] = new double[backwardLayer.LayerSize];
backwardLayer.ComputeLayerErr(backwardHiddenLayers[i + 1], errLayer[curState], srcErrLayer[curState]);
});
bErrLayers[i] = errLayer;
}
}
public void UpdateBigramTransition(Sequence seq)
{
int OutputLayerSize = OutputLayer.LayerSize;
int numStates = seq.States.Length;
Matrix<double> m_DeltaBigramLM = new Matrix<double>(OutputLayerSize, OutputLayerSize);
for (int timeat = 1; timeat < numStates; timeat++)
{
double[] CRFSeqOutput_timeat = CRFSeqOutput[timeat];
double[] CRFSeqOutput_pre_timeat = CRFSeqOutput[timeat - 1];
for (int i = 0; i < OutputLayerSize; i++)
{
double CRFSeqOutput_timeat_i = CRFSeqOutput_timeat[i];
double[] CRFTagTransWeights_i = CRFTagTransWeights[i];
double[] m_DeltaBigramLM_i = m_DeltaBigramLM[i];
int j = 0;
Vector<double> vecCRFSeqOutput_timeat_i = new Vector<double>(CRFSeqOutput_timeat_i);
while (j < OutputLayerSize - Vector<double>.Count)
{
Vector<double> v1 = new Vector<double>(CRFTagTransWeights_i, j);
Vector<double> v2 = new Vector<double>(CRFSeqOutput_pre_timeat, j);
Vector<double> v = new Vector<double>(m_DeltaBigramLM_i, j);
v -= (v1 * vecCRFSeqOutput_timeat_i * v2);
v.CopyTo(m_DeltaBigramLM_i, j);
j += Vector<double>.Count;
}
while (j < OutputLayerSize)
{
m_DeltaBigramLM_i[j] -= (CRFTagTransWeights_i[j] * CRFSeqOutput_timeat_i * CRFSeqOutput_pre_timeat[j]);
j++;
}
}
int iTagId = seq.States[timeat].Label;
int iLastTagId = seq.States[timeat - 1].Label;
m_DeltaBigramLM[iTagId][iLastTagId] += 1;
}
//Update tag Bigram LM
for (int b = 0; b < OutputLayerSize; b++)
{
double[] vector_b = CRFTagTransWeights[b];
double[] vector_delta_b = m_DeltaBigramLM[b];
int a = 0;
while (a < OutputLayerSize - Vector<double>.Count)
{
Vector<double> v1 = new Vector<double>(vector_delta_b, a);
Vector<double> v = new Vector<double>(vector_b, a);
//Normalize delta
v1 = RNNHelper.NormalizeGradient(v1);
//Update weights
v += RNNHelper.vecNormalLearningRate * v1;
v.CopyTo(vector_b, a);
a += Vector<double>.Count;
}
while (a < OutputLayerSize)
{
vector_b[a] += RNNHelper.LearningRate * RNNHelper.NormalizeGradient(vector_delta_b[a]);
a++;
}
}
}
/// <summary>
/// Computing the output of each layer in the neural network
/// </summary>
/// <param name="pSequence"></param>
/// <param name="isTrain"></param>
/// <param name="layerList"></param>
/// <param name="rawOutputLayer"></param>
/// <returns></returns>
private SimpleLayer[] ComputeLayers(Sequence pSequence, bool isTrain, out List<SimpleLayer[]> layerList, out Matrix<double> rawOutputLayer)
{
layerList = new List<SimpleLayer[]>();
SimpleLayer[] layer = ComputeBottomLayer(pSequence, forwardHiddenLayers[0], backwardHiddenLayers[0]);
if (isTrain == true)
{
layerList.Add(layer);
}
for (int i = 1; i < forwardHiddenLayers.Count; i++)
{
layer = ComputeMiddleLayers(layer, forwardHiddenLayers[i], backwardHiddenLayers[i]);
if (isTrain == true)
{
layerList.Add(layer);
}
}
SimpleLayer[] seqFinalOutput = ComputeTopLayer(layer, out rawOutputLayer, isTrain);
return seqFinalOutput;
}
private int GetErrorTokenNum(Sequence seq, int[] predicted)
{
int tknErrCnt = 0;
int numStates = seq.States.Length;
for (int curState = 0; curState < numStates; curState++)
{
if (predicted[curState] != seq.States[curState].Label)
{
tknErrCnt++;
}
}
return tknErrCnt;
}
public void Add(Sequence sequence)
{
m_Data.Add(sequence);
}
public int[] DecodeCRF(Sequence seq)
{
//ys contains the output of RNN for each word
Matrix<double> ys = ProcessSequence(seq, RunningMode.Test);
return Viterbi(ys, seq.States.Length);
}
public override int[] learnSentenceForRNNCRF(Sequence pSequence)
{
//Reset the network
int numStates = pSequence.GetSize();
int[] predicted = new int[numStates];
//Predict output
Matrix m = InnerDecode(pSequence);
ForwardBackward(numStates, m);
//Get the best result
predicted = new int[numStates];
for (int i = 0; i < numStates; i++)
{
State state = pSequence.Get(i);
logp += Math.Log10(m_Diff[i][state.GetLabel()]);
counter++;
predicted[i] = GetBestZIndex(i);
}
UpdateBigramTransition(pSequence);
netReset();
forwardRNN.m_Diff = m_Diff;
backwardRNN.m_Diff = m_Diff;
double[] output_fnn = new double[L2];
double[] output_bnn = new double[L2];
Parallel.Invoke(() =>
{
//Learn forward network
for (int curState = 0; curState < numStates; curState++)
{
// error propogation
State state = pSequence.Get(curState);
forwardRNN.setInputLayer(state, curState, numStates, predicted_fnn);
forwardRNN.computeNet(state, output_fnn); //compute probability distribution
forwardRNN.learnNet(state, curState);
forwardRNN.LearnBackTime(state, numStates, curState);
forwardRNN.copyHiddenLayerToInput();
}
},
() =>
{
for (int curState = numStates - 1; curState >= 0; curState--)
{
// error propogation
State state = pSequence.Get(curState);
backwardRNN.setInputLayer(state, curState, numStates, predicted_bnn, false);
backwardRNN.computeNet(state, output_bnn); //compute probability distribution
backwardRNN.learnNet(state, curState);
backwardRNN.LearnBackTime(state, numStates, curState);
backwardRNN.copyHiddenLayerToInput();
}
});
return predicted;
}
public int[][] DecodeNBestCRF(Sequence seq, int N)
{
//ys contains the output of RNN for each word
Matrix<double> ys = ProcessSequence(seq, RunningMode.Test);
int n = seq.States.Length;
int K = OutputLayer.LayerSize;
Matrix<double> STP = CRFTagTransWeights;
PAIR<int, int>[,,] vPath = new PAIR<int, int>[n, K, N];
int DUMP_LABEL = -1;
double[,] vPreAlpha = new double[K, N];
double[,] vAlpha = new double[K, N];
int nStartTagIndex = 0;
int nEndTagIndex = 0;
double MIN_VALUE = double.MinValue;
//viterbi algorithm
for (int i = 0; i < K; i++)
{
for (int j = 0; j < N; j++)
{
vPreAlpha[i, j] = MIN_VALUE;
vPath[0, i, j] = new PAIR<int, int>(DUMP_LABEL, 0);
}
}
vPreAlpha[nStartTagIndex, 0] = ys[0][nStartTagIndex];
vPath[0, nStartTagIndex, 0].first = nStartTagIndex;
AdvUtils.PriorityQueue<double, PAIR<int, int>> q = new AdvUtils.PriorityQueue<double, PAIR<int, int>>();
for (int t = 1; t < n; t++)
{
for (int j = 0; j < K; j++)
{
while (q.Count() > 0)
q.Dequeue();
double _stp = STP[j][0];
double _y = ys[t][j];
for (int k = 0; k < N; k++)
{
double score = vPreAlpha[0, k] + _stp + _y;
q.Enqueue(score, new PAIR<int, int>(0, k));
}
for (int i = 1; i < K; i++)
{
_stp = STP[j][i];
for (int k = 0; k < N; k++)
{
double score = vPreAlpha[i, k] + _stp + _y;
if (score <= q.Peek().Key)
break;
q.Dequeue();
q.Enqueue(score, new PAIR<int, int>(i, k));
}
}
int idx = N - 1;
while (q.Count() > 0)
{
vAlpha[j, idx] = q.Peek().Key;
vPath[t, j, idx] = q.Peek().Value;
idx--;
q.Dequeue();
}
}
vPreAlpha = vAlpha;
vAlpha = new double[K, N];
}
//backtrace to get the n-best result path
int[][] vTagOutput = new int[N][];
for (int i = 0; i < N; i++)
{
vTagOutput[i] = new int[n];
}
for (int k = 0; k < N; k++)
{
vTagOutput[k][n - 1] = nEndTagIndex;
PAIR<int, int> decision = new PAIR<int, int>(nEndTagIndex, k);
for (int t = n - 2; t >= 0; t--)
{
vTagOutput[k][t] = vPath[t + 1, decision.first, decision.second].first;
decision = vPath[t + 1, decision.first, decision.second];
}
}
return vTagOutput;
}
public override Matrix InnerDecode(Sequence pSequence)
{
//Reset the network
netReset();
int numStates = pSequence.GetSize();
predicted_fnn = new int[numStates];
predicted_bnn = new int[numStates];
Matrix mForward = new Matrix(numStates, forwardRNN.L2);
Matrix mBackward = new Matrix(numStates, backwardRNN.L2);
Parallel.Invoke(() =>
{
//Computing forward RNN
for (int curState = 0; curState < numStates; curState++)
{
State state = pSequence.Get(curState);
forwardRNN.setInputLayer(state, curState, numStates, predicted_fnn);
forwardRNN.computeNet(state, mForward[curState]); //compute probability distribution
predicted_fnn[curState] = forwardRNN.GetBestOutputIndex();
forwardRNN.copyHiddenLayerToInput();
}
},
() =>
{
//Computing backward RNN
for (int curState = numStates - 1; curState >= 0; curState--)
{
State state = pSequence.Get(curState);
backwardRNN.setInputLayer(state, curState, numStates, predicted_bnn, false);
backwardRNN.computeNet(state, mBackward[curState]); //compute probability distribution
predicted_bnn[curState] = backwardRNN.GetBestOutputIndex();
backwardRNN.copyHiddenLayerToInput();
}
});
//Merge forward and backward
Matrix m = new Matrix(numStates, forwardRNN.L2);
for (int curState = 0; curState < numStates; curState++)
{
for (int i = 0; i < forwardRNN.L2; i++)
{
m[curState][i] = mForward[curState][i] + mBackward[curState][i];
}
}
return m;
}
public neuron[][] InnerDecode(Sequence pSequence, out Matrix<neuron> outputHiddenLayer, out Matrix<double> rawOutputLayer)
{
int numStates = pSequence.States.Length;
Matrix<double> mForward = null;
Matrix<double> mBackward = null;
//Reset the network
netReset(false);
Parallel.Invoke(() =>
{
//Computing forward RNN
mForward = new Matrix<double>(numStates, forwardRNN.L1);
for (int curState = 0; curState < numStates; curState++)
{
State state = pSequence.States[curState];
forwardRNN.setInputLayer(state, curState, numStates, null);
forwardRNN.computeNet(state, null); //compute probability distribution
forwardRNN.GetHiddenLayer(mForward, curState);
}
},
() =>
{
//Computing backward RNN
mBackward = new Matrix<double>(numStates, backwardRNN.L1);
for (int curState = numStates - 1; curState >= 0; curState--)
{
State state = pSequence.States[curState];
backwardRNN.setInputLayer(state, curState, numStates, null, false);
backwardRNN.computeNet(state, null); //compute probability distribution
backwardRNN.GetHiddenLayer(mBackward, curState);
}
});
//Merge forward and backward
Matrix<neuron> mergedHiddenLayer = new Matrix<neuron>(numStates, forwardRNN.L1);
Parallel.For(0, numStates, parallelOption, curState =>
{
for (int i = 0; i < forwardRNN.L1; i++)
{
mergedHiddenLayer[curState][i].cellOutput = mForward[curState][i] + mBackward[curState][i];
}
});
//Calculate output layer
Matrix<double> tmp_rawOutputLayer = new Matrix<double>(numStates, L2);
neuron[][] seqOutput = new neuron[numStates][];
Parallel.For(0, numStates, parallelOption, curState =>
{
seqOutput[curState] = new neuron[L2];
matrixXvectorADD(seqOutput[curState], mergedHiddenLayer[curState], Hidden2OutputWeight, 0, L2, 0, L1, 0);
for (int i = 0; i < L2; i++)
{
tmp_rawOutputLayer[curState][i] = seqOutput[curState][i].cellOutput;
}
//Activation on output layer
SoftmaxLayer(seqOutput[curState]);
});
outputHiddenLayer = mergedHiddenLayer;
rawOutputLayer = tmp_rawOutputLayer;
return seqOutput;
}
