protected virtual int[] TrainSequencePair(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix <float> m) { SequencePair pSequence = sequence as SequencePair; var tgtSequence = pSequence.tgtSequence; //Reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(); } Sequence srcSequence; //Extract features from source sentences srcSequence = pSequence.autoEncoder.Config.BuildSequence(pSequence.srcSentence); ExtractSourceSentenceFeature(pSequence.autoEncoder, srcSequence, tgtSequence.SparseFeatureSize); var numStates = pSequence.tgtSequence.States.Length; var numLayers = HiddenLayerList.Count; var predicted = new int[numStates]; var previousLables = new int[numStates]; m = outputRawScore ? new Matrix <float>(numStates, OutputLayer.LayerSize) : null; //Set target sentence labels into short list in output layer OutputLayer.LabelShortList.Clear(); foreach (var state in tgtSequence.States) { OutputLayer.LabelShortList.Add(state.Label); } CreateDenseFeatureList(); for (int i = 0; i < numLayers; i++) { srcHiddenAvgOutput.CopyTo(denseFeaturesList[i], 0); } srcHiddenAvgOutput.CopyTo(denseFeaturesList[numLayers], 0); var sparseVector = new SparseVector(); for (var curState = 0; curState < numStates; curState++) { //Build runtime features var state = tgtSequence.States[curState]; SetRuntimeFeatures(state, curState, numStates, (runningMode == RunningMode.Training) ? previousLables : predicted); //Build sparse features for all layers sparseVector.Clean(); sparseVector.SetLength(tgtSequence.SparseFeatureSize + srcSequence.SparseFeatureSize); sparseVector.AddKeyValuePairData(state.SparseFeature); sparseVector.AddKeyValuePairData(srcSparseFeatures); //Compute first layer state.DenseFeature.CopyTo().CopyTo(denseFeaturesList[0], srcHiddenAvgOutput.Length); HiddenLayerList[0].ForwardPass(sparseVector, denseFeaturesList[0]); //Compute middle layers for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer HiddenLayerList[i - 1].Cells.CopyTo(denseFeaturesList[i], srcHiddenAvgOutput.Length); HiddenLayerList[i].ForwardPass(sparseVector, denseFeaturesList[i]); } //Compute output layer HiddenLayerList[numLayers - 1].Cells.CopyTo(denseFeaturesList[numLayers], srcHiddenAvgOutput.Length); OutputLayer.ForwardPass(sparseVector, denseFeaturesList[numLayers]); if (m != null) { OutputLayer.Cells.CopyTo(m[curState], 0); } predicted[curState] = OutputLayer.GetBestOutputIndex(); if (runningMode == RunningMode.Training) { previousLables[curState] = state.Label; // error propogation OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState); //propogate errors to each layer from output layer to input layer HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer); for (var i = numLayers - 2; i >= 0; i--) { HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]); } //Update net weights OutputLayer.BackwardPass(); for (var i = 0; i < numLayers; i++) { HiddenLayerList[i].BackwardPass(); } } } return(predicted); }
public override int[] ProcessSequenceCRF(Sequence pSequence, RunningMode runningMode) { var numStates = pSequence.States.Length; var numLayers = HiddenLayerList.Count; //Get network output without CRF Matrix <float> nnOutput; ProcessSequence(pSequence, RunningMode.Test, true, out nnOutput); //Compute CRF result ForwardBackward(numStates, nnOutput); //Compute best path in CRF result var predicted = Viterbi(nnOutput, numStates); if (runningMode == RunningMode.Training) { //Update tag bigram transition for CRF model UpdateBigramTransition(pSequence); //Reset all layer states foreach (var layer in HiddenLayerList) { layer.Reset(); } for (var curState = 0; curState < numStates; curState++) { // error propogation var state = pSequence.States[curState]; SetRuntimeFeatures(state, curState, numStates, null); HiddenLayerList[0].SetRunningMode(runningMode); HiddenLayerList[0].ForwardPass(state.SparseFeature, state.DenseFeature.CopyTo()); for (var i = 1; i < numLayers; i++) { HiddenLayerList[i].SetRunningMode(runningMode); HiddenLayerList[i].ForwardPass(state.SparseFeature, HiddenLayerList[i - 1].Cells); } OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState); HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer); for (var i = numLayers - 2; i >= 0; i--) { HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]); } //Update net weights OutputLayer.BackwardPass(); for (var i = 0; i < numLayers; i++) { HiddenLayerList[i].BackwardPass(); } } } return(predicted); }
private int[] TrainSequencePair(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix <float> m) { SequencePair pSequence = sequence as SequencePair; var tgtSequence = pSequence.tgtSequence; //Reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(); } Sequence srcSequence; //Extract features from source sentences srcSequence = pSequence.autoEncoder.Config.BuildSequence(pSequence.srcSentence); List <float[]> srcDenseFeatureGorups = new List <float[]>(); SparseVector srcSparseFeatures = new SparseVector(); ExtractSourceSentenceFeature(pSequence.autoEncoder, srcSequence, tgtSequence.SparseFeatureSize, srcDenseFeatureGorups, srcSparseFeatures); var numStates = pSequence.tgtSequence.States.Length; var numLayers = HiddenLayerList.Count; var predicted = new int[numStates]; m = outputRawScore ? new Matrix <float>(numStates, OutputLayer.LayerSize) : null; //Set target sentence labels into short list in output layer OutputLayer.LabelShortList.Clear(); foreach (var state in tgtSequence.States) { OutputLayer.LabelShortList.Add(state.Label); } //Set sparse feature group from source sequence sparseFeatureGorups.Clear(); sparseFeatureGorups.Add(srcSparseFeatures); sparseFeatureGorups.Add(null); int targetSparseFeatureIndex = sparseFeatureGorups.Count - 1; //Set dense feature groups from source sequence for (var i = 0; i < numLayers; i++) { denseFeatureGroupsList[i].Clear(); denseFeatureGroupsList[i].AddRange(srcDenseFeatureGorups); denseFeatureGroupsList[i].Add(null); } denseFeatureGroupsOutputLayer.Clear(); denseFeatureGroupsOutputLayer.AddRange(srcDenseFeatureGorups); denseFeatureGroupsOutputLayer.Add(null); int targetDenseFeatureIndex = denseFeatureGroupsOutputLayer.Count - 1; for (var curState = 0; curState < numStates; curState++) { var state = tgtSequence.States[curState]; //Set sparse feature groups sparseFeatureGorups[targetSparseFeatureIndex] = state.SparseFeature; //Compute first layer denseFeatureGroupsList[0][targetDenseFeatureIndex] = state.DenseFeature.CopyTo(); HiddenLayerList[0].ForwardPass(sparseFeatureGorups, denseFeatureGroupsList[0]); //Compute middle layers for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer denseFeatureGroupsList[i][targetDenseFeatureIndex] = HiddenLayerList[i - 1].Cells; HiddenLayerList[i].ForwardPass(sparseFeatureGorups, denseFeatureGroupsList[i]); } //Compute output layer denseFeatureGroupsOutputLayer[targetDenseFeatureIndex] = HiddenLayerList[numLayers - 1].Cells; OutputLayer.ForwardPass(sparseFeatureGorups, denseFeatureGroupsOutputLayer); if (m != null) { OutputLayer.Cells.CopyTo(m[curState], 0); } predicted[curState] = OutputLayer.GetBestOutputIndex(); if (runningMode == RunningMode.Training) { // error propogation OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState); //propogate errors to each layer from output layer to input layer HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer); for (var i = numLayers - 2; i >= 0; i--) { HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]); } //Update net weights OutputLayer.BackwardPass(); for (var i = 0; i < numLayers; i++) { HiddenLayerList[i].BackwardPass(); } } } return(predicted); }
public override int[] ProcessSequence(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix <float> m) { Sequence pSequence = sequence as Sequence; var numStates = pSequence.States.Length; var numLayers = HiddenLayerList.Count; m = outputRawScore ? new Matrix <float>(numStates, OutputLayer.LayerSize) : null; var predicted = new int[numStates]; var isTraining = runningMode == RunningMode.Training; //reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(); } //Set current sentence labels into short list in output layer OutputLayer.LabelShortList.Clear(); foreach (var state in pSequence.States) { OutputLayer.LabelShortList.Add(state.Label); } for (var curState = 0; curState < numStates; curState++) { //Compute first layer var state = pSequence.States[curState]; SetRuntimeFeatures(state, curState, numStates, predicted); HiddenLayerList[0].ForwardPass(state.SparseFeature, state.DenseFeature.CopyTo()); //Compute each layer for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer HiddenLayerList[i].ForwardPass(state.SparseFeature, HiddenLayerList[i - 1].Cells); } //Compute output layer OutputLayer.ForwardPass(state.SparseFeature, HiddenLayerList[numLayers - 1].Cells); if (m != null) { OutputLayer.Cells.CopyTo(m[curState], 0); } predicted[curState] = OutputLayer.GetBestOutputIndex(); if (runningMode == RunningMode.Training) { // error propogation OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState); //propogate errors to each layer from output layer to input layer HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer); for (var i = numLayers - 2; i >= 0; i--) { HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]); } //Update net weights OutputLayer.BackwardPass(); for (var i = 0; i < numLayers; i++) { HiddenLayerList[i].BackwardPass(); } } } return(predicted); }
public override int[] ProcessSeq2Seq(SequencePair pSequence, RunningMode runningMode) { var tgtSequence = pSequence.tgtSequence; var isTraining = runningMode == RunningMode.Training; //Reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(isTraining); } //Extract features from source sentences var srcSequence = pSequence.autoEncoder.Config.BuildSequence(pSequence.srcSentence); float[] srcHiddenAvgOutput; Dictionary <int, float> srcSparseFeatures; ExtractSourceSentenceFeature(pSequence.autoEncoder, srcSequence, tgtSequence.SparseFeatureSize, out srcHiddenAvgOutput, out srcSparseFeatures); var numStates = pSequence.tgtSequence.States.Length; var numLayers = HiddenLayerList.Count; var predicted = new int[numStates]; //Set target sentence labels into short list in output layer OutputLayer.LabelShortList = new List <int>(); foreach (var state in tgtSequence.States) { OutputLayer.LabelShortList.Add(state.Label); } for (var curState = 0; curState < numStates; curState++) { //Build runtime features var state = tgtSequence.States[curState]; SetRuntimeFeatures(state, curState, numStates, predicted); //Build sparse features for all layers var sparseVector = new SparseVector(); sparseVector.SetLength(tgtSequence.SparseFeatureSize + srcSequence.SparseFeatureSize); sparseVector.AddKeyValuePairData(state.SparseFeature); sparseVector.AddKeyValuePairData(srcSparseFeatures); //Compute first layer var denseFeatures = RNNHelper.ConcatenateVector(state.DenseFeature, srcHiddenAvgOutput); HiddenLayerList[0].ForwardPass(sparseVector, denseFeatures, isTraining); //Compute middle layers for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[i - 1].Cell, srcHiddenAvgOutput); HiddenLayerList[i].ForwardPass(sparseVector, denseFeatures, isTraining); } //Compute output layer denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[numLayers - 1].Cell, srcHiddenAvgOutput); OutputLayer.ForwardPass(sparseVector, denseFeatures, isTraining); OutputLayer.Softmax(isTraining); predicted[curState] = OutputLayer.GetBestOutputIndex(isTraining); if (runningMode != RunningMode.Test) { logp += Math.Log10(OutputLayer.Cell[state.Label] + 0.0001); } if (runningMode == RunningMode.Training) { // error propogation OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState); //propogate errors to each layer from output layer to input layer HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer); for (var i = numLayers - 2; i >= 0; i--) { HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]); } //Update net weights Parallel.Invoke(() => { OutputLayer.BackwardPass(numStates, curState); }, () => { Parallel.For(0, numLayers, parallelOption, i => { HiddenLayerList[i].BackwardPass(numStates, curState); }); }); } } return(predicted); }