protected virtual int[] TrainSequencePair(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix <float> m) { SequencePair pSequence = sequence as SequencePair; var tgtSequence = pSequence.tgtSequence; //Reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(); } Sequence srcSequence; //Extract features from source sentences srcSequence = pSequence.autoEncoder.Config.BuildSequence(pSequence.srcSentence); ExtractSourceSentenceFeature(pSequence.autoEncoder, srcSequence, tgtSequence.SparseFeatureSize); var numStates = pSequence.tgtSequence.States.Length; var numLayers = HiddenLayerList.Count; var predicted = new int[numStates]; var previousLables = new int[numStates]; m = outputRawScore ? new Matrix <float>(numStates, OutputLayer.LayerSize) : null; //Set target sentence labels into short list in output layer OutputLayer.LabelShortList.Clear(); foreach (var state in tgtSequence.States) { OutputLayer.LabelShortList.Add(state.Label); } CreateDenseFeatureList(); for (int i = 0; i < numLayers; i++) { srcHiddenAvgOutput.CopyTo(denseFeaturesList[i], 0); } srcHiddenAvgOutput.CopyTo(denseFeaturesList[numLayers], 0); var sparseVector = new SparseVector(); for (var curState = 0; curState < numStates; curState++) { //Build runtime features var state = tgtSequence.States[curState]; SetRuntimeFeatures(state, curState, numStates, (runningMode == RunningMode.Training) ? previousLables : predicted); //Build sparse features for all layers sparseVector.Clean(); sparseVector.SetLength(tgtSequence.SparseFeatureSize + srcSequence.SparseFeatureSize); sparseVector.AddKeyValuePairData(state.SparseFeature); sparseVector.AddKeyValuePairData(srcSparseFeatures); //Compute first layer state.DenseFeature.CopyTo().CopyTo(denseFeaturesList[0], srcHiddenAvgOutput.Length); HiddenLayerList[0].ForwardPass(sparseVector, denseFeaturesList[0]); //Compute middle layers for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer HiddenLayerList[i - 1].Cells.CopyTo(denseFeaturesList[i], srcHiddenAvgOutput.Length); HiddenLayerList[i].ForwardPass(sparseVector, denseFeaturesList[i]); } //Compute output layer HiddenLayerList[numLayers - 1].Cells.CopyTo(denseFeaturesList[numLayers], srcHiddenAvgOutput.Length); OutputLayer.ForwardPass(sparseVector, denseFeaturesList[numLayers]); if (m != null) { OutputLayer.Cells.CopyTo(m[curState], 0); } predicted[curState] = OutputLayer.GetBestOutputIndex(); if (runningMode == RunningMode.Training) { previousLables[curState] = state.Label; // error propogation OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState); //propogate errors to each layer from output layer to input layer HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer); for (var i = numLayers - 2; i >= 0; i--) { HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]); } //Update net weights OutputLayer.BackwardPass(); for (var i = 0; i < numLayers; i++) { HiddenLayerList[i].BackwardPass(); } } } return(predicted); }
protected int[] PredictTargetSentence(Sentence sentence, Config featurizer, out Matrix <float> m) { m = null; var curState = featurizer.BuildState(new[] { "<s>" }); curState.Label = featurizer.TagSet.GetIndex("<s>"); //Reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(); } //Extract features from source sentence var srcSequence = featurizer.Seq2SeqAutoEncoder.Config.BuildSequence(sentence); ExtractSourceSentenceFeature(featurizer.Seq2SeqAutoEncoder, srcSequence, curState.SparseFeature.Length); var numLayers = HiddenLayerList.Count; var predicted = new List <int> { curState.Label }; CreateDenseFeatureList(); for (int i = 0; i < numLayers; i++) { srcHiddenAvgOutput.CopyTo(denseFeaturesList[i], 0); } srcHiddenAvgOutput.CopyTo(denseFeaturesList[numLayers], 0); var sparseVector = new SparseVector(); while (true) { //Build sparse features sparseVector.Clean(); sparseVector.SetLength(curState.SparseFeature.Length + srcSequence.SparseFeatureSize); sparseVector.AddKeyValuePairData(curState.SparseFeature); sparseVector.AddKeyValuePairData(srcSparseFeatures); //Compute first layer curState.DenseFeature.CopyTo().CopyTo(denseFeaturesList[0], srcHiddenAvgOutput.Length); HiddenLayerList[0].ForwardPass(sparseVector, denseFeaturesList[0]); //Compute middle layers for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer HiddenLayerList[i - 1].Cells.CopyTo(denseFeaturesList[i], srcHiddenAvgOutput.Length); HiddenLayerList[i].ForwardPass(sparseVector, denseFeaturesList[i]); } //Compute output layer HiddenLayerList[numLayers - 1].Cells.CopyTo(denseFeaturesList[numLayers], srcHiddenAvgOutput.Length); OutputLayer.ForwardPass(sparseVector, denseFeaturesList[numLayers]); var nextTagId = OutputLayer.GetBestOutputIndex(); var nextWord = featurizer.TagSet.GetTagName(nextTagId); curState = featurizer.BuildState(new[] { nextWord }); curState.Label = nextTagId; predicted.Add(nextTagId); if (nextWord == "</s>" || predicted.Count >= 100) { break; } } return(predicted.ToArray()); }