/// <summary> /// Extract features from source sequence /// </summary> /// <param name="decoder"></param> /// <param name="srcSequence"></param> /// <param name="targetSparseFeatureSize"></param> /// <param name="srcHiddenAvgOutput"></param> /// <param name="srcSparseFeatures"></param> private void ExtractSourceSentenceFeature(RNNDecoder decoder, Sequence srcSequence, int targetSparseFeatureSize, List <float[]> srcDenseFeatureGroups, SparseVector srcSparseFeatures) { //Extract dense features from source sequence var srcOutputs = decoder.ComputeTopHiddenLayerOutput(srcSequence); int srcSequenceLength = srcOutputs.Length - 1; srcDenseFeatureGroups.Add(srcOutputs[srcSequenceLength]); if (numSrcDenseFeatureGroups > 1) { srcDenseFeatureGroups.Add(srcOutputs[0]); } if (numSrcDenseFeatureGroups > 2) { float srcOffsetPerBlock = (float)srcSequenceLength / (float)(numSrcDenseFeatureGroups - 1); if (srcOffsetPerBlock < 1.0) { srcOffsetPerBlock = 1.0f; } float idx = srcOffsetPerBlock; while (srcDenseFeatureGroups.Count < numSrcDenseFeatureGroups && idx < srcSequenceLength) { srcDenseFeatureGroups.Add(srcOutputs[(int)idx]); idx += srcOffsetPerBlock; } } //Extract sparse features from source sequence Dictionary <int, float> srcSparseFeaturesDict = new Dictionary <int, float>(); for (var i = 0; i < srcSequence.States.Length; i++) { foreach (var kv in srcSequence.States[i].SparseFeature) { var srcSparseFeatureIndex = kv.Key + targetSparseFeatureSize; if (srcSparseFeaturesDict.ContainsKey(srcSparseFeatureIndex) == false) { srcSparseFeaturesDict.Add(srcSparseFeatureIndex, kv.Value); } else { srcSparseFeaturesDict[srcSparseFeatureIndex] += kv.Value; } } } srcSparseFeatures.SetLength(srcSequence.SparseFeatureSize + targetSparseFeatureSize); srcSparseFeatures.AddKeyValuePairData(srcSparseFeaturesDict); }
protected virtual int[] TrainSequencePair(ISequence sequence, RunningMode runningMode, bool outputRawScore, out Matrix <float> m) { SequencePair pSequence = sequence as SequencePair; var tgtSequence = pSequence.tgtSequence; //Reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(); } Sequence srcSequence; //Extract features from source sentences srcSequence = pSequence.autoEncoder.Config.BuildSequence(pSequence.srcSentence); ExtractSourceSentenceFeature(pSequence.autoEncoder, srcSequence, tgtSequence.SparseFeatureSize); var numStates = pSequence.tgtSequence.States.Length; var numLayers = HiddenLayerList.Count; var predicted = new int[numStates]; var previousLables = new int[numStates]; m = outputRawScore ? new Matrix <float>(numStates, OutputLayer.LayerSize) : null; //Set target sentence labels into short list in output layer OutputLayer.LabelShortList.Clear(); foreach (var state in tgtSequence.States) { OutputLayer.LabelShortList.Add(state.Label); } CreateDenseFeatureList(); for (int i = 0; i < numLayers; i++) { srcHiddenAvgOutput.CopyTo(denseFeaturesList[i], 0); } srcHiddenAvgOutput.CopyTo(denseFeaturesList[numLayers], 0); var sparseVector = new SparseVector(); for (var curState = 0; curState < numStates; curState++) { //Build runtime features var state = tgtSequence.States[curState]; SetRuntimeFeatures(state, curState, numStates, (runningMode == RunningMode.Training) ? previousLables : predicted); //Build sparse features for all layers sparseVector.Clean(); sparseVector.SetLength(tgtSequence.SparseFeatureSize + srcSequence.SparseFeatureSize); sparseVector.AddKeyValuePairData(state.SparseFeature); sparseVector.AddKeyValuePairData(srcSparseFeatures); //Compute first layer state.DenseFeature.CopyTo().CopyTo(denseFeaturesList[0], srcHiddenAvgOutput.Length); HiddenLayerList[0].ForwardPass(sparseVector, denseFeaturesList[0]); //Compute middle layers for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer HiddenLayerList[i - 1].Cells.CopyTo(denseFeaturesList[i], srcHiddenAvgOutput.Length); HiddenLayerList[i].ForwardPass(sparseVector, denseFeaturesList[i]); } //Compute output layer HiddenLayerList[numLayers - 1].Cells.CopyTo(denseFeaturesList[numLayers], srcHiddenAvgOutput.Length); OutputLayer.ForwardPass(sparseVector, denseFeaturesList[numLayers]); if (m != null) { OutputLayer.Cells.CopyTo(m[curState], 0); } predicted[curState] = OutputLayer.GetBestOutputIndex(); if (runningMode == RunningMode.Training) { previousLables[curState] = state.Label; // error propogation OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState); //propogate errors to each layer from output layer to input layer HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer); for (var i = numLayers - 2; i >= 0; i--) { HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]); } //Update net weights OutputLayer.BackwardPass(); for (var i = 0; i < numLayers; i++) { HiddenLayerList[i].BackwardPass(); } } } return(predicted); }
protected int[] PredictTargetSentence(Sentence sentence, Config featurizer, out Matrix <float> m) { m = null; var curState = featurizer.BuildState(new[] { "<s>" }); curState.Label = featurizer.TagSet.GetIndex("<s>"); //Reset all layers foreach (var layer in HiddenLayerList) { layer.Reset(); } //Extract features from source sentence var srcSequence = featurizer.Seq2SeqAutoEncoder.Config.BuildSequence(sentence); ExtractSourceSentenceFeature(featurizer.Seq2SeqAutoEncoder, srcSequence, curState.SparseFeature.Length); var numLayers = HiddenLayerList.Count; var predicted = new List <int> { curState.Label }; CreateDenseFeatureList(); for (int i = 0; i < numLayers; i++) { srcHiddenAvgOutput.CopyTo(denseFeaturesList[i], 0); } srcHiddenAvgOutput.CopyTo(denseFeaturesList[numLayers], 0); var sparseVector = new SparseVector(); while (true) { //Build sparse features sparseVector.Clean(); sparseVector.SetLength(curState.SparseFeature.Length + srcSequence.SparseFeatureSize); sparseVector.AddKeyValuePairData(curState.SparseFeature); sparseVector.AddKeyValuePairData(srcSparseFeatures); //Compute first layer curState.DenseFeature.CopyTo().CopyTo(denseFeaturesList[0], srcHiddenAvgOutput.Length); HiddenLayerList[0].ForwardPass(sparseVector, denseFeaturesList[0]); //Compute middle layers for (var i = 1; i < numLayers; i++) { //We use previous layer's output as dense feature for current layer HiddenLayerList[i - 1].Cells.CopyTo(denseFeaturesList[i], srcHiddenAvgOutput.Length); HiddenLayerList[i].ForwardPass(sparseVector, denseFeaturesList[i]); } //Compute output layer HiddenLayerList[numLayers - 1].Cells.CopyTo(denseFeaturesList[numLayers], srcHiddenAvgOutput.Length); OutputLayer.ForwardPass(sparseVector, denseFeaturesList[numLayers]); var nextTagId = OutputLayer.GetBestOutputIndex(); var nextWord = featurizer.TagSet.GetTagName(nextTagId); curState = featurizer.BuildState(new[] { nextWord }); curState.Label = nextTagId; predicted.Add(nextTagId); if (nextWord == "</s>" || predicted.Count >= 100) { break; } } return(predicted.ToArray()); }