public override int[] ProcessSeq2Seq(SequencePair pSequence, RunningMode runningMode)
{
var tgtSequence = pSequence.tgtSequence;
var isTraining = runningMode == RunningMode.Training;
//Reset all layers
foreach (var layer in HiddenLayerList)
{
layer.Reset(isTraining);
}
//Extract features from source sentences
var srcSequence = pSequence.autoEncoder.Config.BuildSequence(pSequence.srcSentence);
float[] srcHiddenAvgOutput;
Dictionary <int, float> srcSparseFeatures;
ExtractSourceSentenceFeature(pSequence.autoEncoder, srcSequence, tgtSequence.SparseFeatureSize,
out srcHiddenAvgOutput, out srcSparseFeatures);
var numStates = pSequence.tgtSequence.States.Length;
var numLayers = HiddenLayerList.Count;
var predicted = new int[numStates];
//Set target sentence labels into short list in output layer
OutputLayer.LabelShortList = new List <int>();
foreach (var state in tgtSequence.States)
{
OutputLayer.LabelShortList.Add(state.Label);
}
for (var curState = 0; curState < numStates; curState++)
{
//Build runtime features
var state = tgtSequence.States[curState];
SetRuntimeFeatures(state, curState, numStates, predicted);
//Build sparse features for all layers
var sparseVector = new SparseVector();
sparseVector.SetLength(tgtSequence.SparseFeatureSize + srcSequence.SparseFeatureSize);
sparseVector.AddKeyValuePairData(state.SparseFeature);
sparseVector.AddKeyValuePairData(srcSparseFeatures);
//Compute first layer
var denseFeatures = RNNHelper.ConcatenateVector(state.DenseFeature, srcHiddenAvgOutput);
HiddenLayerList[0].ForwardPass(sparseVector, denseFeatures, isTraining);
//Compute middle layers
for (var i = 1; i < numLayers; i++)
{
//We use previous layer's output as dense feature for current layer
denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[i - 1].Cell, srcHiddenAvgOutput);
HiddenLayerList[i].ForwardPass(sparseVector, denseFeatures, isTraining);
}
//Compute output layer
denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[numLayers - 1].Cell,
srcHiddenAvgOutput);
OutputLayer.ForwardPass(sparseVector, denseFeatures, isTraining);
OutputLayer.Softmax(isTraining);
predicted[curState] = OutputLayer.GetBestOutputIndex(isTraining);
if (runningMode != RunningMode.Test)
{
logp += Math.Log10(OutputLayer.Cell[state.Label] + 0.0001);
}
if (runningMode == RunningMode.Training)
{
// error propogation
OutputLayer.ComputeLayerErr(CRFSeqOutput, state, curState);
//propogate errors to each layer from output layer to input layer
HiddenLayerList[numLayers - 1].ComputeLayerErr(OutputLayer);
for (var i = numLayers - 2; i >= 0; i--)
{
HiddenLayerList[i].ComputeLayerErr(HiddenLayerList[i + 1]);
}
//Update net weights
Parallel.Invoke(() => { OutputLayer.BackwardPass(numStates, curState); },
() =>
{
Parallel.For(0, numLayers, parallelOption,
i => { HiddenLayerList[i].BackwardPass(numStates, curState); });
});
}
}
return(predicted);
}
void ExtractSparseFeature(int currentState, int numStates, List <string[]> features, State pState)
{
Dictionary <int, float> sparseFeature = new Dictionary <int, float>();
int start = 0;
var fc = FeatureContext;
//Extract TFeatures in given context window
if (TFeaturizer != null)
{
if (fc.ContainsKey(TFEATURE_CONTEXT) == true)
{
List <int> v = fc[TFEATURE_CONTEXT];
for (int j = 0; j < v.Count; j++)
{
int offset = TruncPosition(currentState + v[j], 0, numStates);
List <int> tfeatureList = TFeaturizer.GetFeatureIds(features, offset);
foreach (int featureId in tfeatureList)
{
if (TFeatureWeightType == TFEATURE_WEIGHT_TYPE_ENUM.BINARY)
{
sparseFeature[start + featureId] = 1;
}
else
{
if (sparseFeature.ContainsKey(start + featureId) == false)
{
sparseFeature.Add(start + featureId, 1);
}
else
{
sparseFeature[start + featureId]++;
}
}
}
start += TFeaturizer.GetFeatureSize();
}
}
}
// Create place hold for run time feature
// The real feature value is calculated at run time
if (fc.ContainsKey(RT_FEATURE_CONTEXT) == true)
{
List <int> v = fc[RT_FEATURE_CONTEXT];
pState.RuntimeFeatures = new PriviousLabelFeature[v.Count];
for (int j = 0; j < v.Count; j++)
{
if (v[j] < 0)
{
pState.AddRuntimeFeaturePlacehold(j, v[j], sparseFeature.Count, start);
sparseFeature[start] = 0; //Placehold a position
start += TagSet.GetSize();
}
else
{
throw new Exception("The offset of run time feature should be negative.");
}
}
}
SparseVector spSparseFeature = pState.SparseFeature;
spSparseFeature.SetLength(SparseFeatureSize);
spSparseFeature.AddKeyValuePairData(sparseFeature);
}
public override int[] TestSeq2Seq(Sentence srcSentence, Config featurizer)
{
var curState = featurizer.BuildState(new[] { "<s>" });
curState.Label = featurizer.TagSet.GetIndex("<s>");
//Reset all layers
foreach (var layer in HiddenLayerList)
{
layer.Reset(false);
}
//Extract features from source sentence
var srcSequence = featurizer.Seq2SeqAutoEncoder.Config.BuildSequence(srcSentence);
float[] srcHiddenAvgOutput;
Dictionary <int, float> srcSparseFeatures;
ExtractSourceSentenceFeature(featurizer.Seq2SeqAutoEncoder, srcSequence, curState.SparseFeature.Length,
out srcHiddenAvgOutput, out srcSparseFeatures);
var numLayers = HiddenLayerList.Count;
var predicted = new List <int> {
curState.Label
};
while (true)
{
//Build sparse features
var sparseVector = new SparseVector();
sparseVector.SetLength(curState.SparseFeature.Length + srcSequence.SparseFeatureSize);
sparseVector.AddKeyValuePairData(curState.SparseFeature);
sparseVector.AddKeyValuePairData(srcSparseFeatures);
//Compute first layer
var denseFeatures = RNNHelper.ConcatenateVector(curState.DenseFeature, srcHiddenAvgOutput);
HiddenLayerList[0].ForwardPass(sparseVector, denseFeatures, false);
//Compute middle layers
for (var i = 1; i < numLayers; i++)
{
//We use previous layer's output as dense feature for current layer
denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[i - 1].Cell, srcHiddenAvgOutput);
HiddenLayerList[i].ForwardPass(sparseVector, denseFeatures, false);
}
//Compute output layer
denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[numLayers - 1].Cell,
srcHiddenAvgOutput);
OutputLayer.ForwardPass(sparseVector, denseFeatures, false);
OutputLayer.Softmax(false);
var nextTagId = OutputLayer.GetBestOutputIndex(false);
var nextWord = featurizer.TagSet.GetTagName(nextTagId);
curState = featurizer.BuildState(new[] { nextWord });
curState.Label = nextTagId;
predicted.Add(nextTagId);
if (nextWord == "</s>" || predicted.Count >= 100)
{
break;
}
}
return(predicted.ToArray());
}
public override int[] TestSeq2Seq(Sentence srcSentence, Featurizer featurizer)
{
State curState = featurizer.ExtractFeatures(new string[] { "<s>" });
curState.Label = featurizer.TagSet.GetIndex("<s>");
//Reset all layers
foreach (SimpleLayer layer in HiddenLayerList)
{
layer.netReset(false);
}
//Extract features from source sentence
Sequence srcSequence = featurizer.AutoEncoder.Featurizer.ExtractFeatures(srcSentence);
double[] srcHiddenAvgOutput;
Dictionary <int, float> srcSparseFeatures;
ExtractSourceSentenceFeature(featurizer.AutoEncoder, srcSequence, curState.SparseFeature.Length, out srcHiddenAvgOutput, out srcSparseFeatures);
int numLayers = HiddenLayerList.Count;
List <int> predicted = new List <int>();
predicted.Add(curState.Label);
while (true)
{
//Build sparse features
SparseVector sparseVector = new SparseVector();
sparseVector.SetLength(curState.SparseFeature.Length + srcSequence.SparseFeatureSize);
sparseVector.AddKeyValuePairData(curState.SparseFeature);
sparseVector.AddKeyValuePairData(srcSparseFeatures);
//Compute first layer
double[] denseFeatures = RNNHelper.ConcatenateVector(curState.DenseFeature, srcHiddenAvgOutput);
HiddenLayerList[0].computeLayer(sparseVector, denseFeatures, false);
//Compute middle layers
for (int i = 1; i < numLayers; i++)
{
//We use previous layer's output as dense feature for current layer
denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[i - 1].cellOutput, srcHiddenAvgOutput);
HiddenLayerList[i].computeLayer(sparseVector, denseFeatures, false);
}
//Compute output layer
denseFeatures = RNNHelper.ConcatenateVector(HiddenLayerList[numLayers - 1].cellOutput, srcHiddenAvgOutput);
OutputLayer.computeLayer(sparseVector, denseFeatures, false);
OutputLayer.Softmax(false);
int nextTagId = OutputLayer.GetBestOutputIndex(false);
string nextWord = featurizer.TagSet.GetTagName(nextTagId);
curState = featurizer.ExtractFeatures(new string[] { nextWord });
curState.Label = nextTagId;
predicted.Add(nextTagId);
if (nextWord == "</s>" || predicted.Count >= 100)
{
break;
}
}
return(predicted.ToArray());
}
