/// <summary>
/// Encode source sentences and output encoded weights
/// </summary>
/// <param name="g"></param>
/// <param name="inputSentences"></param>
/// <param name="encoder"></param>
/// <param name="reversEncoder"></param>
/// <param name="Embedding"></param>
/// <returns></returns>
private IWeightMatrix Encode(IComputeGraph g, List <List <string> > inputSentences, Encoder encoder, Encoder reversEncoder, IWeightMatrix Embedding)
{
PadSentences(inputSentences);
List <IWeightMatrix> forwardOutputs = new List <IWeightMatrix>();
List <IWeightMatrix> backwardOutputs = new List <IWeightMatrix>();
int seqLen = inputSentences[0].Count;
List <IWeightMatrix> forwardInput = new List <IWeightMatrix>();
for (int i = 0; i < seqLen; i++)
{
for (int j = 0; j < inputSentences.Count; j++)
{
var inputSentence = inputSentences[j];
int ix_source = (int)SENTTAGS.UNK;
if (m_srcWordToIndex.ContainsKey(inputSentence[i]))
{
ix_source = m_srcWordToIndex[inputSentence[i]];
}
var x = g.PeekRow(Embedding, ix_source);
forwardInput.Add(x);
}
}
var forwardInputsM = g.ConcatRows(forwardInput);
for (int i = 0; i < seqLen; i++)
{
var eOutput = encoder.Encode(g.PeekRow(forwardInputsM, i * inputSentences.Count, inputSentences.Count), g);
forwardOutputs.Add(eOutput);
var eOutput2 = reversEncoder.Encode(g.PeekRow(forwardInputsM, forwardInputsM.Rows - (i + 1) * inputSentences.Count, inputSentences.Count), g);
backwardOutputs.Add(eOutput2);
}
backwardOutputs.Reverse();
List <IWeightMatrix> encoded = new List <IWeightMatrix>();
for (int i = 0; i < seqLen; i++)
{
encoded.Add(g.ConcatColumns(forwardOutputs[i], backwardOutputs[i]));
}
var encodedOutput = g.ConcatRows(encoded);
return(encodedOutput);
}
public AttentionPreProcessResult PreProcess(IWeightMatrix inputs, IComputeGraph g)
{
AttentionPreProcessResult r = new AttentionPreProcessResult();
IWeightMatrix bUas = g.RepeatRows(bUa, inputs.Rows);
r.uhs = g.MulAdd(inputs, Ua, bUas);
r.inputs = g.ConcatRows(g.UnFolderRow(inputs, m_batchSize));
return(r);
}
public IWeightTensor Encode(IWeightTensor rawInputs, IComputeGraph g)
{
int seqLen = rawInputs.Rows / m_batchSize;
List <IWeightTensor> inputs = new List <IWeightTensor>();
for (int i = 0; i < seqLen; i++)
{
var emb_i = g.PeekRow(rawInputs, i * m_batchSize, m_batchSize);
inputs.Add(emb_i);
}
List <IWeightTensor> forwardOutputs = new List <IWeightTensor>();
List <IWeightTensor> backwardOutputs = new List <IWeightTensor>();
List <IWeightTensor> layerOutputs = inputs.ToList();
for (int i = 0; i < m_depth; i++)
{
for (int j = 0; j < seqLen; j++)
{
var forwardOutput = m_forwardEncoders[i].Step(layerOutputs[j], g);
forwardOutputs.Add(forwardOutput);
var backwardOutput = m_backwardEncoders[i].Step(layerOutputs[inputs.Count - j - 1], g);
backwardOutputs.Add(backwardOutput);
}
backwardOutputs.Reverse();
layerOutputs.Clear();
for (int j = 0; j < seqLen; j++)
{
var concatW = g.ConcatColumns(forwardOutputs[j], backwardOutputs[j]);
layerOutputs.Add(concatW);
}
}
return(g.ConcatRows(layerOutputs));
}