public IWeightTensor Encode(IWeightTensor rawInputs, int batchSize, IComputeGraph g, IWeightTensor srcSelfMask)
{
int seqLen = rawInputs.Rows / batchSize;
rawInputs = g.TransposeBatch(rawInputs, seqLen);
List <IWeightTensor> inputs = new List <IWeightTensor>();
for (int i = 0; i < seqLen; i++)
{
IWeightTensor emb_i = g.Peek(rawInputs, 0, i * batchSize, batchSize);
inputs.Add(emb_i);
}
List <IWeightTensor> forwardOutputs = new List <IWeightTensor>();
List <IWeightTensor> backwardOutputs = new List <IWeightTensor>();
List <IWeightTensor> layerOutputs = inputs.ToList();
for (int i = 0; i < m_depth; i++)
{
for (int j = 0; j < seqLen; j++)
{
IWeightTensor forwardOutput = m_forwardEncoders[i].Step(layerOutputs[j], g);
forwardOutputs.Add(forwardOutput);
IWeightTensor backwardOutput = m_backwardEncoders[i].Step(layerOutputs[inputs.Count - j - 1], g);
backwardOutputs.Add(backwardOutput);
}
backwardOutputs.Reverse();
layerOutputs.Clear();
for (int j = 0; j < seqLen; j++)
{
IWeightTensor concatW = g.Concate(1, forwardOutputs[j], backwardOutputs[j]);
layerOutputs.Add(concatW);
}
}
var result = g.Concate(layerOutputs, 0);
return(g.TransposeBatch(result, batchSize));
}
public static IWeightTensor AddPositionEmbedding(IComputeGraph g, IWeightTensor posEmbedding, int batchSize, IWeightTensor inputEmbs, float dropoutRatio)
{
var Column = posEmbedding.Columns;
int seqLen = inputEmbs.Rows / batchSize;
using (var posEmbeddingPeek = g.Peek(posEmbedding, 0, 0, seqLen))
{
using (var posEmbeddingPeekView = g.View(posEmbeddingPeek, dims: new long[] { 1, seqLen, Column }))
{
using (var posEmbeddingPeekViewExp = g.Expand(posEmbeddingPeekView, dims: new long[] { batchSize, seqLen, Column }))
{
inputEmbs = g.View(inputEmbs, dims: new long[] { batchSize, seqLen, Column });
inputEmbs = g.Add(inputEmbs, posEmbeddingPeekViewExp, inPlace: true);
inputEmbs = g.View(inputEmbs, dims: new long[] { batchSize *seqLen, Column });
}
}
}
inputEmbs = g.Dropout(inputEmbs, batchSize, dropoutRatio, inPlace: true);
return(inputEmbs);
}