public void VisualizeNeuralNetwork(string visNNFilePath)
{
(IEncoder encoder, IDecoder decoder, IWeightTensor srcEmbedding, IWeightTensor tgtEmbedding) = GetNetworksOnDeviceAt(-1);
// Build input sentence
List <List <string> > inputSeqs = ParallelCorpus.ConstructInputTokens(null);
int batchSize = inputSeqs.Count;
IComputeGraph g = CreateComputGraph(m_defaultDeviceId, needBack: false, visNetwork: true);
AttentionDecoder rnnDecoder = decoder as AttentionDecoder;
encoder.Reset(g.GetWeightFactory(), batchSize);
rnnDecoder.Reset(g.GetWeightFactory(), batchSize);
// Run encoder
IWeightTensor encodedWeightMatrix = Encode(g, inputSeqs, encoder, srcEmbedding, null, null);
// Prepare for attention over encoder-decoder
AttentionPreProcessResult attPreProcessResult = rnnDecoder.PreProcess(encodedWeightMatrix, batchSize, g);
// Run decoder
IWeightTensor x = g.PeekRow(tgtEmbedding, (int)SENTTAGS.START);
IWeightTensor eOutput = rnnDecoder.Decode(x, attPreProcessResult, batchSize, g);
IWeightTensor probs = g.Softmax(eOutput);
g.VisualizeNeuralNetToFile(visNNFilePath);
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="computeGraph">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
private float RunForwardOnSingleDevice(IComputeGraph computeGraph, List <List <string> > srcSnts, List <List <string> > tgtSnts, int deviceIdIdx, bool isTraining)
{
(IEncoder encoder, AttentionDecoder decoder, IWeightTensor srcEmbedding, IWeightTensor tgtEmbedding) = GetNetworksOnDeviceAt(deviceIdIdx);
// Reset networks
encoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
decoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
// Encoding input source sentences
IWeightTensor encodedWeightMatrix = Encode(computeGraph, srcSnts, encoder, srcEmbedding);
// Generate output decoder sentences
return(Decode(tgtSnts, computeGraph, encodedWeightMatrix, decoder, tgtEmbedding, srcSnts.Count, isTraining));
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="g">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side. In training mode, it inputs target tokens, otherwise, it outputs target tokens generated by decoder</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
public override List <NetworkResult> RunForwardOnSingleDevice(IComputeGraph g, ISntPairBatch sntPairBatch, int deviceIdIdx, bool isTraining, DecodingOptions decodingOptions)
{
List <NetworkResult> nrs = new List <NetworkResult>();
var srcSnts = sntPairBatch.GetSrcTokens(0);
var tgtSnts = sntPairBatch.GetTgtTokens(0);
(IEncoder encoder, IWeightTensor srcEmbedding, IWeightTensor posEmbedding, FeedForwardLayer decoderFFLayer) = GetNetworksOnDeviceAt(deviceIdIdx);
// Reset networks
encoder.Reset(g.GetWeightFactory(), srcSnts.Count);
var originalSrcLengths = BuildInTokens.PadSentences(srcSnts);
var srcTokensList = m_modelMetaData.SrcVocab.GetWordIndex(srcSnts);
BuildInTokens.PadSentences(tgtSnts);
var tgtTokensLists = m_modelMetaData.ClsVocab.GetWordIndex(tgtSnts);
int seqLen = srcSnts[0].Count;
int batchSize = srcSnts.Count;
// Encoding input source sentences
IWeightTensor encOutput = Encoder.Run(g, sntPairBatch, encoder, m_modelMetaData, m_shuffleType, srcEmbedding, posEmbedding, null, srcTokensList, originalSrcLengths);
IWeightTensor ffLayer = decoderFFLayer.Process(encOutput, batchSize, g);
float cost = 0.0f;
IWeightTensor probs = g.Softmax(ffLayer, inPlace: true);
if (isTraining)
{
var tgtTokensTensor = g.CreateTokensTensor(tgtTokensLists);
cost = g.CrossEntropyLoss(probs, tgtTokensTensor);
}
else
{
// Output "i"th target word
using var targetIdxTensor = g.Argmax(probs, 1);
float[] targetIdx = targetIdxTensor.ToWeightArray();
List <string> targetWords = m_modelMetaData.ClsVocab.ConvertIdsToString(targetIdx.ToList());
for (int k = 0; k < batchSize; k++)
{
tgtSnts[k] = targetWords.GetRange(k * seqLen, seqLen);
}
}
NetworkResult nr = new NetworkResult
{
Cost = cost,
Output = new List <List <List <string> > >()
};
nr.Output.Add(tgtSnts);
nrs.Add(nr);
return(nrs);
}
static public IWeightTensor Run(IComputeGraph computeGraph, ISntPairBatch sntPairBatch, IEncoder encoder, IModel modelMetaData, ShuffleEnums shuffleType,
IWeightTensor srcEmbedding, IWeightTensor posEmbedding, IWeightTensor segmentEmbedding, List <List <int> > srcSntsIds, float[] originalSrcLengths)
{
// Reset networks
encoder.Reset(computeGraph.GetWeightFactory(), srcSntsIds.Count);
IWeightTensor encOutput = InnerRunner(computeGraph, srcSntsIds, originalSrcLengths, shuffleType, encoder, modelMetaData, srcEmbedding, posEmbedding, segmentEmbedding);
return(encOutput);
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="computeGraph">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
private float RunForwardOnSingleDevice(IComputeGraph computeGraph, List <List <string> > srcSnts, List <List <string> > tgtSnts, int deviceIdIdx, bool isTraining)
{
var(encoder, decoder, srcEmbedding, tgtEmbedding, posEmbedding) = this.GetNetworksOnDeviceAt(deviceIdIdx);
// Reset networks
encoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
decoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
var originalSrcLengths = ParallelCorpus.PadSentences(srcSnts);
var srcSeqPaddedLen = srcSnts[0].Count;
var batchSize = srcSnts.Count;
var srcSelfMask = this.m_shuffleType == ShuffleEnums.NoPaddingInSrc ? null : MaskUtils.BuildPadSelfMask(computeGraph, srcSeqPaddedLen, originalSrcLengths, this.DeviceIds[deviceIdIdx]); // The length of source sentences are same in a single mini-batch, so we don't have source mask.
// Encoding input source sentences
var encOutput = this.Encode(computeGraph, srcSnts, encoder, srcEmbedding, srcSelfMask, posEmbedding, originalSrcLengths);
srcSelfMask?.Dispose();
// Generate output decoder sentences
if (decoder is AttentionDecoder)
{
return(this.DecodeAttentionLSTM(tgtSnts, computeGraph, encOutput, decoder as AttentionDecoder, tgtEmbedding, srcSnts.Count, isTraining));
}
else
{
if (isTraining)
{
return(this.DecodeTransformer(tgtSnts, computeGraph, encOutput, decoder as TransformerDecoder, tgtEmbedding, posEmbedding, batchSize, this.DeviceIds[deviceIdIdx], originalSrcLengths, isTraining));
}
else
{
for (var i = 0; i < this.m_maxTgtSntSize; i++)
{
using (var g = computeGraph.CreateSubGraph($"TransformerDecoder_Step_{i}"))
{
this.DecodeTransformer(tgtSnts, g, encOutput, decoder as TransformerDecoder, tgtEmbedding, posEmbedding, batchSize, this.DeviceIds[deviceIdIdx], originalSrcLengths, isTraining);
var allSntsEnd = true;
foreach (var t in tgtSnts)
{
if (t[^ 1] != ParallelCorpus.EOS)
public AttentionPreProcessResult PreProcess(IWeightTensor encOutput, int batchSize, IComputeGraph g)
{
int srcSeqLen = encOutput.Rows / batchSize;
AttentionPreProcessResult r = new AttentionPreProcessResult
{
encOutput = encOutput
};
r.Uhs = g.Affine(r.encOutput, m_Ua, m_bUa);
r.Uhs = g.View(r.Uhs, dims: new long[] { batchSize, srcSeqLen, -1 });
if (m_enableCoverageModel)
{
m_coverage.Reset(g.GetWeightFactory(), r.encOutput.Rows);
}
return(r);
}
public AttentionPreProcessResult PreProcess(IWeightTensor inputs, int batchSize, IComputeGraph g)
{
int srcSeqLen = inputs.Rows / batchSize;
AttentionPreProcessResult r = new AttentionPreProcessResult
{
rawInputs = inputs,
inputsBatchFirst = g.TransposeBatch(inputs, batchSize)
};
r.uhs = g.Affine(r.inputsBatchFirst, m_Ua, m_bUa);
r.uhs = g.View(r.uhs, batchSize, srcSeqLen, -1);
if (m_enableCoverageModel)
{
m_coverage.Reset(g.GetWeightFactory(), r.inputsBatchFirst.Rows);
}
return(r);
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="g">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side. In training mode, it inputs target tokens, otherwise, it outputs target tokens generated by decoder</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
private float RunForwardOnSingleDevice(IComputeGraph g, List <List <string> > srcSnts, List <List <string> > tgtSnts, int deviceIdIdx, bool isTraining)
{
(IEncoder encoder, IWeightTensor srcEmbedding, FeedForwardLayer decoderFFLayer) = GetNetworksOnDeviceAt(deviceIdIdx);
int batchSize = srcSnts.Count;
// Reset networks
encoder.Reset(g.GetWeightFactory(), batchSize);
// Encoding input source sentences
ParallelCorpus.PadSentences(srcSnts);
if (isTraining)
{
ParallelCorpus.PadSentences(tgtSnts);
if (srcSnts[0].Count != tgtSnts[0].Count)
{
throw new ArgumentException($"The length of source side and target side must be equal. source length = '{srcSnts[0].Count}', target length = '{tgtSnts[0].Count}'");
}
}
int seqLen = srcSnts[0].Count;
IWeightTensor encodedWeightMatrix = Encode(g.CreateSubGraph("Encoder"), srcSnts, encoder, srcEmbedding);
IWeightTensor ffLayer = decoderFFLayer.Process(encodedWeightMatrix, batchSize, g);
IWeightTensor ffLayerBatch = g.TransposeBatch(ffLayer, batchSize);
// Logger.WriteLine("1");
float cost = 0.0f;
using (var probs = g.Softmax(ffLayerBatch, runGradients: false, inPlace: true))
{
if (isTraining)
{
//Calculate loss for each word in the batch
for (int k = 0; k < batchSize; k++)
{
for (int j = 0; j < seqLen; j++)
{
using (var probs_k_j = g.PeekRow(probs, k * seqLen + j, runGradients: false))
{
var ix_targets_k_j = m_modelMetaData.Vocab.GetTargetWordIndex(tgtSnts[k][j]);
var score_k = probs_k_j.GetWeightAt(ix_targets_k_j);
cost += (float)-Math.Log(score_k);
probs_k_j.SetWeightAt(score_k - 1, ix_targets_k_j);
}
}
////CRF part
//using (var probs_k = g.PeekRow(probs, k * seqLen, seqLen, runGradients: false))
//{
// var weights_k = probs_k.ToWeightArray();
// var crfOutput_k = m_crfDecoder.ForwardBackward(seqLen, weights_k);
// int[] trueTags = new int[seqLen];
// for (int j = 0; j < seqLen; j++)
// {
// trueTags[j] = m_modelMetaData.Vocab.GetTargetWordIndex(tgtSnts[k][j]);
// }
// m_crfDecoder.UpdateBigramTransition(seqLen, crfOutput_k, trueTags);
//}
}
ffLayerBatch.CopyWeightsToGradients(probs);
}
else
{
// CRF decoder
//for (int k = 0; k < batchSize; k++)
//{
// //CRF part
// using (var probs_k = g.PeekRow(probs, k * seqLen, seqLen, runGradients: false))
// {
// var weights_k = probs_k.ToWeightArray();
// var crfOutput_k = m_crfDecoder.DecodeNBestCRF(weights_k, seqLen, 1);
// var targetWords = m_modelMetaData.Vocab.ConvertTargetIdsToString(crfOutput_k[0].ToList());
// tgtSnts.Add(targetWords);
// }
//}
// Output "i"th target word
var targetIdx = g.Argmax(probs, 1);
var targetWords = m_modelMetaData.Vocab.ConvertTargetIdsToString(targetIdx.ToList());
for (int k = 0; k < batchSize; k++)
{
tgtSnts.Add(targetWords.GetRange(k * seqLen, seqLen));
}
}
}
return(cost);
}
/// <summary>
/// Given input sentence and generate output sentence by seq2seq model with beam search
/// </summary>
/// <param name="input"></param>
/// <param name="beamSearchSize"></param>
/// <param name="maxOutputLength"></param>
/// <returns></returns>
public List <List <string> > Predict(List <string> input, int beamSearchSize = 1, int maxOutputLength = 100)
{
(IEncoder encoder, IDecoder decoder, IWeightTensor srcEmbedding, IWeightTensor tgtEmbedding) = GetNetworksOnDeviceAt(-1);
List <List <string> > inputSeqs = ParallelCorpus.ConstructInputTokens(input);
int batchSize = 1; // For predict with beam search, we currently only supports one sentence per call
IComputeGraph g = CreateComputGraph(m_defaultDeviceId, needBack: false);
AttentionDecoder rnnDecoder = decoder as AttentionDecoder;
encoder.Reset(g.GetWeightFactory(), batchSize);
rnnDecoder.Reset(g.GetWeightFactory(), batchSize);
// Construct beam search status list
List <BeamSearchStatus> bssList = new List <BeamSearchStatus>();
BeamSearchStatus bss = new BeamSearchStatus();
bss.OutputIds.Add((int)SENTTAGS.START);
bss.CTs = rnnDecoder.GetCTs();
bss.HTs = rnnDecoder.GetHTs();
bssList.Add(bss);
IWeightTensor encodedWeightMatrix = Encode(g, inputSeqs, encoder, srcEmbedding, null, null);
AttentionPreProcessResult attPreProcessResult = rnnDecoder.PreProcess(encodedWeightMatrix, batchSize, g);
List <BeamSearchStatus> newBSSList = new List <BeamSearchStatus>();
bool finished = false;
int outputLength = 0;
while (finished == false && outputLength < maxOutputLength)
{
finished = true;
for (int i = 0; i < bssList.Count; i++)
{
bss = bssList[i];
if (bss.OutputIds[bss.OutputIds.Count - 1] == (int)SENTTAGS.END)
{
newBSSList.Add(bss);
}
else if (bss.OutputIds.Count > maxOutputLength)
{
newBSSList.Add(bss);
}
else
{
finished = false;
int ix_input = bss.OutputIds[bss.OutputIds.Count - 1];
rnnDecoder.SetCTs(bss.CTs);
rnnDecoder.SetHTs(bss.HTs);
IWeightTensor x = g.PeekRow(tgtEmbedding, ix_input);
IWeightTensor eOutput = rnnDecoder.Decode(x, attPreProcessResult, batchSize, g);
using (IWeightTensor probs = g.Softmax(eOutput))
{
List <int> preds = probs.GetTopNMaxWeightIdx(beamSearchSize);
for (int j = 0; j < preds.Count; j++)
{
BeamSearchStatus newBSS = new BeamSearchStatus();
newBSS.OutputIds.AddRange(bss.OutputIds);
newBSS.OutputIds.Add(preds[j]);
newBSS.CTs = rnnDecoder.GetCTs();
newBSS.HTs = rnnDecoder.GetHTs();
float score = probs.GetWeightAt(preds[j]);
newBSS.Score = bss.Score;
newBSS.Score += (float)(-Math.Log(score));
//var lengthPenalty = Math.Pow((5.0f + newBSS.OutputIds.Count) / 6, 0.6);
//newBSS.Score /= (float)lengthPenalty;
newBSSList.Add(newBSS);
}
}
}
}
bssList = BeamSearch.GetTopNBSS(newBSSList, beamSearchSize);
newBSSList.Clear();
outputLength++;
}
// Convert output target word ids to real string
List <List <string> > results = new List <List <string> >();
for (int i = 0; i < bssList.Count; i++)
{
results.Add(m_modelMetaData.Vocab.ConvertTargetIdsToString(bssList[i].OutputIds));
}
return(results);
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="computeGraph">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
private float RunForwardOnSingleDevice(IComputeGraph computeGraph, List <List <string> > srcSnts, List <List <string> > tgtSnts, int deviceIdIdx, bool isTraining)
{
(IEncoder encoder, IDecoder decoder, IWeightTensor srcEmbedding, IWeightTensor tgtEmbedding) = GetNetworksOnDeviceAt(deviceIdIdx);
// Reset networks
encoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
decoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
List <int> originalSrcLengths = ParallelCorpus.PadSentences(srcSnts);
int srcSeqPaddedLen = srcSnts[0].Count;
int batchSize = srcSnts.Count;
IWeightTensor encSelfMask = MaskUtils.BuildPadSelfMask(computeGraph, srcSeqPaddedLen, originalSrcLengths, deviceIdIdx);
IWeightTensor encDimMask = MaskUtils.BuildPadDimMask(computeGraph, srcSeqPaddedLen, originalSrcLengths, m_modelMetaData.HiddenDim, deviceIdIdx);
// Encoding input source sentences
IWeightTensor encOutput = Encode(computeGraph, srcSnts, encoder, srcEmbedding, encSelfMask, encDimMask);
// Generate output decoder sentences
if (decoder is AttentionDecoder)
{
return(DecodeAttentionLSTM(tgtSnts, computeGraph, encOutput, decoder as AttentionDecoder, tgtEmbedding, srcSnts.Count, isTraining));
}
else
{
if (isTraining)
{
List <int> originalTgtLengths = ParallelCorpus.PadSentences(tgtSnts);
int tgtSeqPaddedLen = tgtSnts[0].Count;
IWeightTensor encDecMask = MaskUtils.BuildSrcTgtMask(computeGraph, srcSeqPaddedLen, tgtSeqPaddedLen, originalSrcLengths, originalTgtLengths, deviceIdIdx);
return(DecodeTransformer(tgtSnts, computeGraph, encOutput, encDecMask, decoder as TransformerDecoder, tgtEmbedding, batchSize, deviceIdIdx, isTraining));
}
else
{
for (int i = 0; i < m_maxTgtSntSize; i++)
{
using (var g = computeGraph.CreateSubGraph($"TransformerDecoder_Step_{i}"))
{
List <int> originalTgtLengths = ParallelCorpus.PadSentences(tgtSnts);
int tgtSeqPaddedLen = tgtSnts[0].Count;
IWeightTensor encDecMask = MaskUtils.BuildSrcTgtMask(g, srcSeqPaddedLen, tgtSeqPaddedLen, originalSrcLengths, originalTgtLengths, deviceIdIdx);
DecodeTransformer(tgtSnts, g, encOutput, encDecMask, decoder as TransformerDecoder, tgtEmbedding, batchSize, deviceIdIdx, isTraining);
bool allSntsEnd = true;
for (int j = 0; j < tgtSnts.Count; j++)
{
if (tgtSnts[j][tgtSnts[j].Count - 1] != ParallelCorpus.EOS)
{
allSntsEnd = false;
break;
}
}
if (allSntsEnd)
{
break;
}
}
}
return(0.0f);
}
}
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="g">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side. In training mode, it inputs target tokens, otherwise, it outputs target tokens generated by decoder</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
private float RunForwardOnSingleDevice(IComputeGraph g, List <List <string> > srcSnts, List <List <string> > tgtSnts, int deviceIdIdx, bool isTraining)
{
var(encoder, srcEmbedding, posEmbedding, decoderFFLayer) = this.GetNetworksOnDeviceAt(deviceIdIdx);
// Reset networks
encoder.Reset(g.GetWeightFactory(), srcSnts.Count);
var originalSrcLengths = ParallelCorpus.PadSentences(srcSnts);
var seqLen = srcSnts[0].Count;
var batchSize = srcSnts.Count;
// Encoding input source sentences
var encOutput = this.Encode(g, srcSnts, encoder, srcEmbedding, null, posEmbedding, originalSrcLengths);
var ffLayer = decoderFFLayer.Process(encOutput, batchSize, g);
var ffLayerBatch = g.TransposeBatch(ffLayer, batchSize);
var cost = 0.0f;
using (var probs = g.Softmax(ffLayerBatch, runGradients: false, inPlace: true))
{
if (isTraining)
{
//Calculate loss for each word in the batch
for (var k = 0; k < batchSize; k++)
{
for (var j = 0; j < seqLen; j++)
{
using (var probs_k_j = g.PeekRow(probs, k * seqLen + j, runGradients: false))
{
var ix_targets_k_j = this.m_modelMetaData.Vocab.GetTargetWordIndex(tgtSnts[k][j]);
var score_k = probs_k_j.GetWeightAt(ix_targets_k_j);
cost += (float)-Math.Log(score_k);
probs_k_j.SetWeightAt(score_k - 1, ix_targets_k_j);
}
}
////CRF part
//using (var probs_k = g.PeekRow(probs, k * seqLen, seqLen, runGradients: false))
//{
// var weights_k = probs_k.ToWeightArray();
// var crfOutput_k = m_crfDecoder.ForwardBackward(seqLen, weights_k);
// int[] trueTags = new int[seqLen];
// for (int j = 0; j < seqLen; j++)
// {
// trueTags[j] = m_modelMetaData.Vocab.GetTargetWordIndex(tgtSnts[k][j]);
// }
// m_crfDecoder.UpdateBigramTransition(seqLen, crfOutput_k, trueTags);
//}
}
ffLayerBatch.CopyWeightsToGradients(probs);
}
else
{
// CRF decoder
//for (int k = 0; k < batchSize; k++)
//{
// //CRF part
// using (var probs_k = g.PeekRow(probs, k * seqLen, seqLen, runGradients: false))
// {
// var weights_k = probs_k.ToWeightArray();
// var crfOutput_k = m_crfDecoder.DecodeNBestCRF(weights_k, seqLen, 1);
// var targetWords = m_modelMetaData.Vocab.ConvertTargetIdsToString(crfOutput_k[0].ToList());
// tgtSnts.Add(targetWords);
// }
//}
// Output "i"th target word
var targetIdx = g.Argmax(probs, 1);
var targetWords = this.m_modelMetaData.Vocab.ConvertTargetIdsToString(targetIdx.ToList());
for (var k = 0; k < batchSize; k++)
{
tgtSnts[k] = targetWords.GetRange(k * seqLen, seqLen);
}
}
}
return(cost);
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="computeGraph">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
private float RunForwardOnSingleDevice(IComputeGraph computeGraph, List <List <string> > srcSnts, List <List <string> > tgtSnts, int deviceIdIdx, bool isTraining)
{
(IEncoder encoder, IDecoder decoder, IWeightTensor srcEmbedding, IWeightTensor tgtEmbedding, IWeightTensor posEmbedding) = GetNetworksOnDeviceAt(deviceIdIdx);
// Reset networks
encoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
decoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
List <int> originalSrcLengths = ParallelCorpus.PadSentences(srcSnts);
int srcSeqPaddedLen = srcSnts[0].Count;
int batchSize = srcSnts.Count;
IWeightTensor srcSelfMask = m_shuffleType == ShuffleEnums.NoPaddingInSrc ? null : MaskUtils.BuildPadSelfMask(computeGraph, srcSeqPaddedLen, originalSrcLengths, DeviceIds[deviceIdIdx]); // The length of source sentences are same in a single mini-batch, so we don't have source mask.
// Encoding input source sentences
IWeightTensor encOutput = Encode(computeGraph, srcSnts, encoder, srcEmbedding, srcSelfMask, posEmbedding, originalSrcLengths);
if (srcSelfMask != null)
{
srcSelfMask.Dispose();
}
// Generate output decoder sentences
if (decoder is AttentionDecoder)
{
return(DecodeAttentionLSTM(tgtSnts, computeGraph, encOutput, decoder as AttentionDecoder, tgtEmbedding, srcSnts.Count, isTraining));
}
else
{
if (isTraining)
{
return(DecodeTransformer(tgtSnts, computeGraph, encOutput, decoder as TransformerDecoder, tgtEmbedding, posEmbedding, batchSize, DeviceIds[deviceIdIdx], originalSrcLengths, isTraining));
}
else
{
for (int i = 0; i < m_maxTgtSntSize; i++)
{
using (var g = computeGraph.CreateSubGraph($"TransformerDecoder_Step_{i}"))
{
DecodeTransformer(tgtSnts, g, encOutput, decoder as TransformerDecoder, tgtEmbedding, posEmbedding, batchSize, DeviceIds[deviceIdIdx], originalSrcLengths, isTraining);
bool allSntsEnd = true;
for (int j = 0; j < tgtSnts.Count; j++)
{
if (tgtSnts[j][tgtSnts[j].Count - 1] != ParallelCorpus.EOS)
{
allSntsEnd = false;
break;
}
}
if (allSntsEnd)
{
break;
}
}
}
RemoveDuplicatedEOS(tgtSnts);
return(0.0f);
}
}
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="computeGraph">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
public override List <NetworkResult> RunForwardOnSingleDevice(IComputeGraph computeGraph, ISntPairBatch sntPairBatch, int deviceIdIdx, bool isTraining, DecodingOptions decodingOptions)
{
(var encoder, var decoder, var decoderFFLayer, var srcEmbedding, var tgtEmbedding, var posEmbedding, var segmentEmbedding, var pointerGenerator) = GetNetworksOnDeviceAt(deviceIdIdx);
var srcSnts = sntPairBatch.GetSrcTokens(0);
var originalSrcLengths = BuildInTokens.PadSentences(srcSnts);
var srcTokensList = m_modelMetaData.SrcVocab.GetWordIndex(srcSnts);
if (isTraining && srcSnts[0].Count > m_options.MaxTrainSrcSentLength + 2)
{
throw new InvalidDataException($"The source sentence is too long. Its length = '{srcSnts[0].Count}', but MaxTrainSrcSentLength is '{m_options.MaxTrainSrcSentLength}'. The sentence is '{string.Join(" ", srcSnts[0])}'");
}
IWeightTensor encOutput;
if (!isTraining && (m_options.ProcessorType == ProcessorTypeEnums.CPU))
{
// Try to get src tensor from cache
string cacheKey = GenerateCacheKey(srcSnts);
if (!m_memoryCache.TryGetValue(cacheKey, out encOutput))
{
encOutput = Encoder.Run(computeGraph, sntPairBatch, encoder, m_modelMetaData, m_shuffleType, srcEmbedding, posEmbedding, segmentEmbedding, srcTokensList, originalSrcLengths);
var cacheEntryOptions = new MemoryCacheEntryOptions().SetSize(1);
m_memoryCache.Set(cacheKey, encOutput.CopyWeightsRef($"cache_{encOutput.Name}", false), cacheEntryOptions);
}
}
else
{
// Compute src tensor
encOutput = Encoder.Run(computeGraph, sntPairBatch, encoder, m_modelMetaData, m_shuffleType, srcEmbedding, posEmbedding, segmentEmbedding, srcTokensList, originalSrcLengths);
}
List <NetworkResult> nrs = new List <NetworkResult>();
// Generate output decoder sentences
int batchSize = srcSnts.Count;
var tgtSnts = sntPairBatch.GetTgtTokens(0);
var tgtTokensList = m_modelMetaData.TgtVocab.GetWordIndex(tgtSnts);
NetworkResult nr = new NetworkResult();
decoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
if (decoder is AttentionDecoder)
{
nr.Cost = Decoder.DecodeAttentionLSTM(tgtTokensList, computeGraph, encOutput, decoder as AttentionDecoder, decoderFFLayer, tgtEmbedding, m_modelMetaData.TgtVocab, srcSnts.Count, isTraining);
nr.Output = new List <List <List <string> > >
{
m_modelMetaData.TgtVocab.ConvertIdsToString(tgtTokensList)
};
}
else
{
if (isTraining)
{
(var c, _) = Decoder.DecodeTransformer(tgtTokensList, computeGraph, encOutput, decoder as TransformerDecoder, decoderFFLayer, tgtEmbedding, posEmbedding, originalSrcLengths, m_modelMetaData.TgtVocab, m_shuffleType,
m_options.DropoutRatio, null, isTraining, pointerGenerator: pointerGenerator, srcSeqs: srcTokensList);
nr.Cost = c;
nr.Output = null;
}
else
{
Dictionary <string, IWeightTensor> cachedTensors = new Dictionary <string, IWeightTensor>();
List <List <BeamSearchStatus> > beam2batchStatus = Decoder.InitBeamSearchStatusListList(batchSize, tgtTokensList);
for (int i = tgtTokensList[0].Count; i < decodingOptions.MaxTgtSentLength; i++)
{
List <List <BeamSearchStatus> > batch2beam2seq = null; //(batch_size, beam_search_size)
try
{
foreach (var batchStatus in beam2batchStatus)
{
var batch2tgtTokens = Decoder.ExtractBatchTokens(batchStatus);
using var g = computeGraph.CreateSubGraph($"TransformerDecoder_Step_{i}");
(var cost2, var bssSeqList) = Decoder.DecodeTransformer(batch2tgtTokens, g, encOutput, decoder as TransformerDecoder, decoderFFLayer, tgtEmbedding, posEmbedding,
originalSrcLengths, m_modelMetaData.TgtVocab, m_shuffleType, 0.0f, decodingOptions, isTraining,
outputSentScore: decodingOptions.BeamSearchSize > 1, previousBeamSearchResults: batchStatus,
pointerGenerator: pointerGenerator, srcSeqs: srcTokensList,
cachedTensors: cachedTensors);
bssSeqList = Decoder.SwapBeamAndBatch(bssSeqList); // Swap shape: (beam_search_size, batch_size) -> (batch_size, beam_search_size)
batch2beam2seq = Decoder.CombineBeamSearchResults(batch2beam2seq, bssSeqList);
}
}
catch (OutOfMemoryException)
{
GC.Collect();
Logger.WriteLine(Logger.Level.warn, $"We have out of memory while generating '{i}th' tokens, so terminate decoding for current sequences.");
break;
}
if (decodingOptions.BeamSearchSize > 1)
{
// Keep top N result and drop all others
for (int k = 0; k < batchSize; k++)
{
batch2beam2seq[k] = BeamSearch.GetTopNBSS(batch2beam2seq[k], decodingOptions.BeamSearchSize);
}
}
beam2batchStatus = Decoder.SwapBeamAndBatch(batch2beam2seq);
if (Decoder.AreAllSentsCompleted(beam2batchStatus))
{
break;
}
}
nr.Cost = 0.0f;
nr.Output = m_modelMetaData.TgtVocab.ExtractTokens(beam2batchStatus);
if (cachedTensors != null)
{
foreach (var pair in cachedTensors)
{
pair.Value.Dispose();
}
}
}
}
nr.RemoveDuplicatedEOS();
nrs.Add(nr);
return(nrs);
}
/// <summary>
/// Run forward part on given single device
/// </summary>
/// <param name="computeGraph">The computing graph for current device. It gets created and passed by the framework</param>
/// <param name="srcSnts">A batch of input tokenized sentences in source side</param>
/// <param name="tgtSnts">A batch of output tokenized sentences in target side</param>
/// <param name="deviceIdIdx">The index of current device</param>
/// <returns>The cost of forward part</returns>
public override List <NetworkResult> RunForwardOnSingleDevice(IComputeGraph computeGraph, ISntPairBatch sntPairBatch, int deviceIdIdx, bool isTraining, DecodingOptions decodingOptions)
{
(IEncoder encoder, IDecoder decoder, IFeedForwardLayer encoderFFLayer, IFeedForwardLayer decoderFFLayer, IWeightTensor srcEmbedding, IWeightTensor tgtEmbedding, IWeightTensor posEmbedding, IWeightTensor segmentEmbedding) = GetNetworksOnDeviceAt(deviceIdIdx);
var srcSnts = sntPairBatch.GetSrcTokens(0);
var originalSrcLengths = BuildInTokens.PadSentences(srcSnts);
var srcTokensList = m_modelMetaData.SrcVocab.GetWordIndex(srcSnts);
IWeightTensor encOutput = Encoder.Run(computeGraph, sntPairBatch, encoder, m_modelMetaData, m_shuffleType, srcEmbedding, posEmbedding, segmentEmbedding, srcTokensList, originalSrcLengths);
List <NetworkResult> nrs = new List <NetworkResult>();
int srcSeqPaddedLen = srcSnts[0].Count;
int batchSize = srcSnts.Count;
float[] clsIdxs = new float[batchSize];
for (int i = 0; i < batchSize; i++)
{
for (int j = 0; j < srcSnts[i].Count; j++)
{
if (srcSnts[i][j] == BuildInTokens.CLS)
{
clsIdxs[i] = i * srcSeqPaddedLen + j;
break;
}
}
}
IWeightTensor clsWeightTensor = computeGraph.IndexSelect(encOutput, clsIdxs);
float cost = 0.0f;
NetworkResult nrCLS = new NetworkResult
{
Output = new List <List <List <string> > >()
};
IWeightTensor ffLayer = encoderFFLayer.Process(clsWeightTensor, batchSize, computeGraph);
using (IWeightTensor probs = computeGraph.Softmax(ffLayer, runGradients: false, inPlace: true))
{
if (isTraining)
{
var clsSnts = sntPairBatch.GetTgtTokens(0);
for (int k = 0; k < batchSize; k++)
{
int ix_targets_k_j = m_modelMetaData.ClsVocab.GetWordIndex(clsSnts[k][0]);
float score_k = probs.GetWeightAt(new long[] { k, ix_targets_k_j });
cost += (float)-Math.Log(score_k);
probs.SetWeightAt(score_k - 1, new long[] { k, ix_targets_k_j });
}
ffLayer.CopyWeightsToGradients(probs);
nrCLS.Cost = cost / batchSize;
}
else
{
// Output "i"th target word
using var targetIdxTensor = computeGraph.Argmax(probs, 1);
float[] targetIdx = targetIdxTensor.ToWeightArray();
List <string> targetWords = m_modelMetaData.ClsVocab.ConvertIdsToString(targetIdx.ToList());
nrCLS.Output.Add(new List <List <string> >());
for (int k = 0; k < batchSize; k++)
{
nrCLS.Output[0].Add(new List <string>());
nrCLS.Output[0][k].Add(targetWords[k]);
}
}
}
// Reset networks
decoder.Reset(computeGraph.GetWeightFactory(), srcSnts.Count);
// Generate output decoder sentences
var tgtSnts = sntPairBatch.GetTgtTokens(1);
var tgtTokensList = m_modelMetaData.TgtVocab.GetWordIndex(tgtSnts);
NetworkResult nr = new NetworkResult();
if (decoder is AttentionDecoder)
{
nr.Cost = Decoder.DecodeAttentionLSTM(tgtTokensList, computeGraph, encOutput, decoder as AttentionDecoder, decoderFFLayer, tgtEmbedding, m_modelMetaData.TgtVocab, srcSnts.Count, isTraining);
nr.Output = new List <List <List <string> > >
{
m_modelMetaData.TgtVocab.ConvertIdsToString(tgtTokensList)
};
}
else
{
if (isTraining)
{
(var c, _) = Decoder.DecodeTransformer(tgtTokensList, computeGraph, encOutput, decoder as TransformerDecoder, decoderFFLayer, tgtEmbedding, posEmbedding, originalSrcLengths, m_modelMetaData.TgtVocab, m_shuffleType, m_options.DropoutRatio, null, isTraining);
nr.Cost = c;
nr.Output = null;
}
else
{
List <List <BeamSearchStatus> > beam2batchStatus = Decoder.InitBeamSearchStatusListList(batchSize, tgtTokensList);
for (int i = 0; i < decodingOptions.MaxTgtSentLength; i++)
{
List <List <BeamSearchStatus> > batch2beam2seq = null; //(batch_size, beam_search_size)
try
{
foreach (var batchStatus in beam2batchStatus)
{
var batch2tgtTokens = Decoder.ExtractBatchTokens(batchStatus);
using var g = computeGraph.CreateSubGraph($"TransformerDecoder_Step_{i}");
(var cost2, var bssSeqList) = Decoder.DecodeTransformer(batch2tgtTokens, g, encOutput, decoder as TransformerDecoder, decoderFFLayer, tgtEmbedding, posEmbedding,
originalSrcLengths, m_modelMetaData.TgtVocab, m_shuffleType, 0.0f, decodingOptions, isTraining,
outputSentScore: decodingOptions.BeamSearchSize > 1, previousBeamSearchResults: batchStatus);
bssSeqList = Decoder.SwapBeamAndBatch(bssSeqList);
batch2beam2seq = Decoder.CombineBeamSearchResults(batch2beam2seq, bssSeqList);
}
}
catch (OutOfMemoryException)
{
GC.Collect();
Logger.WriteLine(Logger.Level.warn, $"We have out of memory while generating '{i}th' tokens, so terminate decoding for current sequences.");
break;
}
if (decodingOptions.BeamSearchSize > 1)
{
// Keep top N result and drop all others
for (int k = 0; k < batchSize; k++)
{
batch2beam2seq[k] = BeamSearch.GetTopNBSS(batch2beam2seq[k], decodingOptions.BeamSearchSize);
}
}
beam2batchStatus = Decoder.SwapBeamAndBatch(batch2beam2seq);
if (Decoder.AreAllSentsCompleted(beam2batchStatus))
{
break;
}
}
nr.Cost = 0.0f;
nr.Output = m_modelMetaData.TgtVocab.ExtractTokens(beam2batchStatus);
}
}
nr.RemoveDuplicatedEOS();
nrs.Add(nrCLS);
nrs.Add(nr);
return(nrs);
}