private bool CreateTrainableParameters(IModelMetaData mmd)
{
Logger.WriteLine($"Creating encoders and decoders...");
Seq2SeqModelMetaData modelMetaData = mmd as Seq2SeqModelMetaData;
RoundArray <int> raDeviceIds = new RoundArray <int>(DeviceIds);
if (modelMetaData.EncoderType == EncoderTypeEnums.BiLSTM)
{
m_encoder = new MultiProcessorNetworkWrapper <IEncoder>(
new BiEncoder("BiLSTMEncoder", modelMetaData.HiddenDim, modelMetaData.EmbeddingDim, modelMetaData.EncoderLayerDepth, raDeviceIds.GetNextItem()), DeviceIds);
m_decoderFFLayer = new MultiProcessorNetworkWrapper <FeedForwardLayer>(new FeedForwardLayer("FeedForward", modelMetaData.HiddenDim * 2, modelMetaData.Vocab.TargetWordSize, dropoutRatio: 0.0f, deviceId: raDeviceIds.GetNextItem()), DeviceIds);
}
else
{
m_encoder = new MultiProcessorNetworkWrapper <IEncoder>(
new TransformerEncoder("TransformerEncoder", modelMetaData.MultiHeadNum, modelMetaData.HiddenDim, modelMetaData.EmbeddingDim, modelMetaData.EncoderLayerDepth, m_dropoutRatio, raDeviceIds.GetNextItem()), DeviceIds);
m_decoderFFLayer = new MultiProcessorNetworkWrapper <FeedForwardLayer>(new FeedForwardLayer("FeedForward", modelMetaData.HiddenDim, modelMetaData.Vocab.TargetWordSize, dropoutRatio: 0.0f, deviceId: raDeviceIds.GetNextItem()), DeviceIds);
}
m_srcEmbedding = new MultiProcessorNetworkWrapper <IWeightTensor>(new WeightTensor(new long[2] {
modelMetaData.Vocab.SourceWordSize, modelMetaData.EmbeddingDim
}, raDeviceIds.GetNextItem(), normal: true, name: "SrcEmbeddings", isTrainable: true), DeviceIds);
// m_crfDecoder = new CRFDecoder(modelMetaData.Vocab.TargetWordSize);
return(true);
}
public AttentionSeq2Seq(int embeddingDim, int hiddenDim, int encoderLayerDepth, int decoderLayerDepth, Vocab vocab, string srcEmbeddingFilePath, string tgtEmbeddingFilePath,
string modelFilePath, float dropoutRatio, int multiHeadNum, ProcessorTypeEnums processorType, EncoderTypeEnums encoderType, DecoderTypeEnums decoderType, bool enableCoverageModel, int[] deviceIds,
bool isSrcEmbTrainable = true, bool isTgtEmbTrainable = true, bool isEncoderTrainable = true, bool isDecoderTrainable = true, int maxTgtSntSize = 128)
: base(deviceIds, processorType, modelFilePath)
{
m_modelMetaData = new Seq2SeqModelMetaData(hiddenDim, embeddingDim, encoderLayerDepth, decoderLayerDepth, multiHeadNum, encoderType, decoderType, vocab, enableCoverageModel);
m_dropoutRatio = dropoutRatio;
m_isSrcEmbTrainable = isSrcEmbTrainable;
m_isTgtEmbTrainable = isTgtEmbTrainable;
m_isEncoderTrainable = isEncoderTrainable;
m_isDecoderTrainable = isDecoderTrainable;
m_maxTgtSntSize = maxTgtSntSize;
//Initializng weights in encoders and decoders
CreateTrainableParameters(m_modelMetaData);
// Load external embedding from files
for (int i = 0; i < DeviceIds.Length; i++)
{
//If pre-trained embedding weights are speicifed, loading them from files
if (!string.IsNullOrEmpty(srcEmbeddingFilePath))
{
Logger.WriteLine($"Loading ExtEmbedding model from '{srcEmbeddingFilePath}' for source side.");
LoadWordEmbedding(srcEmbeddingFilePath, m_srcEmbedding.GetNetworkOnDevice(i), m_modelMetaData.Vocab.SrcWordToIndex);
}
if (!string.IsNullOrEmpty(tgtEmbeddingFilePath))
{
Logger.WriteLine($"Loading ExtEmbedding model from '{tgtEmbeddingFilePath}' for target side.");
LoadWordEmbedding(tgtEmbeddingFilePath, m_tgtEmbedding.GetNetworkOnDevice(i), m_modelMetaData.Vocab.TgtWordToIndex);
}
}
}
private bool CreateTrainableParameters(IModelMetaData mmd)
{
Logger.WriteLine($"Creating encoders and decoders...");
Seq2SeqModelMetaData modelMetaData = mmd as Seq2SeqModelMetaData;
RoundArray <int> raDeviceIds = new RoundArray <int>(DeviceIds);
if (modelMetaData.EncoderType == EncoderTypeEnums.BiLSTM)
{
m_encoder = new MultiProcessorNetworkWrapper <IEncoder>(
new BiEncoder("BiLSTMEncoder", modelMetaData.HiddenDim, modelMetaData.EmbeddingDim, modelMetaData.EncoderLayerDepth, raDeviceIds.GetNextItem(), isTrainable: m_isEncoderTrainable), DeviceIds);
m_decoder = new MultiProcessorNetworkWrapper <AttentionDecoder>(
new AttentionDecoder("AttnLSTMDecoder", modelMetaData.HiddenDim, modelMetaData.EmbeddingDim, modelMetaData.HiddenDim * 2,
modelMetaData.Vocab.TargetWordSize, m_dropoutRatio, modelMetaData.DecoderLayerDepth, raDeviceIds.GetNextItem(), modelMetaData.EnableCoverageModel, isTrainable: m_isDecoderTrainable), DeviceIds);
}
else
{
m_encoder = new MultiProcessorNetworkWrapper <IEncoder>(
new TransformerEncoder("TransformerEncoder", modelMetaData.MultiHeadNum, modelMetaData.HiddenDim, modelMetaData.EmbeddingDim, modelMetaData.EncoderLayerDepth, m_dropoutRatio, raDeviceIds.GetNextItem(),
isTrainable: m_isEncoderTrainable), DeviceIds);
m_decoder = new MultiProcessorNetworkWrapper <AttentionDecoder>(
new AttentionDecoder("AttnLSTMDecoder", modelMetaData.HiddenDim, modelMetaData.EmbeddingDim, modelMetaData.HiddenDim,
modelMetaData.Vocab.TargetWordSize, m_dropoutRatio, modelMetaData.DecoderLayerDepth, raDeviceIds.GetNextItem(), modelMetaData.EnableCoverageModel, isTrainable: m_isDecoderTrainable), DeviceIds);
}
m_srcEmbedding = new MultiProcessorNetworkWrapper <IWeightTensor>(new WeightTensor(new long[2] {
modelMetaData.Vocab.SourceWordSize, modelMetaData.EmbeddingDim
}, raDeviceIds.GetNextItem(), normal: true, name: "SrcEmbeddings", isTrainable: m_isSrcEmbTrainable), DeviceIds);
m_tgtEmbedding = new MultiProcessorNetworkWrapper <IWeightTensor>(new WeightTensor(new long[2] {
modelMetaData.Vocab.TargetWordSize, modelMetaData.EmbeddingDim
}, raDeviceIds.GetNextItem(), normal: true, name: "TgtEmbeddings", isTrainable: m_isTgtEmbTrainable), DeviceIds);
return(true);
}
private bool CreateTrainableParameters(IModelMetaData mmd)
{
Logger.WriteLine($"Creating encoders and decoders...");
Seq2SeqModelMetaData modelMetaData = mmd as Seq2SeqModelMetaData;
RoundArray <int> raDeviceIds = new RoundArray <int>(DeviceIds);
int contextDim = 0;
if (modelMetaData.EncoderType == EncoderTypeEnums.BiLSTM)
{
m_encoder = new MultiProcessorNetworkWrapper <IEncoder>(
new BiEncoder("BiLSTMEncoder", modelMetaData.HiddenDim, modelMetaData.SrcEmbeddingDim, modelMetaData.EncoderLayerDepth, raDeviceIds.GetNextItem(), isTrainable: m_isEncoderTrainable), DeviceIds);
contextDim = modelMetaData.HiddenDim * 2;
}
else
{
m_encoder = new MultiProcessorNetworkWrapper <IEncoder>(
new TransformerEncoder("TransformerEncoder", modelMetaData.MultiHeadNum, modelMetaData.HiddenDim, modelMetaData.SrcEmbeddingDim, modelMetaData.EncoderLayerDepth, m_dropoutRatio, raDeviceIds.GetNextItem(),
isTrainable: m_isEncoderTrainable), DeviceIds);
contextDim = modelMetaData.HiddenDim;
}
if (modelMetaData.DecoderType == DecoderTypeEnums.AttentionLSTM)
{
m_decoder = new MultiProcessorNetworkWrapper <IDecoder>(
new AttentionDecoder("AttnLSTMDecoder", modelMetaData.HiddenDim, modelMetaData.TgtEmbeddingDim, contextDim,
modelMetaData.Vocab.TargetWordSize, m_dropoutRatio, modelMetaData.DecoderLayerDepth, raDeviceIds.GetNextItem(), modelMetaData.EnableCoverageModel, isTrainable: m_isDecoderTrainable), DeviceIds);
}
else
{
m_decoder = new MultiProcessorNetworkWrapper <IDecoder>(
new TransformerDecoder("TransformerDecoder", modelMetaData.MultiHeadNum, modelMetaData.HiddenDim, modelMetaData.TgtEmbeddingDim, modelMetaData.Vocab.TargetWordSize, modelMetaData.EncoderLayerDepth, m_dropoutRatio, raDeviceIds.GetNextItem(),
isTrainable: m_isDecoderTrainable), DeviceIds);
}
if (modelMetaData.EncoderType == EncoderTypeEnums.Transformer || modelMetaData.DecoderType == DecoderTypeEnums.Transformer)
{
m_posEmbedding = new MultiProcessorNetworkWrapper <IWeightTensor>(BuildPositionWeightTensor(Math.Max(m_maxSrcSntSize, m_maxTgtSntSize) + 2, contextDim, raDeviceIds.GetNextItem(), "PosEmbedding", false), DeviceIds, true);
}
else
{
m_posEmbedding = null;
}
Logger.WriteLine($"Creating embeddings for source side. Shape = '({modelMetaData.Vocab.SourceWordSize} ,{modelMetaData.SrcEmbeddingDim})'");
m_srcEmbedding = new MultiProcessorNetworkWrapper <IWeightTensor>(new WeightTensor(new long[2] {
modelMetaData.Vocab.SourceWordSize, modelMetaData.SrcEmbeddingDim
}, raDeviceIds.GetNextItem(), normType: NormType.Uniform, fanOut: true, name: "SrcEmbeddings", isTrainable: m_isSrcEmbTrainable), DeviceIds);
Logger.WriteLine($"Creating embeddings for target side. Shape = '({modelMetaData.Vocab.TargetWordSize} ,{modelMetaData.TgtEmbeddingDim})'");
m_tgtEmbedding = new MultiProcessorNetworkWrapper <IWeightTensor>(new WeightTensor(new long[2] {
modelMetaData.Vocab.TargetWordSize, modelMetaData.TgtEmbeddingDim
}, raDeviceIds.GetNextItem(), normType: NormType.Uniform, fanOut: true, name: "TgtEmbeddings", isTrainable: m_isTgtEmbTrainable), DeviceIds);
return(true);
}
public SequenceLabel(int hiddenDim, int embeddingDim, int encoderLayerDepth, int multiHeadNum, EncoderTypeEnums encoderType,
float dropoutRatio, Vocab vocab, int[] deviceIds, ProcessorTypeEnums processorType, string modelFilePath) :
base(deviceIds, processorType, modelFilePath)
{
m_modelMetaData = new Seq2SeqModelMetaData(hiddenDim, embeddingDim, encoderLayerDepth, 0, multiHeadNum, encoderType, vocab);
m_dropoutRatio = dropoutRatio;
//Initializng weights in encoders and decoders
CreateTrainableParameters(m_modelMetaData);
}
public AttentionSeq2Seq(string modelFilePath, ProcessorTypeEnums processorType, int[] deviceIds, float dropoutRatio = 0.0f,
bool isSrcEmbTrainable = true, bool isTgtEmbTrainable = true, bool isEncoderTrainable = true, bool isDecoderTrainable = true)
: base(deviceIds, processorType, modelFilePath)
{
m_dropoutRatio = dropoutRatio;
m_isSrcEmbTrainable = isSrcEmbTrainable;
m_isTgtEmbTrainable = isTgtEmbTrainable;
m_isEncoderTrainable = isEncoderTrainable;
m_isDecoderTrainable = isDecoderTrainable;
m_modelMetaData = LoadModel(CreateTrainableParameters) as Seq2SeqModelMetaData;
}
public AttentionSeq2Seq(string modelFilePath, ProcessorTypeEnums processorType, int[] deviceIds, float dropoutRatio = 0.0f,
bool isSrcEmbTrainable = true, bool isTgtEmbTrainable = true, bool isEncoderTrainable = true, bool isDecoderTrainable = true,
int maxSrcSntSize = 128, int maxTgtSntSize = 128, float memoryUsageRatio = 0.9f, ShuffleEnums shuffleType = ShuffleEnums.Random, string[] compilerOptions = null)
: base(deviceIds, processorType, modelFilePath, memoryUsageRatio, compilerOptions)
{
m_dropoutRatio = dropoutRatio;
m_isSrcEmbTrainable = isSrcEmbTrainable;
m_isTgtEmbTrainable = isTgtEmbTrainable;
m_isEncoderTrainable = isEncoderTrainable;
m_isDecoderTrainable = isDecoderTrainable;
m_maxSrcSntSize = maxSrcSntSize;
m_maxTgtSntSize = maxTgtSntSize;
m_shuffleType = shuffleType;
m_modelMetaData = LoadModel(CreateTrainableParameters) as Seq2SeqModelMetaData;
}
public SequenceLabel(string modelFilePath, ProcessorTypeEnums processorType, int[] deviceIds, float dropoutRatio = 0.0f)
: base(deviceIds, processorType, modelFilePath)
{
m_dropoutRatio = dropoutRatio;
m_modelMetaData = LoadModel(CreateTrainableParameters) as Seq2SeqModelMetaData;
}
