private RNN <T> CreateNetwork()
{
RNN <T> rnn;
if (modelDirection == MODELDIRECTION.Forward)
{
var sparseFeatureSize = TrainingSet.SparseFeatureSize;
if (ModelType == MODELTYPE.Seq2Seq)
{
//[Sparse feature set of each state in target sequence][Sparse feature set of entire source sequence]
sparseFeatureSize += featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize;
Logger.WriteLine("Sparse Feature Format: [{0}][{1}] = {2}",
TrainingSet.SparseFeatureSize, featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize,
sparseFeatureSize);
}
var hiddenLayers = new List <SimpleLayer>();
for (var i = 0; i < hiddenLayersConfig.Count; i++)
{
SimpleLayer layer = null;
switch (hiddenLayersConfig[i].LayerType)
{
case LayerType.BPTT:
var bpttLayer = new BPTTLayer(hiddenLayersConfig[i] as BPTTLayerConfig);
layer = bpttLayer;
Logger.WriteLine("Create BPTT layer.");
break;
case LayerType.LSTM:
var lstmLayer = new LSTMLayer(hiddenLayersConfig[i] as LSTMLayerConfig);
layer = lstmLayer;
Logger.WriteLine("Create LSTM layer.");
break;
case LayerType.DropOut:
var dropoutLayer = new DropoutLayer(hiddenLayersConfig[i] as DropoutLayerConfig);
layer = dropoutLayer;
Logger.WriteLine("Create Dropout layer.");
break;
}
layer.InitializeWeights(sparseFeatureSize,
i == 0
? GetCurrentLayerDenseFeatureSize(TrainingSet.DenseFeatureSize)
: GetCurrentLayerDenseFeatureSize(hiddenLayers[i - 1].LayerSize));
Logger.WriteLine(
$"Create hidden layer {i}: size = {layer.LayerSize}, sparse feature size = {layer.SparseFeatureSize}, dense feature size = {layer.DenseFeatureSize}");
hiddenLayers.Add(layer);
}
SimpleLayer outputLayer = null;
outputLayerConfig.LayerSize = TrainingSet.TagSize;
switch (outputLayerConfig.LayerType)
{
case LayerType.NCESoftmax:
Logger.WriteLine("Create NCESoftmax layer as output layer");
var nceOutputLayer = new NCEOutputLayer(outputLayerConfig as NCELayerConfig);
nceOutputLayer.InitializeWeights(0,
GetCurrentLayerDenseFeatureSize(hiddenLayers[hiddenLayers.Count - 1].LayerSize));
outputLayer = nceOutputLayer;
break;
case LayerType.Softmax:
Logger.WriteLine("Create Softmax layer as output layer.");
outputLayer = new SimpleLayer(outputLayerConfig);
outputLayer.InitializeWeights(sparseFeatureSize,
GetCurrentLayerDenseFeatureSize(hiddenLayers[hiddenLayers.Count - 1].LayerSize));
break;
}
rnn = new ForwardRNN <T>(hiddenLayers, outputLayer);
}
else
{
var forwardHiddenLayers = new List <SimpleLayer>();
var backwardHiddenLayers = new List <SimpleLayer>();
for (var i = 0; i < hiddenLayersConfig.Count; i++)
{
SimpleLayer forwardLayer = null;
SimpleLayer backwardLayer = null;
switch (hiddenLayersConfig[i].LayerType)
{
case LayerType.BPTT:
//For BPTT layer
var forwardBPTTLayer = new BPTTLayer(hiddenLayersConfig[i] as BPTTLayerConfig);
forwardLayer = forwardBPTTLayer;
var backwardBPTTLayer = new BPTTLayer(hiddenLayersConfig[i] as BPTTLayerConfig);
backwardLayer = backwardBPTTLayer;
Logger.WriteLine("Create BPTT layer.");
break;
case LayerType.LSTM:
//For LSTM layer
var forwardLSTMLayer = new LSTMLayer(hiddenLayersConfig[i] as LSTMLayerConfig);
forwardLayer = forwardLSTMLayer;
var backwardLSTMLayer = new LSTMLayer(hiddenLayersConfig[i] as LSTMLayerConfig);
backwardLayer = backwardLSTMLayer;
Logger.WriteLine("Create LSTM layer.");
break;
case LayerType.DropOut:
var forwardDropoutLayer = new DropoutLayer(hiddenLayersConfig[i] as DropoutLayerConfig);
var backwardDropoutLayer = new DropoutLayer(hiddenLayersConfig[i] as DropoutLayerConfig);
Logger.WriteLine("Create Dropout layer.");
break;
}
if (i == 0)
{
forwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, TrainingSet.DenseFeatureSize);
backwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, TrainingSet.DenseFeatureSize);
}
else
{
forwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize,
forwardHiddenLayers[i - 1].LayerSize);
backwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize,
backwardHiddenLayers[i - 1].LayerSize);
}
Logger.WriteLine(
$"Create hidden layer {i}: size = {forwardLayer.LayerSize}, sparse feature size = {forwardLayer.SparseFeatureSize}, dense feature size = {forwardLayer.DenseFeatureSize}");
forwardHiddenLayers.Add(forwardLayer);
backwardHiddenLayers.Add(backwardLayer);
}
SimpleLayer outputLayer = null;
outputLayerConfig.LayerSize = TrainingSet.TagSize;
switch (outputLayerConfig.LayerType)
{
case LayerType.NCESoftmax:
Logger.WriteLine("Create NCESoftmax layer as output layer.");
var nceOutputLayer = new NCEOutputLayer(outputLayerConfig as NCELayerConfig);
nceOutputLayer.InitializeWeights(0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
outputLayer = nceOutputLayer;
break;
case LayerType.Softmax:
Logger.WriteLine("Create Softmax layer as output layer.");
outputLayer = new SimpleLayer(outputLayerConfig);
outputLayer.InitializeWeights(TrainingSet.SparseFeatureSize,
forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
break;
}
rnn = new BiRNN <T>(forwardHiddenLayers, backwardHiddenLayers, outputLayer);
}
return(rnn);
}
public void Train()
{
RNN rnn;
if (m_modelSetting.GetModelDirection() == 0)
{
if (m_modelSetting.GetModelType() == 0)
{
SimpleRNN sRNN = new SimpleRNN();
sRNN.setBPTT(m_modelSetting.GetBptt() + 1);
sRNN.setBPTTBlock(30);
rnn = sRNN;
}
else
{
LSTMRNN lstmRNN = new LSTMRNN();
rnn = lstmRNN;
}
}
else
{
BiRNN biRNN = new BiRNN(m_modelSetting.GetModelType());
rnn = biRNN;
}
rnn.SetModelDirection(m_modelSetting.GetModelDirection());
rnn.SetTrainingSet(m_TrainingSet);
rnn.SetValidationSet(m_ValidationSet);
rnn.SetModelFile(m_modelSetting.GetModelFile());
rnn.SetSaveStep(m_modelSetting.GetSaveStep());
rnn.SetMaxIter(m_modelSetting.GetMaxIteration());
rnn.SetCRFTraining(m_modelSetting.IsCRFTraining());
rnn.SetLearningRate(m_modelSetting.GetLearningRate());
rnn.SetGradientCutoff(15.0);
rnn.SetRegularization(m_modelSetting.GetRegularization());
rnn.SetHiddenLayerSize(m_modelSetting.GetNumHidden());
rnn.SetTagBigramTransitionWeight(m_modelSetting.GetTagTransitionWeight());
rnn.initMem();
//Create tag-bigram transition probability matrix only for sequence RNN mode
if (m_modelSetting.IsCRFTraining() == true)
{
rnn.setTagBigramTransition(m_LabelBigramTransition);
}
Console.WriteLine();
Console.WriteLine("[TRACE] Iterative training begins ...");
while (rnn.ShouldTrainingStop() == false)
{
//Start to train model
rnn.TrainNet();
//Validate the model by validated corpus
if (rnn.ValidateNet() == true)
{
//If current model is better than before, save it into file
Console.Write("Saving better model into file {0}...", m_modelSetting.GetModelFile());
rnn.saveNetBin(m_modelSetting.GetModelFile());
Console.WriteLine("Done.");
}
//else
//{
// Console.Write("Loading previous best model from file {0}...", m_modelSetting.GetModelFile());
// rnn.loadNetBin(m_modelSetting.GetModelFile());
// Console.WriteLine("Done.");
//}
}
}
public void Train()
{
RNN rnn;
if (m_modelSetting.ModelDirection == 0)
{
if (m_modelSetting.ModelType == 0)
{
SimpleRNN sRNN = new SimpleRNN();
sRNN.setBPTT(m_modelSetting.Bptt + 1);
sRNN.setBPTTBlock(10);
rnn = sRNN;
}
else
{
rnn = new LSTMRNN();
}
}
else
{
if (m_modelSetting.ModelType == 0)
{
SimpleRNN sForwardRNN = new SimpleRNN();
SimpleRNN sBackwardRNN = new SimpleRNN();
sForwardRNN.setBPTT(m_modelSetting.Bptt + 1);
sForwardRNN.setBPTTBlock(10);
sBackwardRNN.setBPTT(m_modelSetting.Bptt + 1);
sBackwardRNN.setBPTTBlock(10);
rnn = new BiRNN(sForwardRNN, sBackwardRNN);
}
else
{
rnn = new BiRNN(new LSTMRNN(), new LSTMRNN());
}
}
rnn.ModelDirection = (MODELDIRECTION)m_modelSetting.ModelDirection;
rnn.bVQ = (m_modelSetting.VQ != 0) ? true : false;
rnn.ModelFile = m_modelSetting.ModelFile;
rnn.SaveStep = m_modelSetting.SaveStep;
rnn.MaxIter = m_modelSetting.MaxIteration;
rnn.IsCRFTraining = m_modelSetting.IsCRFTraining;
rnn.LearningRate = m_modelSetting.LearningRate;
rnn.GradientCutoff = m_modelSetting.GradientCutoff;
rnn.Dropout = m_modelSetting.Dropout;
rnn.L1 = m_modelSetting.NumHidden;
rnn.DenseFeatureSize = TrainingSet.DenseFeatureSize();
rnn.L0 = TrainingSet.GetSparseDimension();
rnn.L2 = TrainingSet.TagSize;
rnn.InitMem();
//Create tag-bigram transition probability matrix only for sequence RNN mode
if (m_modelSetting.IsCRFTraining)
{
rnn.setTagBigramTransition(TrainingSet.CRFLabelBigramTransition);
}
Logger.WriteLine("");
Logger.WriteLine("Iterative training begins ...");
double lastPPL = double.MaxValue;
double lastAlpha = rnn.LearningRate;
int iter = 0;
while (true)
{
Logger.WriteLine("Cleaning training status...");
rnn.CleanStatus();
if (rnn.MaxIter > 0 && iter > rnn.MaxIter)
{
Logger.WriteLine("We have trained this model {0} iteration, exit.");
break;
}
//Start to train model
double ppl = rnn.TrainNet(TrainingSet, iter);
if (ppl >= lastPPL && lastAlpha != rnn.LearningRate)
{
//Although we reduce alpha value, we still cannot get better result.
Logger.WriteLine("Current perplexity({0}) is larger than the previous one({1}). End training early.", ppl, lastPPL);
Logger.WriteLine("Current alpha: {0}, the previous alpha: {1}", rnn.LearningRate, lastAlpha);
break;
}
lastAlpha = rnn.LearningRate;
//Validate the model by validated corpus
if (ValidationSet != null)
{
Logger.WriteLine("Verify model on validated corpus.");
if (rnn.ValidateNet(ValidationSet, iter) == true)
{
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", m_modelSetting.ModelFile);
rnn.SaveModel(m_modelSetting.ModelFile);
}
}
else if (ppl < lastPPL)
{
//We don't have validate corpus, but we get a better result on training corpus
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", m_modelSetting.ModelFile);
rnn.SaveModel(m_modelSetting.ModelFile);
}
if (ppl >= lastPPL)
{
//We cannot get a better result on training corpus, so reduce learning rate
rnn.LearningRate = rnn.LearningRate / 2.0f;
}
lastPPL = ppl;
iter++;
}
}
public void Train()
{
RNN <T> rnn;
if (ModelSettings.ModelDirection == 0)
{
int sparseFeatureSize = TrainingSet.SparseFeatureSize;
if (ModelSettings.IsSeq2SeqTraining)
{
//[Sparse feature set of each state in target sequence][Sparse feature set of entire source sequence]
sparseFeatureSize += Featurizer.AutoEncoder.Featurizer.SparseFeatureSize;
Logger.WriteLine("Sparse Feature Format: [{0}][{1}] = {2}",
TrainingSet.SparseFeatureSize, Featurizer.AutoEncoder.Featurizer.SparseFeatureSize, sparseFeatureSize);
}
List <SimpleLayer> hiddenLayers = new List <SimpleLayer>();
for (int i = 0; i < ModelSettings.HiddenLayerSizeList.Count; i++)
{
SimpleLayer layer = null;
if (ModelSettings.ModelType == LayerType.BPTT)
{
BPTTLayer bpttLayer = new BPTTLayer(ModelSettings.HiddenLayerSizeList[i], ModelSettings);
layer = bpttLayer;
}
else if (ModelSettings.ModelType == LayerType.LSTM)
{
LSTMLayer lstmLayer = new LSTMLayer(ModelSettings.HiddenLayerSizeList[i], ModelSettings);
layer = lstmLayer;
}
else
{
throw new System.Exception(string.Format("Invalidate hidden layer type: {0}", ModelSettings.ModelType.ToString()));
}
if (i == 0)
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, ModelSettings.HiddenLayerSizeList[i], sparseFeatureSize, TrainingSet.DenseFeatureSize);
if (ModelSettings.IsSeq2SeqTraining)
{
Logger.WriteLine("For seq2seq training, we have {0} sprase feature and {1} dense feature from source sequence.",
Featurizer.AutoEncoder.Featurizer.SparseFeatureSize, Featurizer.AutoEncoder.GetTopHiddenLayerSize());
}
layer.InitializeWeights(sparseFeatureSize, GetCurrentLayerDenseFeatureSize(TrainingSet.DenseFeatureSize));
}
else
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, ModelSettings.HiddenLayerSizeList[i], sparseFeatureSize, hiddenLayers[i - 1].LayerSize);
layer.InitializeWeights(sparseFeatureSize, GetCurrentLayerDenseFeatureSize(hiddenLayers[i - 1].LayerSize));
}
hiddenLayers.Add(layer);
}
if (ModelSettings.Dropout > 0)
{
Logger.WriteLine("Adding dropout layer");
DropoutLayer dropoutLayer = new DropoutLayer(hiddenLayers[hiddenLayers.Count - 1].LayerSize, ModelSettings);
dropoutLayer.InitializeWeights(0, GetCurrentLayerDenseFeatureSize(hiddenLayers[hiddenLayers.Count - 1].LayerSize));
hiddenLayers.Add(dropoutLayer);
}
SimpleLayer outputLayer;
if (ModelSettings.OutputLayerType == LayerType.NCESoftmax)
{
Logger.WriteLine("Create NCESoftmax layer as output layer");
NCEOutputLayer nceOutputLayer = new NCEOutputLayer(TrainingSet.TagSize, ModelSettings);
nceOutputLayer.BuildStatisticData <T>(TrainingSet);
nceOutputLayer.InitializeWeights(0, GetCurrentLayerDenseFeatureSize(hiddenLayers[hiddenLayers.Count - 1].LayerSize));
outputLayer = nceOutputLayer;
}
else if (ModelSettings.OutputLayerType == LayerType.Softmax)
{
Logger.WriteLine("Create Softmax layer as output layer.");
outputLayer = new SimpleLayer(TrainingSet.TagSize);
outputLayer.InitializeWeights(sparseFeatureSize, GetCurrentLayerDenseFeatureSize(hiddenLayers[hiddenLayers.Count - 1].LayerSize));
}
else
{
throw new System.Exception(string.Format("Invalidate output layer type: {0}", ModelSettings.OutputLayerType.ToString()));
}
rnn = new ForwardRNN <T>(hiddenLayers, outputLayer);
}
else
{
List <SimpleLayer> forwardHiddenLayers = new List <SimpleLayer>();
List <SimpleLayer> backwardHiddenLayers = new List <SimpleLayer>();
for (int i = 0; i < ModelSettings.HiddenLayerSizeList.Count; i++)
{
SimpleLayer forwardLayer = null;
SimpleLayer backwardLayer = null;
if (ModelSettings.ModelType == LayerType.BPTT)
{
//For BPTT layer
BPTTLayer forwardBPTTLayer = new BPTTLayer(ModelSettings.HiddenLayerSizeList[i], ModelSettings);
forwardLayer = forwardBPTTLayer;
BPTTLayer backwardBPTTLayer = new BPTTLayer(ModelSettings.HiddenLayerSizeList[i], ModelSettings);
backwardLayer = backwardBPTTLayer;
}
else if (ModelSettings.ModelType == LayerType.LSTM)
{
//For LSTM layer
LSTMLayer forwardLSTMLayer = new LSTMLayer(ModelSettings.HiddenLayerSizeList[i], ModelSettings);
forwardLayer = forwardLSTMLayer;
LSTMLayer backwardLSTMLayer = new LSTMLayer(ModelSettings.HiddenLayerSizeList[i], ModelSettings);
backwardLayer = backwardLSTMLayer;
}
else
{
throw new System.Exception(string.Format("Invalidate hidden layer type: {0}", ModelSettings.ModelType.ToString()));
}
if (i == 0)
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, ModelSettings.HiddenLayerSizeList[i], TrainingSet.SparseFeatureSize, TrainingSet.DenseFeatureSize);
forwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, TrainingSet.DenseFeatureSize);
backwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, TrainingSet.DenseFeatureSize);
}
else
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, ModelSettings.HiddenLayerSizeList[i], TrainingSet.SparseFeatureSize, forwardHiddenLayers[i - 1].LayerSize);
forwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, forwardHiddenLayers[i - 1].LayerSize);
backwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, backwardHiddenLayers[i - 1].LayerSize);
}
forwardHiddenLayers.Add(forwardLayer);
backwardHiddenLayers.Add(backwardLayer);
}
if (ModelSettings.Dropout > 0)
{
Logger.WriteLine("Adding dropout layers");
DropoutLayer forwardDropoutLayer = new DropoutLayer(forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize, ModelSettings);
DropoutLayer backwardDropoutLayer = new DropoutLayer(backwardHiddenLayers[backwardHiddenLayers.Count - 1].LayerSize, ModelSettings);
forwardDropoutLayer.InitializeWeights(0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
backwardDropoutLayer.InitializeWeights(0, backwardHiddenLayers[backwardHiddenLayers.Count - 1].LayerSize);
forwardHiddenLayers.Add(forwardDropoutLayer);
backwardHiddenLayers.Add(backwardDropoutLayer);
}
SimpleLayer outputLayer;
if (ModelSettings.OutputLayerType == LayerType.NCESoftmax)
{
Logger.WriteLine("Create NCESoftmax layer as output layer.");
NCEOutputLayer nceOutputLayer = new NCEOutputLayer(TrainingSet.TagSize, ModelSettings);
nceOutputLayer.BuildStatisticData <T>(TrainingSet);
nceOutputLayer.InitializeWeights(0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
outputLayer = nceOutputLayer;
}
else if (ModelSettings.OutputLayerType == LayerType.Softmax)
{
Logger.WriteLine("Create Softmax layer as output layer.");
outputLayer = new SimpleLayer(TrainingSet.TagSize);
outputLayer.InitializeWeights(TrainingSet.SparseFeatureSize, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
}
else
{
throw new System.Exception(string.Format("Invalidate output layer type: {0}", ModelSettings.OutputLayerType.ToString()));
}
rnn = new BiRNN <T>(forwardHiddenLayers, backwardHiddenLayers, outputLayer);
}
rnn.ModelDirection = (MODELDIRECTION)ModelSettings.ModelDirection;
rnn.bVQ = (ModelSettings.VQ != 0) ? true : false;
rnn.ModelFile = ModelSettings.ModelFile;
rnn.SaveStep = ModelSettings.SaveStep;
rnn.MaxIter = ModelSettings.MaxIteration;
rnn.IsCRFTraining = ModelSettings.IsCRFTraining;
rnn.ModelType = ModelSettings.IsSeq2SeqTraining ? MODELTYPE.SEQ2SEQ : MODELTYPE.SEQLABEL;
if (rnn.ModelDirection == MODELDIRECTION.BI_DIRECTIONAL && rnn.ModelType == MODELTYPE.SEQ2SEQ)
{
throw new System.Exception("Bi-directional RNN model doesn't support sequence-to-sequence model.");
}
RNNHelper.LearningRate = ModelSettings.LearningRate;
RNNHelper.GradientCutoff = ModelSettings.GradientCutoff;
RNNHelper.IsConstAlpha = ModelSettings.IsConstAlpha;
//Create tag-bigram transition probability matrix only for sequence RNN mode
if (ModelSettings.IsCRFTraining)
{
rnn.setTagBigramTransition(TrainingSet.CRFLabelBigramTransition);
}
Logger.WriteLine("");
Logger.WriteLine("Iterative training begins ...");
double lastPPL = double.MaxValue;
double lastAlpha = RNNHelper.LearningRate;
int iter = 0;
while (true)
{
Logger.WriteLine("Cleaning training status...");
rnn.CleanStatus();
if (rnn.MaxIter > 0 && iter > rnn.MaxIter)
{
Logger.WriteLine("We have trained this model {0} iteration, exit.");
break;
}
//Start to train model
double ppl = rnn.TrainNet(TrainingSet, iter);
if (ppl >= lastPPL && lastAlpha != RNNHelper.LearningRate)
{
//Although we reduce alpha value, we still cannot get better result.
Logger.WriteLine("Current perplexity({0}) is larger than the previous one({1}). End training early.", ppl, lastPPL);
Logger.WriteLine("Current alpha: {0}, the previous alpha: {1}", RNNHelper.LearningRate, lastAlpha);
break;
}
lastAlpha = RNNHelper.LearningRate;
//Validate the model by validated corpus
if (ValidationSet != null)
{
Logger.WriteLine("Verify model on validated corpus.");
if (rnn.ValidateNet(ValidationSet, iter) == true)
{
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", ModelSettings.ModelFile);
rnn.SaveModel(ModelSettings.ModelFile);
}
}
else if (ppl < lastPPL)
{
//We don't have validate corpus, but we get a better result on training corpus
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", ModelSettings.ModelFile);
rnn.SaveModel(ModelSettings.ModelFile);
}
if (ppl >= lastPPL)
{
//We cannot get a better result on training corpus, so reduce learning rate
RNNHelper.LearningRate = RNNHelper.LearningRate / 2.0f;
}
lastPPL = ppl;
iter++;
}
}
public void Train()
{
RNN rnn;
if (m_modelSetting.ModelDirection == 0)
{
List <SimpleLayer> hiddenLayers = new List <SimpleLayer>();
for (int i = 0; i < m_modelSetting.HiddenLayerSizeList.Count; i++)
{
SimpleLayer layer = null;
if (m_modelSetting.ModelType == LayerType.BPTT)
{
BPTTLayer bpttLayer = new BPTTLayer(m_modelSetting.HiddenLayerSizeList[i], m_modelSetting);
layer = bpttLayer;
}
else if (m_modelSetting.ModelType == LayerType.LSTM)
{
LSTMLayer lstmLayer = new LSTMLayer(m_modelSetting.HiddenLayerSizeList[i], m_modelSetting);
layer = lstmLayer;
}
else
{
throw new System.Exception(string.Format("Invalidate hidden layer type: {0}", m_modelSetting.ModelType.ToString()));
}
if (i == 0)
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.HiddenLayerSizeList[i], TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
layer.InitializeWeights(TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
}
else
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.HiddenLayerSizeList[i], TrainingSet.GetSparseDimension(), hiddenLayers[i - 1].LayerSize);
layer.InitializeWeights(TrainingSet.GetSparseDimension(), hiddenLayers[i - 1].LayerSize);
}
hiddenLayers.Add(layer);
}
if (m_modelSetting.Dropout > 0)
{
Logger.WriteLine("Adding dropout layer");
DropoutLayer dropoutLayer = new DropoutLayer(hiddenLayers[hiddenLayers.Count - 1].LayerSize, m_modelSetting);
dropoutLayer.InitializeWeights(0, hiddenLayers[hiddenLayers.Count - 1].LayerSize);
hiddenLayers.Add(dropoutLayer);
}
SimpleLayer outputLayer;
if (m_modelSetting.OutputLayerType == LayerType.NCESoftmax)
{
Logger.WriteLine("Create NCESoftmax layer as output layer.");
NCEOutputLayer nceOutputLayer = new NCEOutputLayer(TrainingSet.TagSize, m_modelSetting);
nceOutputLayer.InitializeWeights(0, hiddenLayers[hiddenLayers.Count - 1].LayerSize);
outputLayer = nceOutputLayer;
}
else if (m_modelSetting.OutputLayerType == LayerType.Softmax)
{
Logger.WriteLine("Create Softmax layer as output layer.");
outputLayer = new SimpleLayer(TrainingSet.TagSize);
outputLayer.InitializeWeights(TrainingSet.GetSparseDimension(), hiddenLayers[hiddenLayers.Count - 1].LayerSize);
}
else
{
throw new System.Exception(string.Format("Invalidate output layer type: {0}", m_modelSetting.OutputLayerType.ToString()));
}
rnn = new ForwardRNN(hiddenLayers, outputLayer);
}
else
{
List <SimpleLayer> forwardHiddenLayers = new List <SimpleLayer>();
List <SimpleLayer> backwardHiddenLayers = new List <SimpleLayer>();
for (int i = 0; i < m_modelSetting.HiddenLayerSizeList.Count; i++)
{
SimpleLayer forwardLayer = null;
SimpleLayer backwardLayer = null;
if (m_modelSetting.ModelType == LayerType.BPTT)
{
//For BPTT layer
BPTTLayer forwardBPTTLayer = new BPTTLayer(m_modelSetting.HiddenLayerSizeList[i], m_modelSetting);
forwardLayer = forwardBPTTLayer;
BPTTLayer backwardBPTTLayer = new BPTTLayer(m_modelSetting.HiddenLayerSizeList[i], m_modelSetting);
backwardLayer = backwardBPTTLayer;
}
else if (m_modelSetting.ModelType == LayerType.LSTM)
{
//For LSTM layer
LSTMLayer forwardLSTMLayer = new LSTMLayer(m_modelSetting.HiddenLayerSizeList[i], m_modelSetting);
forwardLayer = forwardLSTMLayer;
LSTMLayer backwardLSTMLayer = new LSTMLayer(m_modelSetting.HiddenLayerSizeList[i], m_modelSetting);
backwardLayer = backwardLSTMLayer;
}
else
{
throw new System.Exception(string.Format("Invalidate hidden layer type: {0}", m_modelSetting.ModelType.ToString()));
}
if (i == 0)
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.HiddenLayerSizeList[i], TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
forwardLayer.InitializeWeights(TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
backwardLayer.InitializeWeights(TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
}
else
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.HiddenLayerSizeList[i], TrainingSet.GetSparseDimension(), forwardHiddenLayers[i - 1].LayerSize);
forwardLayer.InitializeWeights(TrainingSet.GetSparseDimension(), forwardHiddenLayers[i - 1].LayerSize);
backwardLayer.InitializeWeights(TrainingSet.GetSparseDimension(), backwardHiddenLayers[i - 1].LayerSize);
}
forwardHiddenLayers.Add(forwardLayer);
backwardHiddenLayers.Add(backwardLayer);
}
if (m_modelSetting.Dropout > 0)
{
Logger.WriteLine("Adding dropout layers");
DropoutLayer forwardDropoutLayer = new DropoutLayer(forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize, m_modelSetting);
DropoutLayer backwardDropoutLayer = new DropoutLayer(backwardHiddenLayers[backwardHiddenLayers.Count - 1].LayerSize, m_modelSetting);
forwardDropoutLayer.InitializeWeights(0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
backwardDropoutLayer.InitializeWeights(0, backwardHiddenLayers[backwardHiddenLayers.Count - 1].LayerSize);
forwardHiddenLayers.Add(forwardDropoutLayer);
backwardHiddenLayers.Add(backwardDropoutLayer);
}
SimpleLayer outputLayer;
if (m_modelSetting.OutputLayerType == LayerType.NCESoftmax)
{
Logger.WriteLine("Create NCESoftmax layer as output layer.");
NCEOutputLayer nceOutputLayer = new NCEOutputLayer(TrainingSet.TagSize, m_modelSetting);
nceOutputLayer.InitializeWeights(0, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
outputLayer = nceOutputLayer;
}
else if (m_modelSetting.OutputLayerType == LayerType.Softmax)
{
Logger.WriteLine("Create Softmax layer as output layer.");
outputLayer = new SimpleLayer(TrainingSet.TagSize);
outputLayer.InitializeWeights(TrainingSet.GetSparseDimension(), forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize);
}
else
{
throw new System.Exception(string.Format("Invalidate output layer type: {0}", m_modelSetting.OutputLayerType.ToString()));
}
rnn = new BiRNN(forwardHiddenLayers, backwardHiddenLayers, outputLayer);
}
rnn.ModelDirection = (MODELDIRECTION)m_modelSetting.ModelDirection;
rnn.bVQ = (m_modelSetting.VQ != 0) ? true : false;
rnn.ModelFile = m_modelSetting.ModelFile;
rnn.SaveStep = m_modelSetting.SaveStep;
rnn.MaxIter = m_modelSetting.MaxIteration;
rnn.IsCRFTraining = m_modelSetting.IsCRFTraining;
RNNHelper.LearningRate = m_modelSetting.LearningRate;
RNNHelper.GradientCutoff = m_modelSetting.GradientCutoff;
//Create tag-bigram transition probability matrix only for sequence RNN mode
if (m_modelSetting.IsCRFTraining)
{
rnn.setTagBigramTransition(TrainingSet.CRFLabelBigramTransition);
}
Logger.WriteLine("");
Logger.WriteLine("Iterative training begins ...");
double lastPPL = double.MaxValue;
double lastAlpha = RNNHelper.LearningRate;
int iter = 0;
while (true)
{
Logger.WriteLine("Cleaning training status...");
rnn.CleanStatus();
if (rnn.MaxIter > 0 && iter > rnn.MaxIter)
{
Logger.WriteLine("We have trained this model {0} iteration, exit.");
break;
}
//Start to train model
double ppl = rnn.TrainNet(TrainingSet, iter);
if (ppl >= lastPPL && lastAlpha != RNNHelper.LearningRate)
{
//Although we reduce alpha value, we still cannot get better result.
Logger.WriteLine("Current perplexity({0}) is larger than the previous one({1}). End training early.", ppl, lastPPL);
Logger.WriteLine("Current alpha: {0}, the previous alpha: {1}", RNNHelper.LearningRate, lastAlpha);
break;
}
lastAlpha = RNNHelper.LearningRate;
//Validate the model by validated corpus
if (ValidationSet != null)
{
Logger.WriteLine("Verify model on validated corpus.");
if (rnn.ValidateNet(ValidationSet, iter) == true)
{
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", m_modelSetting.ModelFile);
rnn.SaveModel(m_modelSetting.ModelFile);
}
}
else if (ppl < lastPPL)
{
//We don't have validate corpus, but we get a better result on training corpus
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", m_modelSetting.ModelFile);
rnn.SaveModel(m_modelSetting.ModelFile);
}
if (ppl >= lastPPL)
{
//We cannot get a better result on training corpus, so reduce learning rate
RNNHelper.LearningRate = RNNHelper.LearningRate / 2.0f;
}
lastPPL = ppl;
iter++;
}
}
public void Train()
{
RNN rnn;
if (m_modelSetting.ModelDirection == 0)
{
if (m_modelSetting.ModelType == 0)
{
SimpleRNN sRNN = new SimpleRNN();
sRNN.setBPTT(m_modelSetting.Bptt + 1);
sRNN.setBPTTBlock(10);
rnn = sRNN;
}
else
{
rnn = new LSTMRNN();
}
}
else
{
if (m_modelSetting.ModelType == 0)
{
SimpleRNN sForwardRNN = new SimpleRNN();
SimpleRNN sBackwardRNN = new SimpleRNN();
sForwardRNN.setBPTT(m_modelSetting.Bptt + 1);
sForwardRNN.setBPTTBlock(10);
sBackwardRNN.setBPTT(m_modelSetting.Bptt + 1);
sBackwardRNN.setBPTTBlock(10);
rnn = new BiRNN(sForwardRNN, sBackwardRNN);
}
else
{
rnn = new BiRNN(new LSTMRNN(), new LSTMRNN());
}
}
rnn.ModelDirection = (MODELDIRECTION)m_modelSetting.ModelDirection;
rnn.ModelFile = m_modelSetting.ModelFile;
rnn.SaveStep = m_modelSetting.SaveStep;
rnn.MaxIter = m_modelSetting.MaxIteration;
rnn.IsCRFTraining = m_modelSetting.IsCRFTraining;
rnn.LearningRate = m_modelSetting.LearningRate;
rnn.GradientCutoff = 15.0f;
rnn.Dropout = m_modelSetting.Dropout;
rnn.L1 = m_modelSetting.NumHidden;
rnn.DenseFeatureSize = TrainingSet.DenseFeatureSize();
rnn.L0 = TrainingSet.GetSparseDimension();
rnn.L2 = TrainingSet.TagSize;
rnn.initMem();
//Create tag-bigram transition probability matrix only for sequence RNN mode
if (m_modelSetting.IsCRFTraining)
{
rnn.setTagBigramTransition(TrainingSet.CRFLabelBigramTransition);
}
Logger.WriteLine(Logger.Level.info, "");
Logger.WriteLine(Logger.Level.info, "[TRACE] Iterative training begins ...");
double lastPPL = double.MaxValue;
double lastAlpha = rnn.LearningRate;
int iter = 0;
while (true)
{
if (rnn.MaxIter > 0 && iter > rnn.MaxIter)
{
Logger.WriteLine(Logger.Level.info, "We have trained this model {0} iteration, exit.");
break;
}
//Start to train model
double ppl = rnn.TrainNet(TrainingSet, iter);
//Validate the model by validated corpus
bool betterValidateNet = false;
if (rnn.ValidateNet(ValidationSet, iter) == true)
{
//If current model is better than before, save it into file
Logger.WriteLine(Logger.Level.info, "Saving better model into file {0}...", m_modelSetting.ModelFile);
rnn.saveNetBin(m_modelSetting.ModelFile);
betterValidateNet = true;
}
if (ppl >= lastPPL && lastAlpha != rnn.LearningRate)
{
//Although we reduce alpha value, we still cannot get better result.
Logger.WriteLine(Logger.Level.info, "Current perplexity({0}) is larger than the previous one({1}). End training early.", ppl, lastPPL);
Logger.WriteLine(Logger.Level.info, "Current alpha: {0}, the previous alpha: {1}", rnn.LearningRate, lastAlpha);
break;
}
lastAlpha = rnn.LearningRate;
if (betterValidateNet == false)
{
rnn.LearningRate = rnn.LearningRate / 2.0f;
}
lastPPL = ppl;
iter++;
}
}
public void Train()
{
RNN rnn;
if (m_modelSetting.ModelDirection == 0)
{
List<SimpleLayer> hiddenLayers = new List<SimpleLayer>();
for (int i = 0; i < m_modelSetting.NumHidden.Count; i++)
{
SimpleLayer layer = null;
if (m_modelSetting.ModelType == 0)
{
BPTTLayer bpttLayer = new BPTTLayer(m_modelSetting.NumHidden[i]);
bpttLayer.bptt = m_modelSetting.Bptt + 1;
bpttLayer.bptt_block = 10;
bpttLayer.Dropout = m_modelSetting.Dropout;
layer = bpttLayer;
}
else
{
LSTMLayer lstmLayer = new LSTMLayer(m_modelSetting.NumHidden[i]);
lstmLayer.Dropout = m_modelSetting.Dropout;
layer = lstmLayer;
}
if (i == 0)
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.NumHidden[i], TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
layer.InitializeWeights(TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
}
else
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.NumHidden[i], 0, hiddenLayers[i - 1].LayerSize);
layer.InitializeWeights(0, hiddenLayers[i - 1].LayerSize);
}
hiddenLayers.Add(layer);
}
rnn = new ForwardRNN(hiddenLayers, TrainingSet.TagSize);
}
else
{
List<SimpleLayer> forwardHiddenLayers = new List<SimpleLayer>();
List<SimpleLayer> backwardHiddenLayers = new List<SimpleLayer>();
for (int i = 0; i < m_modelSetting.NumHidden.Count; i++)
{
SimpleLayer forwardLayer = null;
SimpleLayer backwardLayer = null;
if (m_modelSetting.ModelType == 0)
{
//For BPTT layer
BPTTLayer forwardBPTTLayer = new BPTTLayer(m_modelSetting.NumHidden[i]);
forwardBPTTLayer.bptt = m_modelSetting.Bptt + 1;
forwardBPTTLayer.bptt_block = 10;
forwardBPTTLayer.Dropout = m_modelSetting.Dropout;
forwardLayer = forwardBPTTLayer;
BPTTLayer backwardBPTTLayer = new BPTTLayer(m_modelSetting.NumHidden[i]);
backwardBPTTLayer.bptt = m_modelSetting.Bptt + 1;
backwardBPTTLayer.bptt_block = 10;
backwardBPTTLayer.Dropout = m_modelSetting.Dropout;
backwardLayer = backwardBPTTLayer;
}
else
{
//For LSTM layer
LSTMLayer forwardLSTMLayer = new LSTMLayer(m_modelSetting.NumHidden[i]);
forwardLSTMLayer.Dropout = m_modelSetting.Dropout;
forwardLayer = forwardLSTMLayer;
LSTMLayer backwardLSTMLayer = new LSTMLayer(m_modelSetting.NumHidden[i]);
backwardLSTMLayer.Dropout = m_modelSetting.Dropout;
backwardLayer = backwardLSTMLayer;
}
if (i == 0)
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.NumHidden[i], TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
forwardLayer.InitializeWeights(TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
backwardLayer.InitializeWeights(TrainingSet.GetSparseDimension(), TrainingSet.DenseFeatureSize());
}
else
{
Logger.WriteLine("Create hidden layer {0}: size = {1}, sparse feature size = {2}, dense feature size = {3}",
i, m_modelSetting.NumHidden[i], 0, forwardHiddenLayers[i - 1].LayerSize);
forwardLayer.InitializeWeights(0, forwardHiddenLayers[i - 1].LayerSize);
backwardLayer.InitializeWeights(0, backwardHiddenLayers[i - 1].LayerSize);
}
forwardHiddenLayers.Add(forwardLayer);
backwardHiddenLayers.Add(backwardLayer);
}
rnn = new BiRNN(forwardHiddenLayers, backwardHiddenLayers, TrainingSet.TagSize);
}
rnn.ModelDirection = (MODELDIRECTION)m_modelSetting.ModelDirection;
rnn.bVQ = (m_modelSetting.VQ != 0) ? true : false;
rnn.ModelFile = m_modelSetting.ModelFile;
rnn.SaveStep = m_modelSetting.SaveStep;
rnn.MaxIter = m_modelSetting.MaxIteration;
rnn.IsCRFTraining = m_modelSetting.IsCRFTraining;
RNNHelper.LearningRate = m_modelSetting.LearningRate;
RNNHelper.GradientCutoff = m_modelSetting.GradientCutoff;
//Create tag-bigram transition probability matrix only for sequence RNN mode
if (m_modelSetting.IsCRFTraining)
{
rnn.setTagBigramTransition(TrainingSet.CRFLabelBigramTransition);
}
Logger.WriteLine("");
Logger.WriteLine("Iterative training begins ...");
double lastPPL = double.MaxValue;
double lastAlpha = RNNHelper.LearningRate;
int iter = 0;
while (true)
{
Logger.WriteLine("Cleaning training status...");
rnn.CleanStatus();
if (rnn.MaxIter > 0 && iter > rnn.MaxIter)
{
Logger.WriteLine("We have trained this model {0} iteration, exit.");
break;
}
//Start to train model
double ppl = rnn.TrainNet(TrainingSet, iter);
if (ppl >= lastPPL && lastAlpha != RNNHelper.LearningRate)
{
//Although we reduce alpha value, we still cannot get better result.
Logger.WriteLine("Current perplexity({0}) is larger than the previous one({1}). End training early.", ppl, lastPPL);
Logger.WriteLine("Current alpha: {0}, the previous alpha: {1}", RNNHelper.LearningRate, lastAlpha);
break;
}
lastAlpha = RNNHelper.LearningRate;
//Validate the model by validated corpus
if (ValidationSet != null)
{
Logger.WriteLine("Verify model on validated corpus.");
if (rnn.ValidateNet(ValidationSet, iter) == true)
{
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", m_modelSetting.ModelFile);
rnn.SaveModel(m_modelSetting.ModelFile);
}
}
else if (ppl < lastPPL)
{
//We don't have validate corpus, but we get a better result on training corpus
//We got better result on validated corpus, save this model
Logger.WriteLine("Saving better model into file {0}...", m_modelSetting.ModelFile);
rnn.SaveModel(m_modelSetting.ModelFile);
}
if (ppl >= lastPPL)
{
//We cannot get a better result on training corpus, so reduce learning rate
RNNHelper.LearningRate = RNNHelper.LearningRate / 2.0f;
}
lastPPL = ppl;
iter++;
}
}
