public override void LoadModel(string filename, bool bTrain = false)
{
Logger.WriteLine("Loading SimpleRNN model: {0}", filename);
var sr = new StreamReader(filename);
var br = new BinaryReader(sr.BaseStream);
IsCRFTraining = br.ReadBoolean();
//Create cells of each layer
var layerSize = br.ReadInt32();
LayerType layerType = LayerType.None;
HiddenLayerList = new List <SimpleLayer>();
for (var i = 0; i < layerSize; i++)
{
layerType = (LayerType)br.ReadInt32();
HiddenLayerList.Add(Load(layerType, br));
SimpleLayer layer = HiddenLayerList[HiddenLayerList.Count - 1];
if (bTrain)
{
layer.SetRunningMode(RunningMode.Training);
layer.InitializeInternalTrainingParameters();
}
else
{
layer.SetRunningMode(RunningMode.Test);
}
}
Logger.WriteLine("Create output layer");
layerType = (LayerType)br.ReadInt32();
OutputLayer = Load(layerType, br);
if (bTrain)
{
OutputLayer.SetRunningMode(RunningMode.Training);
OutputLayer.InitializeInternalTrainingParameters();
}
else
{
OutputLayer.SetRunningMode(RunningMode.Test);
}
if (IsCRFTraining)
{
Logger.WriteLine("Loading CRF tag trans weights...");
CRFTagTransWeights = RNNHelper.LoadMatrix(br);
crfLocker = new object();
}
sr.Close();
}
/// <summary>
/// Create network for forward sequence-to-sequence
/// Sparse feature size = source sparse feature size + target sparse feature size
/// Dense feature size: For first layer, (source dense feature size + target dense), for other layers, (source dense feature size + previous hidden layer size)
/// </summary>
/// <param name="hiddenLayersConfig"></param>
/// <param name="outputLayerConfig"></param>
/// <param name="TrainingSet"></param>
/// <param name="featurizer"></param>
public override void CreateNetwork(List <LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet <T> TrainingSet, Config featurizer)
{
var srcDenseFeatureSize = featurizer.Seq2SeqAutoEncoder.GetTopHiddenLayerSize() * 2;
var sparseFeatureSize = TrainingSet.SparseFeatureSize;
sparseFeatureSize += featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize;
Logger.WriteLine("Sparse Feature Format: [{0}][{1}] = {2}", TrainingSet.SparseFeatureSize, featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize, sparseFeatureSize);
HiddenLayerList = CreateLayers(hiddenLayersConfig);
for (var i = 0; i < HiddenLayerList.Count; i++)
{
SimpleLayer layer = HiddenLayerList[i];
layer.InitializeWeights(sparseFeatureSize, i == 0 ? (srcDenseFeatureSize + TrainingSet.DenseFeatureSize) : (srcDenseFeatureSize + HiddenLayerList[i - 1].LayerSize));
layer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create hidden layer {i}: size = {layer.LayerSize}, sparse feature size = {layer.SparseFeatureSize}, dense feature size = {layer.DenseFeatureSize}");
}
outputLayerConfig.LayerSize = TrainingSet.TagSize;
OutputLayer = CreateOutputLayer(outputLayerConfig, 0, (srcDenseFeatureSize + HiddenLayerList[HiddenLayerList.Count - 1].LayerSize));
OutputLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create a Forward recurrent neural sequence-to-sequence network with {HiddenLayerList.Count} hidden layers");
}
public override void CreateNetwork(List <LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet <T> TrainingSet, Config featurizer)
{
var forwardHiddenLayers = CreateLayers(hiddenLayersConfig);
var backwardHiddenLayers = CreateLayers(hiddenLayersConfig);
for (var i = 0; i < hiddenLayersConfig.Count; i++)
{
SimpleLayer forwardLayer = forwardHiddenLayers[i];
SimpleLayer backwardLayer = backwardHiddenLayers[i];
var denseFeatureSize = TrainingSet.DenseFeatureSize;
if (i > 0)
{
denseFeatureSize = forwardHiddenLayers[i - 1].LayerSize * 2;
}
forwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, denseFeatureSize);
backwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, denseFeatureSize);
forwardLayer.SetRunningMode(RunningMode.Training);
backwardLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create hidden layer {i}: size = {forwardLayer.LayerSize}, sparse feature size = {forwardLayer.SparseFeatureSize}, dense feature size = {forwardLayer.DenseFeatureSize}");
}
outputLayerConfig.LayerSize = TrainingSet.TagSize;
SimpleLayer outputLayer = CreateOutputLayer(outputLayerConfig, TrainingSet.SparseFeatureSize, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize * 2);
outputLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create a bi-directional recurrent neural network with {forwardHiddenLayers.Count} hidden layers. Forward and backward layers are concatnated.");
InitCache(forwardHiddenLayers, backwardHiddenLayers, outputLayer);
}
public override void CreateNetwork(List <LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet <T> TrainingSet, Config featurizer)
{
HiddenLayerList = CreateLayers(hiddenLayersConfig);
for (var i = 0; i < HiddenLayerList.Count; i++)
{
SimpleLayer layer = HiddenLayerList[i];
layer.InitializeWeights(TrainingSet.SparseFeatureSize, i == 0 ? TrainingSet.DenseFeatureSize : HiddenLayerList[i - 1].LayerSize);
layer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create hidden layer {i}: size = {layer.LayerSize}, sparse feature size = {layer.SparseFeatureSize}, dense feature size = {layer.DenseFeatureSize}");
}
outputLayerConfig.LayerSize = TrainingSet.TagSize;
OutputLayer = CreateOutputLayer(outputLayerConfig, TrainingSet.SparseFeatureSize, HiddenLayerList[HiddenLayerList.Count - 1].LayerSize);
OutputLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create a Forward recurrent neural network with {HiddenLayerList.Count} hidden layers");
}
public override void LoadModel(string filename, bool bTrain = false)
{
Logger.WriteLine(Logger.Level.info, "Loading bi-directional model: {0}", filename);
using (var sr = new StreamReader(filename))
{
var br = new BinaryReader(sr.BaseStream);
IsCRFTraining = br.ReadBoolean();
var layerSize = br.ReadInt32();
LayerType layerType = LayerType.None;
//Load forward layers from file
forwardHiddenLayers = new List <SimpleLayer>();
for (var i = 0; i < layerSize; i++)
{
layerType = (LayerType)br.ReadInt32();
forwardHiddenLayers.Add(Load(layerType, br));
SimpleLayer layer = forwardHiddenLayers[forwardHiddenLayers.Count - 1];
if (bTrain)
{
layer.SetRunningMode(RunningMode.Training);
layer.InitializeInternalTrainingParameters();
}
else
{
layer.SetRunningMode(RunningMode.Test);
}
}
//Load backward layers from file
backwardHiddenLayers = new List <SimpleLayer>();
for (var i = 0; i < layerSize; i++)
{
layerType = (LayerType)br.ReadInt32();
backwardHiddenLayers.Add(Load(layerType, br));
SimpleLayer layer = backwardHiddenLayers[backwardHiddenLayers.Count - 1];
if (bTrain)
{
layer.SetRunningMode(RunningMode.Training);
layer.InitializeInternalTrainingParameters();
}
else
{
layer.SetRunningMode(RunningMode.Test);
}
}
Logger.WriteLine("Create output layer");
layerType = (LayerType)br.ReadInt32();
OutputLayer = Load(layerType, br);
if (bTrain)
{
OutputLayer.SetRunningMode(RunningMode.Training);
OutputLayer.InitializeInternalTrainingParameters();
}
else
{
OutputLayer.SetRunningMode(RunningMode.Test);
}
if (IsCRFTraining)
{
Logger.WriteLine("Loading CRF tag trans weights...");
CRFWeights = RNNHelper.LoadMatrix(br);
}
if (bTrain)
{
InitCache(forwardHiddenLayers, backwardHiddenLayers, OutputLayer.CreateLayerSharedWegiths());
}
}
}
