/// <summary>
/// Create network for forward sequence-to-sequence
/// Sparse feature size = source sparse feature size + target sparse feature size
/// Dense feature size: For first layer, (source dense feature size + target dense), for other layers, (source dense feature size + previous hidden layer size)
/// </summary>
/// <param name="hiddenLayersConfig"></param>
/// <param name="outputLayerConfig"></param>
/// <param name="TrainingSet"></param>
/// <param name="featurizer"></param>
public override void CreateNetwork(List <LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet <T> TrainingSet, Config featurizer)
{
var srcDenseFeatureSize = featurizer.Seq2SeqAutoEncoder.GetTopHiddenLayerSize() * 2;
var sparseFeatureSize = TrainingSet.SparseFeatureSize;
sparseFeatureSize += featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize;
Logger.WriteLine("Sparse Feature Format: [{0}][{1}] = {2}", TrainingSet.SparseFeatureSize, featurizer.Seq2SeqAutoEncoder.Config.SparseFeatureSize, sparseFeatureSize);
HiddenLayerList = CreateLayers(hiddenLayersConfig);
for (var i = 0; i < HiddenLayerList.Count; i++)
{
SimpleLayer layer = HiddenLayerList[i];
layer.InitializeWeights(sparseFeatureSize, i == 0 ? (srcDenseFeatureSize + TrainingSet.DenseFeatureSize) : (srcDenseFeatureSize + HiddenLayerList[i - 1].LayerSize));
layer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create hidden layer {i}: size = {layer.LayerSize}, sparse feature size = {layer.SparseFeatureSize}, dense feature size = {layer.DenseFeatureSize}");
}
outputLayerConfig.LayerSize = TrainingSet.TagSize;
OutputLayer = CreateOutputLayer(outputLayerConfig, 0, (srcDenseFeatureSize + HiddenLayerList[HiddenLayerList.Count - 1].LayerSize));
OutputLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create a Forward recurrent neural sequence-to-sequence network with {HiddenLayerList.Count} hidden layers");
}
public override void CreateNetwork(List <LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet <T> TrainingSet, Config featurizer)
{
var forwardHiddenLayers = CreateLayers(hiddenLayersConfig);
var backwardHiddenLayers = CreateLayers(hiddenLayersConfig);
for (var i = 0; i < hiddenLayersConfig.Count; i++)
{
SimpleLayer forwardLayer = forwardHiddenLayers[i];
SimpleLayer backwardLayer = backwardHiddenLayers[i];
var denseFeatureSize = TrainingSet.DenseFeatureSize;
if (i > 0)
{
denseFeatureSize = forwardHiddenLayers[i - 1].LayerSize * 2;
}
forwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, denseFeatureSize);
backwardLayer.InitializeWeights(TrainingSet.SparseFeatureSize, denseFeatureSize);
forwardLayer.SetRunningMode(RunningMode.Training);
backwardLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create hidden layer {i}: size = {forwardLayer.LayerSize}, sparse feature size = {forwardLayer.SparseFeatureSize}, dense feature size = {forwardLayer.DenseFeatureSize}");
}
outputLayerConfig.LayerSize = TrainingSet.TagSize;
SimpleLayer outputLayer = CreateOutputLayer(outputLayerConfig, TrainingSet.SparseFeatureSize, forwardHiddenLayers[forwardHiddenLayers.Count - 1].LayerSize * 2);
outputLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create a bi-directional recurrent neural network with {forwardHiddenLayers.Count} hidden layers. Forward and backward layers are concatnated.");
InitCache(forwardHiddenLayers, backwardHiddenLayers, outputLayer);
}
protected SimpleLayer CreateOutputLayer(LayerConfig outputLayerConfig, int sparseFeatureSize, int denseFeatureSize)
{
SimpleLayer outputLayer = null;
switch (outputLayerConfig.LayerType)
{
case LayerType.SampledSoftmax:
Logger.WriteLine("Create sampled softmax layer as output layer");
outputLayer = new SampledSoftmaxLayer(outputLayerConfig as SampledSoftmaxLayerConfig);
outputLayer.InitializeWeights(0, denseFeatureSize);
break;
case LayerType.Softmax:
Logger.WriteLine("Create softmax layer as output layer.");
outputLayer = new SoftmaxLayer(outputLayerConfig as SoftmaxLayerConfig);
outputLayer.InitializeWeights(sparseFeatureSize, denseFeatureSize);
break;
case LayerType.Simple:
Logger.WriteLine("Create simple layer as output layer.");
outputLayer = new SimpleLayer(outputLayerConfig as SimpleLayerConfig);
outputLayer.InitializeWeights(sparseFeatureSize, denseFeatureSize);
break;
}
outputLayer.LabelShortList = new List <int>();
return(outputLayer);
}
public override void CreateNetwork(List <LayerConfig> hiddenLayersConfig, LayerConfig outputLayerConfig, DataSet <T> TrainingSet, Config featurizer)
{
HiddenLayerList = CreateLayers(hiddenLayersConfig);
for (var i = 0; i < HiddenLayerList.Count; i++)
{
SimpleLayer layer = HiddenLayerList[i];
layer.InitializeWeights(TrainingSet.SparseFeatureSize, i == 0 ? TrainingSet.DenseFeatureSize : HiddenLayerList[i - 1].LayerSize);
layer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create hidden layer {i}: size = {layer.LayerSize}, sparse feature size = {layer.SparseFeatureSize}, dense feature size = {layer.DenseFeatureSize}");
}
outputLayerConfig.LayerSize = TrainingSet.TagSize;
OutputLayer = CreateOutputLayer(outputLayerConfig, TrainingSet.SparseFeatureSize, HiddenLayerList[HiddenLayerList.Count - 1].LayerSize);
OutputLayer.SetRunningMode(RunningMode.Training);
Logger.WriteLine($"Create a Forward recurrent neural network with {HiddenLayerList.Count} hidden layers");
}