public static LSTMLayer Load(BinaryReader br, LayerType layerType)
{
LSTMLayerConfig config = new LSTMLayerConfig();
config.LayerSize = br.ReadInt32();
config.LayerType = layerType;
LSTMLayer layer = new LSTMLayer(config);
layer.SparseFeatureSize = br.ReadInt32();
layer.DenseFeatureSize = br.ReadInt32();
//Create cells of each layer
layer.InitializeCellWeights(br);
//Load weight matrix between each two layer pairs
//weight input->hidden
if (layer.SparseFeatureSize > 0)
{
Logger.WriteLine("Loading sparse feature weights...");
layer.sparseFeatureWeights = LoadLSTMWeights(br);
}
if (layer.DenseFeatureSize > 0)
{
//weight fea->hidden
Logger.WriteLine("Loading dense feature weights...");
layer.wDenseInputGate = LoadLSTMGateWeights(br);
layer.wDenseCellGate = LoadLSTMGateWeights(br);
layer.wDenseForgetGate = LoadLSTMGateWeights(br);
layer.wDenseOutputGate = LoadLSTMGateWeights(br);
}
return(layer);
}
private void SetHiddenLayers()
{
//Get hidden layer settings
//Example: LSTM:200, Dropout:0.5
HiddenLayersConfig = new List <LayerConfig>();
var hiddenLayers = config.GetValueRequired(HIDDEN_LAYER);
foreach (var layer in hiddenLayers.Split(','))
{
var items = layer.Split(':');
var sLayerType = items[0].Trim();
var layerType = LayerType.None;
foreach (
var type in
Enum.GetValues(typeof(LayerType))
.Cast <LayerType>()
.Where(
type => sLayerType.Equals(type.ToString(), StringComparison.InvariantCultureIgnoreCase))
)
{
layerType = type;
break;
}
LayerConfig baseLayerConfig;
switch (layerType)
{
case LayerType.LSTM:
{
var layerConfig = new LSTMLayerConfig();
layerConfig.LayerSize = int.Parse(items[1]);
layerConfig.LayerType = layerType;
baseLayerConfig = layerConfig;
Logger.WriteLine($"Initialize configuration for LSTM layer. Layer size = {layerConfig.LayerSize}");
}
break;
case LayerType.DropOut:
{
var layerConfig = new DropoutLayerConfig {
DropoutRatio = float.Parse(items[1])
};
layerConfig.LayerSize = (NetworkType == NETWORKTYPE.BiDirectional) ? HiddenLayersConfig[HiddenLayersConfig.Count - 1].LayerSize * 2 : HiddenLayersConfig[HiddenLayersConfig.Count - 1].LayerSize;
layerConfig.LayerType = layerType;
baseLayerConfig = layerConfig;
Logger.WriteLine(
$"Initialize configuration for Dropout layer. Dropout ratio = '{layerConfig.DropoutRatio}'");
}
break;
default:
throw new ArgumentException($"Invalidated layer type: {sLayerType}");
}
HiddenLayersConfig.Add(baseLayerConfig);
}
Logger.WriteLine($"Hidden layer : {HiddenLayersConfig.Count}");
Logger.WriteLine($"Hidden layer : {hiddenLayers}");
}
public LSTMLayer(LSTMLayerConfig config) : base(config)
{
this.config = config;
LSTMCells = new LSTMCell[LayerSize];
for (var i = 0; i < LayerSize; i++)
{
LSTMCells[i] = new LSTMCell();
}
}
private void SetHiddenLayers()
{
//Get hidden layer settings
HiddenLayersConfig = new List <LayerConfig>();
var hiddenLayers = config.GetValueRequired(HIDDEN_LAYER);
foreach (var layer in hiddenLayers.Split(','))
{
var items = layer.Split(':');
var sLayerSize = items[0].Trim();
var sLayerType = items[1].Trim();
//Parse layer size and type
var layerSize = int.Parse(sLayerSize);
var layerType = LayerType.None;
foreach (
var type in
Enum.GetValues(typeof(LayerType))
.Cast <LayerType>()
.Where(
type => sLayerType.Equals(type.ToString(), StringComparison.InvariantCultureIgnoreCase))
)
{
layerType = type;
break;
}
LayerConfig baseLayerConfig;
switch (layerType)
{
case LayerType.LSTM:
{
var layerConfig = new LSTMLayerConfig();
baseLayerConfig = layerConfig;
Logger.WriteLine("Initialize configuration for LSTM layer.");
}
break;
case LayerType.BPTT:
{
if (items.Length != 3)
{
throw new ArgumentException(
$"Invalidated settings for BPTT: {layer}, it should be [size:BPTT:bptt_value], such as [200:BPTT:5] .");
}
var layerConfig = new BPTTLayerConfig {
Bptt = int.Parse(items[2])
};
baseLayerConfig = layerConfig;
Logger.WriteLine($"Initialize configuration for BPTT layer. BPTT = '{layerConfig.Bptt}'");
}
break;
case LayerType.DropOut:
{
var layerConfig = new DropoutLayerConfig {
DropoutRatio = float.Parse(items[2])
};
baseLayerConfig = layerConfig;
Logger.WriteLine(
$"Initialize configuration for Dropout layer. Dropout ratio = '{layerConfig.DropoutRatio}'");
}
break;
default:
throw new ArgumentException($"Invalidated layer type: {sLayerType}");
}
baseLayerConfig.LayerType = layerType;
baseLayerConfig.LayerSize = layerSize;
HiddenLayersConfig.Add(baseLayerConfig);
}
Logger.WriteLine($"Hidden layer : {HiddenLayersConfig.Count}");
Logger.WriteLine($"Hidden layer : {hiddenLayers}");
}
public LSTMLayer(LSTMLayerConfig config) : base(config)
{
AllocateMemoryForLSTMCells();
}
