protected virtual List <SimpleLayer> CreateLayers(List <LayerConfig> hiddenLayersConfig)
{
var hiddenLayers = new List <SimpleLayer>();
for (var i = 0; i < hiddenLayersConfig.Count; i++)
{
SimpleLayer layer = null;
switch (hiddenLayersConfig[i].LayerType)
{
case LayerType.LSTM:
layer = new LSTMLayer(hiddenLayersConfig[i] as LSTMLayerConfig);
Logger.WriteLine("Create LSTM layer.");
break;
case LayerType.DropOut:
layer = new DropoutLayer(hiddenLayersConfig[i] as DropoutLayerConfig);
Logger.WriteLine("Create Dropout layer.");
break;
}
hiddenLayers.Add(layer);
}
return(hiddenLayers);
}
public static ILayer Load(LayerType layerType, BinaryReader br, bool forTraining = false)
{
ILayer layer = null;
switch (layerType)
{
case LayerType.LSTM:
layer = new LSTMLayer();
break;
case LayerType.DropOut:
layer = new DropoutLayer();
break;
case LayerType.Softmax:
layer = new SoftmaxLayer();
break;
case LayerType.SampledSoftmax:
layer = new SampledSoftmaxLayer();
break;
case LayerType.Simple:
layer = new SimpleLayer();
break;
}
layer.Load(br, layerType, forTraining);
return(layer);
}
public void Test_LSTM_WrongSizeinWidth_Weights()
{
Data2D weights = new Data2D(1, 3, 5, 4);
LSTMLayer rnn = new LSTMLayer(5, 3, TanHLayer.TanHLambda, p => { });
rnn.SetWeights(weights);
}
public void Test_LSTM_Null_Weights()
{
Data2D weights = null;
LSTMLayer rnn = new LSTMLayer(5, 3, TanHLayer.TanHLambda, TanHLayer.TanHLambda);
rnn.SetWeights(weights);
}
public void EnumGradientsTest()
{
Shape shape = new Shape(new int[] { 1, 20, 20, 10 });
LSTMLayer layer = new LSTMLayer(shape, RNNDirection.ForwardOnly, new[] { 20, 30 }, LSTMCell.DefaultForgetBias, MatrixLayout.ColumnMajor, null);
Assert.AreEqual(5, layer.EnumGradients().Count());
}
public void CloneTest()
{
Shape shape = new Shape(new int[] { -1, 20, 20, 10 });
LSTMLayer layer1 = new LSTMLayer(shape, RNNDirection.ForwardOnly, new[] { 2, 3 }, LSTMCell.DefaultForgetBias, MatrixLayout.ColumnMajor, null);
LSTMLayer layer2 = layer1.Clone() as LSTMLayer;
Assert.AreEqual(JsonConvert.SerializeObject(layer1), JsonConvert.SerializeObject(layer2));
}
public void Test_LSTM_DifferentData_Input()
{
DataArray data = new DataArray(5);
Data2D weights = new Data2D(1, 5, 5, 3);
LSTMLayer rnn = new LSTMLayer(5, 3, TanHLayer.TanHLambda, p => { });
rnn.SetWeights(weights);
rnn.SetInput(data);
}
public void Test_LSTM_Null_Input()
{
Data2D data = null;
Data2D weights = new Data2D(1, 5, 5, 3);
LSTMLayer rnn = new LSTMLayer(5, 3, TanHLayer.TanHLambda, TanHLayer.TanHLambda);
rnn.SetWeights(weights);
rnn.SetInput(data);
}
public void SerializeTest()
{
Shape shape = new Shape(new int[] { -1, 20, 20, 10 });
LSTMLayer layer1 = new LSTMLayer(shape, RNNDirection.ForwardOnly, new[] { 2, 3 }, LSTMCell.DefaultForgetBias, MatrixLayout.ColumnMajor, null);
string s1 = JsonConvert.SerializeObject(layer1);
LSTMLayer layer2 = JsonConvert.DeserializeObject <LSTMLayer>(s1);
string s2 = JsonConvert.SerializeObject(layer2);
Assert.AreEqual(s1, s2);
}
public void ConstructorTest1()
{
Shape shape = new Shape(new int[] { 1, 10, 12, 3 });
LSTMLayer layer = new LSTMLayer(shape, RNNDirection.ForwardOnly, new[] { 20, 30 }, LSTMCell.DefaultForgetBias, MatrixLayout.ColumnMajor, null);
Assert.AreEqual(20, ((StochasticLayer)layer.Graph.Vertices.ElementAt(0)).NumberOfNeurons);
Assert.AreEqual(30, ((StochasticLayer)layer.Graph.Vertices.ElementAt(1)).NumberOfNeurons);
Assert.AreEqual("20-30LSTM", layer.Architecture);
Assert.AreEqual(1, layer.NumberOfOutputs);
CollectionAssert.AreEqual(new[] { 1, 30 }, layer.OutputShape.Axes);
}
public void ArchitectureConstructorTest2()
{
const string Architecture = "20-30-40LSTM(Bi=1,ForgetBias=3.6)";
Shape shape = new Shape(new int[] { -1, 20, 20, 10 });
LSTMLayer layer = new LSTMLayer(shape, Architecture, null);
Assert.AreEqual(20, ((LSTMCell)layer.Graph.Vertices.ElementAt(0)).NumberOfNeurons);
Assert.AreEqual(30, ((LSTMCell)layer.Graph.Vertices.ElementAt(1)).NumberOfNeurons);
Assert.AreEqual(40, ((FullyConnectedLayer)layer.Graph.Vertices.ElementAt(2)).NumberOfNeurons);
Assert.AreEqual(Architecture, layer.Architecture);
Assert.IsTrue(layer.Graph.Vertices.Take(2).Cast <LSTMCell>().All(x => x.Direction == RNNDirection.BiDirectional));
Assert.IsTrue(layer.Graph.Vertices.Take(2).Cast <LSTMCell>().All(x => x.ForgetBias == 3.6f));
Assert.AreEqual(1, layer.NumberOfOutputs);
CollectionAssert.AreEqual(new[] { -1, 40 }, layer.OutputShape.Axes);
}
public void ArchitectureConstructorTest3()
{
string architecture = "100LSTM";
try
{
LSTMLayer layer = new LSTMLayer(new Shape(new int[] { 1, 20, 20, 10 }), architecture, null);
}
catch (ArgumentException e)
{
Assert.AreEqual(
new ArgumentException(string.Format(CultureInfo.InvariantCulture, Properties.Resources.E_InvalidLayerArchitecture, architecture), nameof(architecture)).Message,
e.Message);
throw;
}
}
public static SimpleLayer Load(LayerType layerType, BinaryReader br)
{
switch (layerType)
{
case LayerType.LSTM:
return(LSTMLayer.Load(br, LayerType.LSTM));
case LayerType.DropOut:
return(DropoutLayer.Load(br, LayerType.DropOut));
case LayerType.Softmax:
return(SoftmaxLayer.Load(br, LayerType.Softmax));
case LayerType.SampledSoftmax:
return(SampledSoftmaxLayer.Load(br, LayerType.SampledSoftmax));
case LayerType.Simple:
return(SimpleLayer.Load(br, LayerType.Simple));
}
return(null);
}
public Brain(MyCaffeControl <T> mycaffe, PropertySet properties, CryptoRandom random, IxTrainerCallbackRNN icallback, Phase phase, BucketCollection rgVocabulary, bool bUsePreloadData, string strRunProperties = null)
{
string strOutputBlob = null;
if (strRunProperties != null)
{
m_runProperties = new PropertySet(strRunProperties);
}
m_icallback = icallback;
m_mycaffe = mycaffe;
m_properties = properties;
m_random = random;
m_rgVocabulary = rgVocabulary;
m_bUsePreloadData = bUsePreloadData;
m_nSolverSequenceLength = m_properties.GetPropertyAsInt("SequenceLength", -1);
m_bDisableVocabulary = m_properties.GetPropertyAsBool("DisableVocabulary", false);
m_nThreads = m_properties.GetPropertyAsInt("Threads", 1);
m_dfScale = m_properties.GetPropertyAsDouble("Scale", 1.0);
if (m_nThreads > 1)
{
m_dataPool.Initialize(m_nThreads, icallback);
}
if (m_runProperties != null)
{
m_dfTemperature = Math.Abs(m_runProperties.GetPropertyAsDouble("Temperature", 0));
if (m_dfTemperature > 1.0)
{
m_dfTemperature = 1.0;
}
string strPhaseOnRun = m_runProperties.GetProperty("PhaseOnRun", false);
switch (strPhaseOnRun)
{
case "RUN":
m_phaseOnRun = Phase.RUN;
break;
case "TEST":
m_phaseOnRun = Phase.TEST;
break;
case "TRAIN":
m_phaseOnRun = Phase.TRAIN;
break;
}
if (phase == Phase.RUN && m_phaseOnRun != Phase.NONE)
{
if (m_phaseOnRun != Phase.RUN)
{
m_mycaffe.Log.WriteLine("Warning: Running on the '" + m_phaseOnRun.ToString() + "' network.");
}
strOutputBlob = m_runProperties.GetProperty("OutputBlob", false);
if (strOutputBlob == null)
{
throw new Exception("You must specify the 'OutputBlob' when Running with a phase other than RUN.");
}
strOutputBlob = Utility.Replace(strOutputBlob, '~', ';');
phase = m_phaseOnRun;
}
}
m_net = mycaffe.GetInternalNet(phase);
if (m_net == null)
{
mycaffe.Log.WriteLine("WARNING: Test net does not exist, set test_iteration > 0. Using TRAIN phase instead.");
m_net = mycaffe.GetInternalNet(Phase.TRAIN);
}
// Find the first LSTM layer to determine how to load the data.
// NOTE: Only LSTM has a special loading order, other layers use the standard N, C, H, W ordering.
LSTMLayer <T> lstmLayer = null;
LSTMSimpleLayer <T> lstmSimpleLayer = null;
foreach (Layer <T> layer1 in m_net.layers)
{
if (layer1.layer_param.type == LayerParameter.LayerType.LSTM)
{
lstmLayer = layer1 as LSTMLayer <T>;
m_lstmType = LayerParameter.LayerType.LSTM;
break;
}
else if (layer1.layer_param.type == LayerParameter.LayerType.LSTM_SIMPLE)
{
lstmSimpleLayer = layer1 as LSTMSimpleLayer <T>;
m_lstmType = LayerParameter.LayerType.LSTM_SIMPLE;
break;
}
}
if (lstmLayer == null && lstmSimpleLayer == null)
{
throw new Exception("Could not find the required LSTM or LSTM_SIMPLE layer!");
}
if (m_phaseOnRun != Phase.NONE && m_phaseOnRun != Phase.RUN && strOutputBlob != null)
{
if ((m_blobOutput = m_net.FindBlob(strOutputBlob)) == null)
{
throw new Exception("Could not find the 'Output' layer top named '" + strOutputBlob + "'!");
}
}
if ((m_blobData = m_net.FindBlob("data")) == null)
{
throw new Exception("Could not find the 'Input' layer top named 'data'!");
}
if ((m_blobClip = m_net.FindBlob("clip")) == null)
{
throw new Exception("Could not find the 'Input' layer top named 'clip'!");
}
Layer <T> layer = m_net.FindLastLayer(LayerParameter.LayerType.INNERPRODUCT);
m_mycaffe.Log.CHECK(layer != null, "Could not find an ending INNERPRODUCT layer!");
if (!m_bDisableVocabulary)
{
m_nVocabSize = (int)layer.layer_param.inner_product_param.num_output;
if (rgVocabulary != null)
{
m_mycaffe.Log.CHECK_EQ(m_nVocabSize, rgVocabulary.Count, "The vocabulary count = '" + rgVocabulary.Count.ToString() + "' and last inner product output count = '" + m_nVocabSize.ToString() + "' - these do not match but they should!");
}
}
if (m_lstmType == LayerParameter.LayerType.LSTM)
{
m_nSequenceLength = m_blobData.shape(0);
m_nBatchSize = m_blobData.shape(1);
}
else
{
m_nBatchSize = (int)lstmSimpleLayer.layer_param.lstm_simple_param.batch_size;
m_nSequenceLength = m_blobData.shape(0) / m_nBatchSize;
if (phase == Phase.RUN)
{
m_nBatchSize = 1;
List <int> rgNewShape = new List <int>()
{
m_nSequenceLength, 1
};
m_blobData.Reshape(rgNewShape);
m_blobClip.Reshape(rgNewShape);
m_net.Reshape();
}
}
m_mycaffe.Log.CHECK_EQ(m_nSequenceLength, m_blobData.num, "The data num must equal the sequence lengh of " + m_nSequenceLength.ToString());
m_rgDataInput = new T[m_nSequenceLength * m_nBatchSize];
T[] rgClipInput = new T[m_nSequenceLength * m_nBatchSize];
m_mycaffe.Log.CHECK_EQ(rgClipInput.Length, m_blobClip.count(), "The clip count must equal the sequence length * batch size: " + rgClipInput.Length.ToString());
m_tZero = (T)Convert.ChangeType(0, typeof(T));
m_tOne = (T)Convert.ChangeType(1, typeof(T));
for (int i = 0; i < rgClipInput.Length; i++)
{
if (m_lstmType == LayerParameter.LayerType.LSTM)
{
rgClipInput[i] = (i < m_nBatchSize) ? m_tZero : m_tOne;
}
else
{
rgClipInput[i] = (i % m_nSequenceLength == 0) ? m_tZero : m_tOne;
}
}
m_blobClip.mutable_cpu_data = rgClipInput;
if (phase != Phase.RUN)
{
m_solver = mycaffe.GetInternalSolver();
m_solver.OnStart += m_solver_OnStart;
m_solver.OnTestStart += m_solver_OnTestStart;
m_solver.OnTestingIteration += m_solver_OnTestingIteration;
m_solver.OnTrainingIteration += m_solver_OnTrainingIteration;
if ((m_blobLabel = m_net.FindBlob("label")) == null)
{
throw new Exception("Could not find the 'Input' layer top named 'label'!");
}
m_nSequenceLengthLabel = m_blobLabel.count(0, 2);
m_rgLabelInput = new T[m_nSequenceLengthLabel];
m_mycaffe.Log.CHECK_EQ(m_rgLabelInput.Length, m_blobLabel.count(), "The label count must equal the label sequence length * batch size: " + m_rgLabelInput.Length.ToString());
m_mycaffe.Log.CHECK(m_nSequenceLengthLabel == m_nSequenceLength * m_nBatchSize || m_nSequenceLengthLabel == 1, "The label sqeuence length must be 1 or equal the length of the sequence: " + m_nSequenceLength.ToString());
}
}
public void Test_LSTM_Execute_Linear()
{
// Initialize data.
Data2D data = new Data2D(1, 3, 3, 5);
int l = 0;
for (int b = 0; b < 5; ++b)
{
for (int w = 0; w < 3; ++w)
{
for (int c = 0; c < 3; ++c)
{
l += 1;
data[0, w, c, b] = (l % 5 + 1) / 10.0;
}
}
}
// Initialize parameters.
Data2D pms = new Data2D(2, 3, 2, 12);
int k = 0;
int bc = 0;
for (int i = 0; i < 3; ++i)
{
for (int u = 0; u < 8; ++u)
{
k += 1;
pms[u % 2, i, 0, bc] = (k % 5 - 2) / 10.0;
if (k % 2 == 0)
{
bc += 1;
bc = bc % 4;
}
}
}
k = 0;
bc = 0;
for (int i = 0; i < 2; ++i)
{
for (int u = 0; u < 8; ++u)
{
k += 1;
pms[u % 2, i, 0, 4 + bc] = (k % 5 - 2) / 10.0;
if (k % 2 == 0)
{
bc += 1;
bc = bc % 4;
}
}
}
pms[0, 0, 0, 8] = 1.0 / 10.0;
pms[0, 0, 1, 8] = 2.0 / 10.0;
pms[0, 0, 0, 9] = -1.0 / 10.0;
pms[0, 0, 1, 9] = 0.0 / 10;
pms[0, 0, 0, 10] = 3.0 / 10;
pms[0, 0, 1, 10] = 4.0 / 10;
pms[0, 0, 0, 11] = 5.0 / 10;
pms[0, 0, 1, 11] = -2.0 / 10;
LSTMLayer rnn = new LSTMLayer(2, 3, p => { }, p => { });
rnn.SetWeights(pms);
rnn.SetInput(data);
rnn.Execute();
Data2D output = rnn.GetOutput() as Data2D;
// Checking sizes
Dimension dim = output.GetDimension();
Assert.AreEqual(dim.b, 5);
Assert.AreEqual(dim.c, 2);
Assert.AreEqual(dim.h, 1);
Assert.AreEqual(dim.w, 1);
// Checking calculation
Assert.AreEqual(output[0, 0, 0, 0], 0.01582, 0.00001);
Assert.AreEqual(output[0, 0, 1, 0], -0.00801, 0.00001);
Assert.AreEqual(output[0, 0, 0, 1], 0.01568, 0.00001);
Assert.AreEqual(output[0, 0, 1, 1], -0.00986, 0.00001);
Assert.AreEqual(output[0, 0, 0, 2], 0.01580, 0.00001);
Assert.AreEqual(output[0, 0, 1, 2], -0.011669, 0.00001);
Assert.AreEqual(output[0, 0, 0, 3], 0.00591, 0.00001);
Assert.AreEqual(output[0, 0, 1, 3], -0.009268, 0.00001);
Assert.AreEqual(output[0, 0, 0, 4], 0.01530, 0.00001);
Assert.AreEqual(output[0, 0, 1, 4], -0.010527, 0.00001);
}
public Brain(MyCaffeControl <T> mycaffe, PropertySet properties, CryptoRandom random, IxTrainerCallbackRNN icallback, Phase phase, BucketCollection rgVocabulary, string strRunProperties = null)
{
string strOutputBlob = null;
if (strRunProperties != null)
{
m_runProperties = new PropertySet(strRunProperties);
}
m_icallback = icallback;
m_mycaffe = mycaffe;
m_properties = properties;
m_random = random;
m_rgVocabulary = rgVocabulary;
if (m_runProperties != null)
{
m_dfTemperature = m_runProperties.GetPropertyAsDouble("Temperature", 0);
string strPhaseOnRun = m_runProperties.GetProperty("PhaseOnRun", false);
switch (strPhaseOnRun)
{
case "RUN":
m_phaseOnRun = Phase.RUN;
break;
case "TEST":
m_phaseOnRun = Phase.TEST;
break;
case "TRAIN":
m_phaseOnRun = Phase.TRAIN;
break;
}
if (phase == Phase.RUN && m_phaseOnRun != Phase.NONE)
{
if (m_phaseOnRun != Phase.RUN)
{
m_mycaffe.Log.WriteLine("Warning: Running on the '" + m_phaseOnRun.ToString() + "' network.");
}
strOutputBlob = m_runProperties.GetProperty("OutputBlob", false);
if (strOutputBlob == null)
{
throw new Exception("You must specify the 'OutputBlob' when Running with a phase other than RUN.");
}
strOutputBlob = Utility.Replace(strOutputBlob, '~', ';');
phase = m_phaseOnRun;
}
}
m_net = mycaffe.GetInternalNet(phase);
// Find the first LSTM layer to determine how to load the data.
// NOTE: Only LSTM has a special loading order, other layers use the standard N, C, H, W ordering.
LSTMLayer <T> lstmLayer = null;
LSTMSimpleLayer <T> lstmSimpleLayer = null;
foreach (Layer <T> layer1 in m_net.layers)
{
if (layer1.layer_param.type == LayerParameter.LayerType.LSTM)
{
lstmLayer = layer1 as LSTMLayer <T>;
m_lstmType = LayerParameter.LayerType.LSTM;
break;
}
else if (layer1.layer_param.type == LayerParameter.LayerType.LSTM_SIMPLE)
{
lstmSimpleLayer = layer1 as LSTMSimpleLayer <T>;
m_lstmType = LayerParameter.LayerType.LSTM_SIMPLE;
break;
}
}
if (lstmLayer == null && lstmSimpleLayer == null)
{
throw new Exception("Could not find the required LSTM or LSTM_SIMPLE layer!");
}
if (m_phaseOnRun != Phase.NONE && m_phaseOnRun != Phase.RUN && strOutputBlob != null)
{
if ((m_blobOutput = m_net.FindBlob(strOutputBlob)) == null)
{
throw new Exception("Could not find the 'Output' layer top named '" + strOutputBlob + "'!");
}
}
if ((m_blobData = m_net.FindBlob("data")) == null)
{
throw new Exception("Could not find the 'Input' layer top named 'data'!");
}
if ((m_blobClip = m_net.FindBlob("clip")) == null)
{
throw new Exception("Could not find the 'Input' layer top named 'clip'!");
}
Layer <T> layer = m_net.FindLastLayer(LayerParameter.LayerType.INNERPRODUCT);
m_mycaffe.Log.CHECK(layer != null, "Could not find an ending INNERPRODUCT layer!");
m_nVocabSize = (int)layer.layer_param.inner_product_param.num_output;
if (rgVocabulary != null)
{
m_mycaffe.Log.CHECK_EQ(m_nVocabSize, rgVocabulary.Count, "The vocabulary count and last inner product output count should match!");
}
if (m_lstmType == LayerParameter.LayerType.LSTM)
{
m_nSequenceLength = m_blobData.shape(0);
m_nBatchSize = m_blobData.shape(1);
}
else
{
m_nBatchSize = (int)lstmSimpleLayer.layer_param.lstm_simple_param.batch_size;
m_nSequenceLength = m_blobData.shape(0) / m_nBatchSize;
if (phase == Phase.RUN)
{
m_nBatchSize = 1;
List <int> rgNewShape = new List <int>()
{
m_nSequenceLength, 1
};
m_blobData.Reshape(rgNewShape);
m_blobClip.Reshape(rgNewShape);
m_net.Reshape();
}
}
m_mycaffe.Log.CHECK_EQ(m_blobData.count(), m_blobClip.count(), "The data and clip blobs must have the same count!");
m_rgDataInput = new T[m_nSequenceLength * m_nBatchSize];
T[] rgClipInput = new T[m_nSequenceLength * m_nBatchSize];
m_tZero = (T)Convert.ChangeType(0, typeof(T));
m_tOne = (T)Convert.ChangeType(1, typeof(T));
for (int i = 0; i < rgClipInput.Length; i++)
{
if (m_lstmType == LayerParameter.LayerType.LSTM)
{
rgClipInput[i] = (i < m_nBatchSize) ? m_tZero : m_tOne;
}
else
{
rgClipInput[i] = (i % m_nSequenceLength == 0) ? m_tZero : m_tOne;
}
}
m_blobClip.mutable_cpu_data = rgClipInput;
if (phase != Phase.RUN)
{
m_solver = mycaffe.GetInternalSolver();
m_solver.OnStart += m_solver_OnStart;
m_solver.OnTestStart += m_solver_OnTestStart;
m_solver.OnTestingIteration += m_solver_OnTestingIteration;
m_solver.OnTrainingIteration += m_solver_OnTrainingIteration;
if ((m_blobLabel = m_net.FindBlob("label")) == null)
{
throw new Exception("Could not find the 'Input' layer top named 'label'!");
}
m_rgLabelInput = new T[m_nSequenceLength * m_nBatchSize];
m_mycaffe.Log.CHECK_EQ(m_blobData.count(), m_blobLabel.count(), "The data and label blobs must have the same count!");
}
}
