public void LSTM_Test_Params_Count()
{
//define values, and variables
Variable x = Variable.InputVariable(new int[] { 2 }, DataType.Float, "input");
Variable y = Variable.InputVariable(new int[] { 3 }, DataType.Float, "output");
//Number of LSTM parameters
var lstm1 = RNN.RecurrenceLSTM(x, 3, 3, DataType.Float, device, false, Activation.TanH, false, false, 1);
var ft = lstm1.Inputs.Where(l => l.Uid.StartsWith("Parameter")).ToList();
var consts = lstm1.Inputs.Where(l => l.Uid.StartsWith("Constant")).ToList();
var inp = lstm1.Inputs.Where(l => l.Uid.StartsWith("Input")).ToList();
//bias params
var bs = ft.Where(p => p.Name.Contains("_b")).ToList();
var totalBs = bs.Sum(v => v.Shape.TotalSize);
Assert.Equal(12, totalBs);
//weights
var ws = ft.Where(p => p.Name.Contains("_w")).ToList();
var totalWs = ws.Sum(v => v.Shape.TotalSize);
Assert.Equal(24, totalWs);
//update
var us = ft.Where(p => p.Name.Contains("_u")).ToList();
var totalUs = us.Sum(v => v.Shape.TotalSize);
Assert.Equal(36, totalUs);
//total number opf parameters
var totalOnly = totalBs + totalWs + totalUs;
Assert.Equal(72, totalOnly);
}
// Use this for initialization
void Start()
{
num_neurons = 10;
double[] p_list = new double[120] {
7.513874, -13.31366, -5.891386, 3.89313, -13.35374, -5.470473, -28.48326, -0.8950226, 1.676227, -7.829721, 3.001554, 0.9742008, 6.85763, 12.07218, -1.542749, 6.42987, -6.777609, -12.28891, -11.6297, -3.275043, -0.3349386, 13.02803, -3.171752, 2.909704, 7.935295, 2.154472, 5.825492, 6.763169, 6.5986, 0.846593, 6.619426, 0.1207947, 7.378351, 5.546535, -0.8831579, -3.48157, 2.024852, -5.732147, 16.51345, -11.30504, -4.979899, 0.1936812, -7.330712, -2.363033, -5.555974, -11.48479, 6.027504, -0.5046204, -5.604547, 6.99612, 8.643599, -0.5479781, 1.765478, 12.20771, -14.86998, -10.28358, 1.827907, -1.541205, 8.282341, 14.34195, 0.6747534, 8.42286, -10.29843, -0.5448059, 6.860302, -7.558879, -3.571216, -3.13385, 12.60011, -0.07997914, -4.452103, 1.369113, 7.464754, -8.164039, -4.246214, -7.685563, -1.948951, 15.26882, -5.696559, -1.464601, 1.735656, 13.39631, 12.12768, -10.33641, 1.26995, 6.143419, -3.200521, -6.426282, -4.033798, -9.858385, 3.240334, 5.088798, -8.307716, 4.577082, 9.247719, -4.710557, 11.59537, 1.740587, 1.173246, 5.654369, -0.01115066, 3.742572, 0.01587177, 5.836065, -1.146728, -0.4543813, -0.09580191, -3.896233, -1.911579, 4.912868, -0.4526708, 3.798782, 4.286755, 2.614983, 3.773993, -1.182102, 3.293242, 3.547439, 4.012696, 0.9868906,
};
float[] param_list = new float[120];
for (int i = 0; i < 120; i++)
{
param_list [i] = (float)p_list [i];
}
run = 1;
spring_const = 1000;
damper = 100;
//System.IO.File.Delete("C:/UnityLogs/best_costs.txt");
//Assign joint references
hip_right = upper_right.GetComponent <ConfigurableJoint> ();
knee_right = lower_right.GetComponent <HingeJoint> ();
hip_left = upper_left.GetComponent <ConfigurableJoint> ();
knee_left = lower_left.GetComponent <HingeJoint> ();
gen_alg = new GenAlg();
//net = new RNN (gen_alg.GetCurrentChrom());
net = new RNN(new Chromosome(param_list));
Debug.Log("weights: " + net.weights.ToString());
counter = 0;
//SetAngles (new float[]{0f, 0f, 0.5f, 0.5f, 0.5f, 0.5f});
//SetAngles (new float[]{0f, 0f, 0f, 0f, 0f, 0f});
}
public void lstm_test01()
{
//define values, and variables
Variable x = Variable.InputVariable(new int[] { 4 }, DataType.Float, "input");
var xValues = Value.CreateBatchOfSequences <float>(new int[] { 4 }, mData, device);
//
var lstm00 = RNN.RecurrenceLSTM(x, 3, 3, DataType.Float, device, false, Activation.TanH, true, true, 1);
//
LSTMReccurentNN lstmNN = new LSTMReccurentNN(1, 1, device);
//lstm implme reference 01
var lstmCell = lstmNN.CreateLSTM(x, "output1");
var lstm01 = CNTKLib.SequenceLast(lstmCell.h);
//lstme implementation refe 02
var lstm02 = LSTMSequenceClassifier.LSTMNet(x, 1, device, "output1");
//
var wParams00 = lstm00.Inputs.Where(p => p.Uid.Contains("Parameter")).ToList();
var wParams01 = lstm00.Inputs.Where(p => p.Uid.Contains("Parameter")).ToList();
var wParams02 = lstm00.Inputs.Where(p => p.Uid.Contains("Parameter")).ToList();
//parameter count
Assert.Equal(wParams00.Count, wParams01.Count);
Assert.Equal(wParams00.Count, wParams02.Count);
//structure of parameters test
Assert.Equal(wParams00.Where(p => p.Name.Contains("_b")).Count(), wParams01.Where(p => p.Name.Contains("_b")).Count());
Assert.Equal(wParams00.Where(p => p.Name.Contains("_w")).Count(), wParams01.Where(p => p.Name.Contains("_w")).Count());
Assert.Equal(wParams00.Where(p => p.Name.Contains("_u")).Count(), wParams01.Where(p => p.Name.Contains("_u")).Count());
Assert.Equal(wParams00.Where(p => p.Name.Contains("peep")).Count(), wParams01.Where(p => p.Name.Contains("peep")).Count());
Assert.Equal(wParams00.Where(p => p.Name.Contains("stabilize")).Count(), wParams01.Where(p => p.Name.Contains("stabilize")).Count());
//check structure of parameters with originaly developed lstm
//chech for arguments
Assert.True(lstm01.Arguments.Count == lstm02.Arguments.Count);
for (int i = 0; i < lstm01.Arguments.Count; i++)
{
testVariable(lstm01.Arguments[i], lstm01.Arguments[i]);
}
///
Assert.True(lstm01.Inputs.Count == lstm02.Inputs.Count);
for (int i = 0; i < lstm01.Inputs.Count; i++)
{
testVariable(lstm01.Inputs[i], lstm02.Inputs[i]);
}
///
Assert.True(lstm01.Outputs.Count == lstm02.Outputs.Count);
for (int i = 0; i < lstm01.Outputs.Count; i++)
{
testVariable(lstm01.Outputs[i], lstm02.Outputs[i]);
}
}
static RnnTrainResult Train(DataSet data, int layer1NodeCount, int layer2NodeCount, int epochMax)
{
var numInputs = data.Input.RowCount;
var layers = new List <LayerSpec> {
new LayerSpec(layer1NodeCount, true, ActivationType.LogisticSigmoid),
new LayerSpec(layer2NodeCount, true, ActivationType.LogisticSigmoid),
new LayerSpec(1, false, ActivationType.Linear)
};
var weightCount = RNN.GetWeightCount(layers, numInputs);
var initialWeights = QuqeUtil.MakeRandomVector(weightCount, -1, 1);
return(RNN.TrainSCG(layers, initialWeights, epochMax, data.Input, data.Output));
}
/// <summary>
/// Implementation of custom NN model
/// </summary>
/// <param name="data"></param>
/// <param name="yearVar"></param>
/// <param name="montVar"></param>
/// <param name="shopVar"></param>
/// <param name="itemVar"></param>
/// <param name="cnt3Var"></param>
/// <param name="label"></param>
/// <param name="device"></param>
/// <returns></returns>
private static Function PredictFutureSalesModel(List <Variable> variables, DeviceDescriptor device)
{
//define features and label vars
Variable yearVar = variables[0];
Variable montVar = variables[1];
Variable shopVar = variables[2];
Variable itemVar = variables[3];
Variable cnt3Var = variables[4];
Variable label = variables[5];
//create rnn object
var ffNet = new FeedForwaredNN(device);
//predefined parameters
var H_DIMS = 11;
var CELL_DIMS = 3;
var DROPRATRE = 0.2f;
var outDim = label.Shape.Dimensions.Last();
//embedding layer and dimensionality reduction
var yearEmb = Embedding.Create(yearVar, yearVar.Shape.Dimensions[0] - 1, DataType.Float, device, 1, yearVar.Name + "_emb");
var monthEmb = Embedding.Create(montVar, montVar.Shape.Dimensions[0] / 2, DataType.Float, device, 1, montVar.Name + "_emb");
var varshopEmb = Embedding.Create(shopVar, shopVar.Shape.Dimensions[0] / 2, DataType.Float, device, 1, shopVar.Name + "_emb");
var itemEmb = Embedding.Create(itemVar, itemVar.Shape.Dimensions[0] / 2, DataType.Float, device, 1, itemVar.Name + "_emb");
var itemEmb2 = Embedding.Create(itemEmb, itemEmb.Output.Shape.Dimensions[0] / 4, DataType.Float, device, 1, itemEmb.Name + "_emb");
//join all embedding layers with input variable of previous product sales
var emb = CNTKLib.Splice(new VariableVector()
{
yearEmb, monthEmb, varshopEmb, itemEmb2, cnt3Var
}, new Axis(0));
//create recurrence for time series on top of joined layer
var lstmLayer = RNN.RecurrenceLSTM(emb, H_DIMS, CELL_DIMS, DataType.Float, device, false, Activation.TanH, true, true);
//create dense on top of LSTM recurrence layers
var denseLayer = ffNet.Dense(lstmLayer, 33, Activation.TanH);
//create dropout layer on top of dense layer
var dropoutLay = CNTKLib.Dropout(denseLayer, DROPRATRE);
//create dense layer without activation function
var outLayer = ffNet.Dense(dropoutLay, outDim, Activation.None, label.Name);
//
return(outLayer);
}
public void CreateNewPopulation(uint nPopulation)
{
for (int i = 0; i < nPopulation; i++)
{
// Sequential model = new Sequential();
// model.Add(new Layer(8, 16, Activations.Tanh));
// model.Add(new Layer(16, 8, Activations.Tanh));
// model.Add(new Layer(8, 4, Activations.Tanh));
// model.Add(new Layer(4, 2, Activations.Tanh));
// model.Mutate(2f);
// Individual individual = new Individual(model);
// individual.Mutate(.5f);
// Population.Add(individual);
RNN model = new RNN(8, 3, new uint[] { 16, 8, 4, 2 }, Activations.Tanh);
Individual individual = new Individual(model);
Population.Add(individual);
}
}
/// <summary>
/// Create cntk model function by providing parameters. The method is able for create:
/// - feedforward with one hidden layer and any number of neurons
/// - deep neural network with any number of hidden layers and any number of neurons. Each hidden number has the same number of neurons
/// - LSTM NN with any number of hidden layers of LSTM , and any number of LSTM Cells in each layer. Also at the top of the network you can define
/// one dense layer and one dropout layer.
/// </summary>
/// <param name="nnParams"></param>
/// <returns></returns>
public static Function CreateNetwrok(List <NNLayer> layers, List <Variable> inputVars, Variable outpuVar, DeviceDescriptor device)
{
DataType type = DataType.Float;
Variable inputLayer = null;
if (inputVars.Count > 1)
{
var vv = new VariableVector();
foreach (var v in inputVars)
{
//check if variable is stores as Sparse then we should create one embedding layer before slice
//since mixing sparse and dense data is not supported
if (v.IsSparse)
{
var v1 = Embedding.Create(v, v.Shape.Dimensions.Last(), type, device, 1, v.Name + "_sp_emb");
vv.Add(v1);
}
else
{
vv.Add(v);
}
}
//
inputLayer = (Variable)CNTKLib.Splice(vv, new Axis(0));
}
else //define input layer
{
inputLayer = inputVars.First();
}
//Create network
var net = inputLayer;
var ff = new FeedForwaredNN(device, type);
//set last layer name to label name
layers.Last().Name = outpuVar.Name;
//get last LSTM layer
var lastLSTM = layers.Where(x => x.Type == LayerType.LSTM).LastOrDefault();
//
foreach (var layer in layers)
{
if (layer.Type == LayerType.Dense)
{
net = ff.Dense(net, layer.HDimension, layer.Activation, layer.Name);
}
else if (layer.Type == LayerType.Drop)
{
net = CNTKLib.Dropout(net, layer.Value / 100.0f);
}
else if (layer.Type == LayerType.Embedding)
{
net = Embedding.Create(net, layer.HDimension, type, device, 1, layer.Name);
}
else if (layer.Type == LayerType.LSTM)
{
var returnSequence = true;
if (layers.IndexOf(lastLSTM) == layers.IndexOf(layer))
{
returnSequence = false;
}
net = RNN.RecurrenceLSTM(net, layer.HDimension, layer.CDimension, type, device, returnSequence, layer.Activation,
layer.Peephole, layer.SelfStabilization, 1);
}
}
//check if last layer is compatible with the output
if (net.Shape.Dimensions.Last() != outpuVar.Shape.Dimensions.Last())
{
ff.CreateOutputLayer(net, outpuVar, Activation.None);
}
return(net);
}
public void CopyWeights(INNBase source)
{
RNN src = (RNN)source;
h_layers.CopyWeights(src.h_layers);
}
public void LSTM_Test_Params_Count_with_peep_selfstabilize()
{
//define values, and variables
Variable x = Variable.InputVariable(new int[] { 2 }, DataType.Float, "input");
Variable y = Variable.InputVariable(new int[] { 3 }, DataType.Float, "output");
#region lstm org implemented in cntk for reference
//lstme implementation refe 02
var lstmTest02 = LSTMSequenceClassifier.LSTMNet(x, 3, device, "output1");
var ft2 = lstmTest02.Inputs.Where(l => l.Uid.StartsWith("Parameter")).ToList();
var totalSize = ft2.Sum(p => p.Shape.TotalSize);
//bias params
var bs2 = ft2.Where(p => p.Name.Contains("_b")).ToList();
var totalBs2 = bs2.Sum(v => v.Shape.TotalSize);
//weights
var ws2 = ft2.Where(p => p.Name.Contains("_w")).ToList();
var totalWs2 = ws2.Sum(v => v.Shape.TotalSize);
//update
var us2 = ft2.Where(p => p.Name.Contains("_u")).ToList();
var totalUs2 = us2.Sum(v => v.Shape.TotalSize);
//peephole
var ph2 = ft2.Where(p => p.Name.Contains("_peep")).ToList();
var totalph2 = ph2.Sum(v => v.Shape.TotalSize);
//stabilize
var st2 = ft2.Where(p => p.Name.Contains("_stabilize")).ToList();
var totalst2 = st2.Sum(v => v.Shape.TotalSize);
#endregion
#region anndotnet old implementation
//
//LSTMReccurentNN lstmNN = new LSTMReccurentNN(3, 3, device);
////lstm implme reference 01
//var lstmCell11 = lstmNN.CreateLSTM(x, "output1");
//var lstmTest01 = CNTKLib.SequenceLast(lstmCell11.h);
//var ft1 = lstmTest01.Inputs.Where(l => l.Uid.StartsWith("Parameter")).ToList();
//var consts1 = lstmTest01.Inputs.Where(l => l.Uid.StartsWith("Constant")).ToList();
//var inp1 = lstmTest01.Inputs.Where(l => l.Uid.StartsWith("Input")).ToList();
//var pparams1 = ft1.Sum(v => v.Shape.TotalSize);
#endregion
//Number of LSTM parameters
var lstm1 = RNN.RecurrenceLSTM(x, 3, 3, DataType.Float, device, false, Activation.TanH, true, true, 1);
var ft = lstm1.Inputs.Where(l => l.Uid.StartsWith("Parameter")).ToList();
var consts = lstm1.Inputs.Where(l => l.Uid.StartsWith("Constant")).ToList();
var inp = lstm1.Inputs.Where(l => l.Uid.StartsWith("Input")).ToList();
//bias params
var bs = ft.Where(p => p.Name.Contains("_b")).ToList();
var totalBs = bs.Sum(v => v.Shape.TotalSize);
Assert.Equal(12, totalBs);
//weights
var ws = ft.Where(p => p.Name.Contains("_w")).ToList();
var totalWs = ws.Sum(v => v.Shape.TotalSize);
Assert.Equal(24, totalWs);
//update
var us = ft.Where(p => p.Name.Contains("_u")).ToList();
var totalUs = us.Sum(v => v.Shape.TotalSize);
Assert.Equal(36, totalUs);
//peephole
var ph = ft.Where(p => p.Name.Contains("_peep")).ToList();
var totalPh = ph.Sum(v => v.Shape.TotalSize);
Assert.Equal(9, totalPh);
//stabilize
var st = ft.Where(p => p.Name.Contains("_stabilize")).ToList();
var totalst = st.Sum(v => v.Shape.TotalSize);
Assert.Equal(6, totalst);
var totalOnly = totalBs + totalWs + totalUs;
var totalWithSTabilize = totalOnly + totalst;
var totalWithPeep = totalOnly + totalPh;
var totalP = totalOnly + totalst + totalPh;
var totalParams = ft.Sum(v => v.Shape.TotalSize);
Assert.Equal(totalP, totalParams);
//72- without peep and stab
//75 - witout peep with stabil +3xm =
//81 - with peephole and without stabil
//87 - with peep ans stab 3+9
}
