private static Function createModel(Variable x, int outDim, int nnType, DeviceDescriptor device)
{
//scale data before first layer
var scaledInput = CNTKLib.ElementTimes(Constant.Scalar <float>(0.00391f, device), x);
if (nnType == 0 /*useNALU*/)
{
FeedForwaredNN ffn = new FeedForwaredNN(device, DataType.Float);
Function dense1 = ffn.Dense(scaledInput, 200, Activation.Sigmoid, "");
var nalu = new NALU(dense1, outDim, DataType.Float, device, 1);
return(nalu.H);
}
else if (nnType == 1 /*FeedForward*/)
{
FeedForwaredNN ffn = new FeedForwaredNN(device, DataType.Float);
Function dense1 = ffn.Dense(scaledInput, 200, Activation.Sigmoid, "");
Function classifierOutput = ffn.Dense(dense1, outDim, Activation.None, "MNIST Classifier");
return(classifierOutput);
}
else if (nnType == 2 /*Convolution*/)
{
return(null);
}
else
{
return(null);
}
}
/// <summary>
/// Implementation of custom NN model
/// </summary>
/// <param name="data"></param>
/// <param name="yearVar"></param>
/// <param name="montVar"></param>
/// <param name="shopVar"></param>
/// <param name="itemVar"></param>
/// <param name="cnt3Var"></param>
/// <param name="label"></param>
/// <param name="device"></param>
/// <returns></returns>
private static Function PredictFutureSalesModel(List <Variable> variables, DeviceDescriptor device)
{
//define features and label vars
Variable yearVar = variables[0];
Variable montVar = variables[1];
Variable shopVar = variables[2];
Variable itemVar = variables[3];
Variable cnt3Var = variables[4];
Variable label = variables[5];
//create rnn object
var ffNet = new FeedForwaredNN(device);
//predefined parameters
var H_DIMS = 11;
var CELL_DIMS = 3;
var DROPRATRE = 0.2f;
var outDim = label.Shape.Dimensions.Last();
//embedding layer and dimensionality reduction
var yearEmb = Embedding.Create(yearVar, yearVar.Shape.Dimensions[0] - 1, DataType.Float, device, 1, yearVar.Name + "_emb");
var monthEmb = Embedding.Create(montVar, montVar.Shape.Dimensions[0] / 2, DataType.Float, device, 1, montVar.Name + "_emb");
var varshopEmb = Embedding.Create(shopVar, shopVar.Shape.Dimensions[0] / 2, DataType.Float, device, 1, shopVar.Name + "_emb");
var itemEmb = Embedding.Create(itemVar, itemVar.Shape.Dimensions[0] / 2, DataType.Float, device, 1, itemVar.Name + "_emb");
var itemEmb2 = Embedding.Create(itemEmb, itemEmb.Output.Shape.Dimensions[0] / 4, DataType.Float, device, 1, itemEmb.Name + "_emb");
//join all embedding layers with input variable of previous product sales
var emb = CNTKLib.Splice(new VariableVector()
{
yearEmb, monthEmb, varshopEmb, itemEmb2, cnt3Var
}, new Axis(0));
//create recurrence for time series on top of joined layer
var lstmLayer = RNN.RecurrenceLSTM(emb, H_DIMS, CELL_DIMS, DataType.Float, device, false, Activation.TanH, true, true);
//create dense on top of LSTM recurrence layers
var denseLayer = ffNet.Dense(lstmLayer, 33, Activation.TanH);
//create dropout layer on top of dense layer
var dropoutLay = CNTKLib.Dropout(denseLayer, DROPRATRE);
//create dense layer without activation function
var outLayer = ffNet.Dense(dropoutLay, outDim, Activation.None, label.Name);
//
return(outLayer);
}
private static void cntkModelToGraphviz1()
{
var net = new FeedForwaredNN(DeviceDescriptor.UseDefaultDevice(), DataType.Float);
//define input and output variable and connecting to the stream configuration
var feature = Variable.InputVariable(new NDShape(1, 4), DataType.Float, "features");
var label = Variable.InputVariable(new NDShape(1, 3), DataType.Float, "flower");
//firs hidden layer
var model = net.Dense(feature, 5, Activation.ReLU, "hidden");
model = net.Dense(model, 3, Activation.Softmax, "flower");
NetToGraph fg = new NetToGraph();
var dot = fg.ToGraph(model);
// Save it to a file
File.WriteAllText("myFile.dot", dot);
}
/// <summary>
/// Create cntk model function by providing parameters. The method is able for create:
/// - feedforward with one hidden layer and any number of neurons
/// - deep neural network with any number of hidden layers and any number of neurons. Each hidden number has the same number of neurons
/// - LSTM NN with any number of hidden layers of LSTM , and any number of LSTM Cells in each layer. Also at the top of the network you can define
/// one dense layer and one dropout layer.
/// </summary>
/// <param name="nnParams"></param>
/// <returns></returns>
public static Function CreateNetwrok(List <NNLayer> layers, List <Variable> inputVars, Variable outpuVar, DeviceDescriptor device)
{
DataType type = DataType.Float;
Variable inputLayer = null;
if (inputVars.Count > 1)
{
var vv = new VariableVector();
foreach (var v in inputVars)
{
//check if variable is stores as Sparse then we should create one embedding layer before slice
//since mixing sparse and dense data is not supported
if (v.IsSparse)
{
var v1 = Embedding.Create(v, v.Shape.Dimensions.Last(), type, device, 1, v.Name + "_sp_emb");
vv.Add(v1);
}
else
{
vv.Add(v);
}
}
//
inputLayer = (Variable)CNTKLib.Splice(vv, new Axis(0));
}
else //define input layer
{
inputLayer = inputVars.First();
}
//Create network
var net = inputLayer;
var ff = new FeedForwaredNN(device, type);
//set last layer name to label name
layers.Last().Name = outpuVar.Name;
//get last LSTM layer
var lastLSTM = layers.Where(x => x.Type == LayerType.LSTM).LastOrDefault();
//
foreach (var layer in layers)
{
if (layer.Type == LayerType.Dense)
{
net = ff.Dense(net, layer.HDimension, layer.Activation, layer.Name);
}
else if (layer.Type == LayerType.Drop)
{
net = CNTKLib.Dropout(net, layer.Value / 100.0f);
}
else if (layer.Type == LayerType.Embedding)
{
net = Embedding.Create(net, layer.HDimension, type, device, 1, layer.Name);
}
else if (layer.Type == LayerType.LSTM)
{
var returnSequence = true;
if (layers.IndexOf(lastLSTM) == layers.IndexOf(layer))
{
returnSequence = false;
}
net = RNN.RecurrenceLSTM(net, layer.HDimension, layer.CDimension, type, device, returnSequence, layer.Activation,
layer.Peephole, layer.SelfStabilization, 1);
}
}
//check if last layer is compatible with the output
if (net.Shape.Dimensions.Last() != outpuVar.Shape.Dimensions.Last())
{
ff.CreateOutputLayer(net, outpuVar, Activation.None);
}
return(net);
}
