/// <summary>
/// 1D convolution layer (e.g. temporal convolution). This layer creates a convolution kernel that is convolved with the layer input over a single spatial (or temporal) dimension to produce a tensor of outputs. If use_bias is True, a bias vector is created and added to the outputs. Finally, if activation is not None, it is applied to the outputs as well.
/// </summary>
/// <param name="layer">The output of the last layer.</param>
/// <param name="channels">Integer, the dimensionality of the output space</param>
/// <param name="kernalSize">An integer specifying the length of the 1D convolution window.</param>
/// <param name="strides">An integer specifying the stride length of the convolution.</param>
/// <param name="padding">Boolean, if true results in padding the input such that the output has the same length as the original input.</param>
/// <param name="dialation">An integer specifying the dilation rate to use for dilated convolution. Currently, specifying any dilation_rate value != 1 is incompatible with specifying any strides value != 1.</param>
/// <param name="activation">Activation function to use. If you don't specify anything, no activation is applied (ie. "linear" activation: a(x) = x). <see cref="SiaNet.Common.OptActivations"/></param>
/// <param name="useBias">Boolean, whether the layer uses a bias vector.</param>
/// <param name="weightInitializer">Initializer for the kernel weights matrix. <see cref="SiaNet.Common.OptInitializers"/></param>
/// <param name="biasInitializer">Initializer for the bias vector. <see cref="SiaNet.Common.OptInitializers"/></param>
/// <returns></returns>
public static Function Conv1D(Variable layer, int channels, int kernalSize, int strides=1, bool padding=true, int dialation=1, string activation = OptActivations.None, bool useBias = false, string weightInitializer = OptInitializers.Xavier, string biasInitializer = OptInitializers.Zeros)
{
Parameter convParams = null;
Function conv = null;
if (layer.Shape.Rank > 1)
{
int numInputChannels = layer.Shape[layer.Shape.Rank - 1];
convParams = new Parameter(new int[] { kernalSize, numInputChannels, channels }, DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
conv = CNTKLib.Convolution(convParams, layer, new int[] { strides }, new BoolVector(new bool[] { true }), new BoolVector(new bool[] { padding, false, false }), new int[] { dialation });
}
else
{
convParams = new Parameter(new int[] { kernalSize, channels }, DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
conv = CNTKLib.Convolution(convParams, layer, new int[] { strides }, new BoolVector(new bool[] { true }), new BoolVector(new bool[] { padding }), new int[] { dialation });
}
Parameter bias = null;
if (useBias)
{
bias = new Parameter(conv.Output.Shape, DataType.Float, Initializers.Get(biasInitializer), GlobalParameters.Device);
conv = CNTKLib.Plus(bias, conv);
}
return Basic.Activation(conv, activation);
}
/// <summary>
/// 2D convolution layer (e.g. spatial convolution over images). This layer creates a convolution kernel that is convolved with the layer input to produce a tensor of outputs. If use_bias is True, a bias vector is created and added to the outputs. Finally, if activation is not None, it is applied to the outputs as well.
/// </summary>
/// <param name="layer">The output of the last layer.</param>
/// <param name="channels">Integer, the dimensionality of the output space.</param>
/// <param name="kernalSize">A tuple of 2 integers, specifying the width and height of the 2D convolution window. Can be a single integer to specify the same value for all spatial dimensions.</param>
/// <param name="strides">A tuple of 2 integers, specifying the strides of the convolution along the width and height. Can be a single integer to specify the same value for all spatial dimensions. Specifying any stride value != 1 is incompatible with specifying any dilation_rate value != 1.</param>
/// <param name="padding">Boolean, if true results in padding the input such that the output has the same length as the original input.</param>
/// <param name="dialation">A tuple of 2 integers, specifying the dilation rate to use for dilated convolution. Can be a single integer to specify the same value for all spatial dimensions. Currently, specifying any dilation_rate value != 1 is incompatible with specifying any stride value != 1.</param>
/// <param name="activation">Activation function to use. If you don't specify anything, no activation is applied (ie. "linear" activation: a(x) = x). <see cref="SiaNet.Common.OptActivations"/></param>
/// <param name="useBias">Boolean, whether the layer uses a bias vector.</param>
/// <param name="weightInitializer">Initializer for the kernel weights matrix. <see cref="SiaNet.Common.OptInitializers"/></param>
/// <param name="biasInitializer">Initializer for the bias vector. <see cref="SiaNet.Common.OptInitializers"/></param>
/// <returns></returns>
public static Function Conv2D(Variable layer, int channels, Tuple<int, int> kernalSize, Tuple<int, int> strides = null, bool padding = true, Tuple<int, int> dialation = null, string activation = OptActivations.None, bool useBias = false, string weightInitializer = OptInitializers.Xavier, string biasInitializer = OptInitializers.Zeros)
{
int numInputChannels = layer.Shape[layer.Shape.Rank - 1];
var convParams = new Parameter(new int[] { kernalSize.Item1, kernalSize.Item2, numInputChannels, channels }, DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
if (dialation == null)
{
dialation = new Tuple<int, int>(1, 1);
}
int[] stridesParams = null;
if(strides!=null)
{
stridesParams = new int[] { strides.Item1, strides.Item2 };
}
var conv = CNTKLib.Convolution(convParams, layer, stridesParams, new BoolVector(new bool[] { true, true }), new BoolVector(new bool[] { padding, padding, false }), new int[] { dialation.Item1, dialation.Item2 });
Parameter bias = null;
if (useBias)
{
bias = new Parameter(conv.Output.Shape, DataType.Float, Initializers.Get(biasInitializer), GlobalParameters.Device);
conv = CNTKLib.Plus(bias, conv);
}
return Basic.Activation(conv, activation);
}
/// <summary>
/// Builds the RNN.
/// </summary>
/// <param name="inputDim">The input dim.</param>
/// <param name="hiddenSize">Size of the hidden.</param>
/// <param name="numLayers">The number layers.</param>
/// <param name="bidirectional">if set to <c>true</c> [bidirectional].</param>
/// <param name="weightInitializer">The weight initializer.</param>
/// <param name="rnnName">Name of the RNN.</param>
/// <returns></returns>
private static Function BuildRNN(int inputDim, uint hiddenSize, uint numLayers, bool bidirectional = false, string weightInitializer = OptInitializers.Xavier, string rnnName = "")
{
int[] s = { inputDim };
var weights = new Parameter(s, DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
return(CNTKLib.OptimizedRNNStack(Variable.InputVariable(new int[] { inputDim }, DataType.Float), weights, hiddenSize, numLayers, bidirectional, rnnName));
}
/// <summary>
/// Builds the RNN.
/// </summary>
/// <param name="input">The input.</param>
/// <param name="hiddenSize">Size of the hidden.</param>
/// <param name="numLayers">The number layers.</param>
/// <param name="bidirectional">if set to <c>true</c> [bidirectional].</param>
/// <param name="weightInitializer">The weight initializer.</param>
/// <param name="rnnName">Name of the RNN.</param>
/// <returns></returns>
private static Function BuildRNN(Variable input, int dim, uint hiddenSize, uint numLayers, bool bidirectional = false, string weightInitializer = OptInitializers.Xavier, string rnnName = "")
{
int[] s = input.Shape.Dimensions.ToArray();
var weights = new Parameter(s, DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
return CNTKLib.OptimizedRNNStack(Variable.InputVariable(s, DataType.Float), weights, hiddenSize, numLayers, bidirectional, rnnName);
}
/// <summary>
/// Builds the RNN.
/// </summary>
/// <param name="inputDim">The input dim.</param>
/// <param name="hiddenSize">Size of the hidden.</param>
/// <param name="numLayers">The number layers.</param>
/// <param name="bidirectional">if set to <c>true</c> [bidirectional].</param>
/// <param name="weightInitializer">The weight initializer.</param>
/// <param name="rnnName">Name of the RNN.</param>
/// <returns></returns>
private static Function BuildRNN(int[] shape, int dim, uint hiddenSize, uint numLayers, bool bidirectional = false, string weightInitializer = OptInitializers.Xavier, string rnnName = "")
{
List<int> s = new List<int>();
s.AddRange(shape);
s.Add(dim);
var weights = new Parameter(s.ToArray(), DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
return CNTKLib.OptimizedRNNStack(Variable.InputVariable(s.ToArray(), DataType.Float), weights, hiddenSize, numLayers, bidirectional, rnnName);
}
/// <summary>
/// Fully connected layer.
/// </summary>
/// <param name="input">The input.</param>
/// <param name="outputDim">The output dim.</param>
/// <param name="useBias">if set to <c>true</c> [use bias].</param>
/// <param name="weightInitializer">The weight initializer.</param>
/// <param name="biasInitializer">The bias initializer.</param>
/// <returns></returns>
private static Function FullyConnected(Variable input, int outputDim, bool useBias, string weightInitializer, string biasInitializer)
{
int inputDim = input.Shape[0];
int[] s = { outputDim, inputDim };
var weights = new Parameter(s, DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
Parameter bias = null;
if (useBias)
{
int[] s2 = { outputDim };
bias = new Parameter(s2, DataType.Float, Initializers.Get(biasInitializer), GlobalParameters.Device);
}
else
{
int[] s2 = { outputDim };
bias = new Parameter(s2, DataType.Float, 0.0f, GlobalParameters.Device);
}
return(CNTKLib.Plus(bias, CNTKLib.Times(weights, input)));
}
public static Function LSTM(Variable layer, int dim, int?cellDim = null, string activation = OptActivations.Tanh, string recurrentActivation = OptActivations.Sigmoid, string weightInitializer = OptInitializers.GlorotUniform, string recurrentInitializer = OptInitializers.GlorotUniform, bool useBias = true, string biasInitializer = OptInitializers.Zeros, bool returnSequence = false)
{
cellDim = cellDim.HasValue ? cellDim : dim;
Variable prevOutput = Variable.PlaceholderVariable(new int[] { dim }, layer.DynamicAxes);
Variable prevCellState = cellDim.HasValue ? Variable.PlaceholderVariable(new int[] { cellDim.Value }, layer.DynamicAxes) : null;
Func <int, Parameter> createBiasParam = (d) => new Parameter(new int[] { d }, DataType.Float, Initializers.Get(biasInitializer), GlobalParameters.Device);
Func <int, Parameter> createProjectionParam = (oDim) => new Parameter(new int[] { oDim, NDShape.InferredDimension },
DataType.Float, Initializers.Get(weightInitializer), GlobalParameters.Device);
Func <int, Parameter> createDiagWeightParam = (d) =>
new Parameter(new int[] { d }, DataType.Float, Initializers.Get(recurrentInitializer), GlobalParameters.Device);
Function stabilizedPrevOutput = Stabilize <float>(prevOutput, GlobalParameters.Device);
Function stabilizedPrevCellState = prevCellState != null?Stabilize <float>(prevCellState, GlobalParameters.Device) : null;
Func <Variable> projectInput = null;
if (cellDim.HasValue)
{
projectInput = () => createBiasParam(cellDim.Value) + (createProjectionParam(cellDim.Value) * layer);
}
else
{
projectInput = () => layer;
}
//Input gate
Function it = null;
if (cellDim.HasValue)
{
it = Basic.Activation((Variable)(projectInput() + (createProjectionParam(cellDim.Value) * stabilizedPrevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim.Value), stabilizedPrevCellState), recurrentActivation);
}
else
{
it = Basic.Activation((Variable)(projectInput()), recurrentActivation);
}
Function bit = null;
if (cellDim.HasValue)
{
bit = CNTKLib.ElementTimes(it, Basic.Activation(projectInput() + (createProjectionParam(cellDim.Value) * stabilizedPrevOutput), activation));
}
else
{
bit = CNTKLib.ElementTimes(it, Basic.Activation(projectInput(), activation));
}
// Forget-me-not gate
Function ft = null;
if (cellDim.HasValue)
{
ft = Basic.Activation((Variable)(projectInput() + (createProjectionParam(cellDim.Value) * stabilizedPrevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim.Value), stabilizedPrevCellState), recurrentActivation);
}
else
{
ft = Basic.Activation(projectInput(), recurrentActivation);
}
Function bft = prevCellState != null?CNTKLib.ElementTimes(ft, prevCellState) : ft;
Function ct = (Variable)bft + bit;
//Output gate
Function ot = null;
if (cellDim.HasValue)
{
ot = Basic.Activation((Variable)(projectInput() + (createProjectionParam(cellDim.Value) * stabilizedPrevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim.Value), Stabilize <float>(ct, GlobalParameters.Device)), recurrentActivation);
}
else
{
ot = Basic.Activation((Variable)(projectInput()) + Stabilize <float>(ct, GlobalParameters.Device), recurrentActivation);
}
Function ht = CNTKLib.ElementTimes(ot, CNTKLib.Tanh(ct));
Function c = ct;
Function h = (dim != cellDim) ? (createProjectionParam(dim) * Stabilize <float>(ht, GlobalParameters.Device)) : ht;
Func <Variable, Function> recurrenceHookH = (x) => CNTKLib.PastValue(x);
Func <Variable, Function> recurrenceHookC = (x) => CNTKLib.PastValue(x);
var actualDh = recurrenceHookH(h);
var actualDc = recurrenceHookC(c);
if (prevCellState != null)
{
h.ReplacePlaceholders(new Dictionary <Variable, Variable> {
{ prevOutput, actualDh }, { prevCellState, actualDc }
});
}
else
{
h.ReplacePlaceholders(new Dictionary <Variable, Variable> {
{ prevOutput, actualDh }
});
}
if (returnSequence)
{
return(h);
}
return(CNTKLib.SequenceLast(h));
}
/// <summary>
/// Batches the norm.
/// </summary>
/// <param name="shape">The input shape for batch norm layer.</param>
/// <param name="epsilon">Small float added to variance to avoid dividing by zero.</param>
/// <param name="betaInitializer">Initializer for the beta weight.</param>
/// <param name="gammaInitializers">Initializer for the gamma weight.</param>
/// <param name="runningMeanInitializer">Initializer for the running mean weight.</param>
/// <param name="runningStdInvInitializer">Initializer for the running standard inv weight.</param>
/// <param name="spatial">Boolean, if yes the input data is spatial (2D). If not, then sets to 1D</param>
/// <param name="normalizationTimeConstant">The time constant in samples of the first-order low-pass filter that is used to compute mean/variance statistics for use in inference</param>
/// <param name="blendTimeConst">The blend time constant in samples.</param>
/// <returns></returns>
public static Function BatchNorm(int shape, float epsilon = 0.001f, string betaInitializer = OptInitializers.Zeros, string gammaInitializers = OptInitializers.Ones,
string runningMeanInitializer = OptInitializers.Zeros, string runningStdInvInitializer = OptInitializers.Ones, bool spatial = true,
float normalizationTimeConstant = 4096f, float blendTimeConst = 0.0f)
{
var input = CNTKLib.InputVariable(new int[] { shape }, DataType.Float);
var biasParams = new Parameter(new int[] { NDShape.InferredDimension }, DataType.Float, Initializers.Get(betaInitializer), GlobalParameters.Device, "");
var scaleParams = new Parameter(new int[] { NDShape.InferredDimension }, DataType.Float, Initializers.Get(gammaInitializers), GlobalParameters.Device, "");
var runningMean = new Parameter(new int[] { NDShape.InferredDimension }, DataType.Float, Initializers.Get(runningMeanInitializer), GlobalParameters.Device, "");
var runningInvStd = new Constant(new int[] { NDShape.InferredDimension }, 0.0f, GlobalParameters.Device);
var runningCount = Constant.Scalar(0.0f, GlobalParameters.Device);
bool useCudnn = false;
if (GlobalParameters.Device.Type == DeviceKind.GPU)
{
useCudnn = true;
}
return(CNTKLib.BatchNormalization(input, scaleParams, biasParams, runningMean, runningInvStd, runningCount, spatial, normalizationTimeConstant, blendTimeConst, epsilon, useCudnn));
}
List <IConstructor> ICodingStyle.GetInitializers(IType type) => Initializers.Get(type);
/// <summary>
/// Embeddings the specified layer.
/// </summary>
/// <param name="layer">The layer.</param>
/// <param name="embeddingDim">The embedding dim.</param>
/// <param name="initializers">The initializers.</param>
/// <returns></returns>
public static Function Embedding(Variable layer, int embeddingDim, string initializers = OptInitializers.GlorotUniform)
{
var embeddingParameters = new Parameter(new int[] { embeddingDim, layer.Shape[0] }, DataType.Float, Initializers.Get(initializers), GlobalParameters.Device);
return(CNTKLib.Times(embeddingParameters, layer));
}
/// <summary>
/// Embeddings layer
/// </summary>
/// <param name="shape">The shape.</param>
/// <param name="embeddingDim">The dim.</param>
/// <returns></returns>
public static Function Embedding(int shape, int embeddingDim, string initializers = OptInitializers.GlorotUniform)
{
var input = CNTKLib.InputVariable(new int[] { shape }, true, DataType.Float);
var embeddingParameters = new Parameter(new int[] { embeddingDim, shape }, DataType.Float, Initializers.Get(initializers), GlobalParameters.Device);
return(CNTKLib.Times(embeddingParameters, input));
}