/// <summary>
/// Initializes a new instance of the <see cref="LSTMCell"/> class.
/// </summary>
/// <param name="shape">The shape of the layer's input tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the layer.</param>
public LSTMCell(
Shape shape,
RNNDirection direction,
int numberOfNeurons)
: this(shape, direction, numberOfNeurons, LSTMCell.DefaultForgetBias, MatrixLayout.ColumnMajor, null)
{
}
/// <summary>
/// Initializes the <see cref="StochasticLayer"/>.
/// </summary>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the layer.</param>
/// <param name="matrixLayout">Specifies whether the weight matrices are row-major or column-major.</param>
/// <param name="weightsShape">The dimensions of the layer's weights tensor.</param>
/// <param name="hiddenShape">The dimensions of the layer's hidden weights tensor.</param>
/// <param name="biasesShape">The dimensions of the layer's biases tensor.</param>
/// <param name="random">The random numbers generator.</param>
private protected void Initialize(
RNNDirection direction,
int numberOfNeurons,
MatrixLayout matrixLayout,
int[] weightsShape,
int[] hiddenShape,
int[] biasesShape,
RandomNumberGenerator <float> random)
{
if (hiddenShape == null)
{
throw new ArgumentNullException(nameof(hiddenShape));
}
if (random == null)
{
random = new RandomRangeGenerator(-0.08f, 0.08f);
}
if (direction == RNNDirection.BiDirectional && (numberOfNeurons % 2) != 0)
{
throw new ArgumentException("The number of neurons in a bi-directional RNN must be even.", nameof(numberOfNeurons));
}
this.Direction = direction;
this.Initialize(numberOfNeurons, matrixLayout, weightsShape, biasesShape, random);
this.U = new Tensor("hidden weights", hiddenShape);
this.U.Randomize(random);
}
/// <summary>
/// Initializes the <see cref="SRNCell"/>.
/// </summary>
/// <param name="shape">The shape of the layer's input tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the layer.</param>
/// <param name="matrixLayout">Specifies whether the weight matrices are row-major or column-major.</param>
/// <param name="random">The random numbers generator.</param>
private void Initialize(
Shape shape,
RNNDirection direction,
int numberOfNeurons,
MatrixLayout matrixLayout,
RandomNumberGenerator <float> random)
{
if (shape == null)
{
throw new ArgumentNullException(nameof(shape));
}
int[] axes = shape.Axes;
// column-major matrix organization - each row contains all weights for one neuron
// row-major matrix organization - each column contains all weights for one neuron
int xlen = axes.Skip(1).Aggregate(1, (total, next) => total * next);
int[] weightsShape = matrixLayout == MatrixLayout.ColumnMajor ?
new[] { xlen, numberOfNeurons } :
new[] { numberOfNeurons, xlen };
int hlen = direction == RNNDirection.ForwardOnly ? numberOfNeurons : numberOfNeurons / 2;
int[] hiddenShape = matrixLayout == MatrixLayout.ColumnMajor ?
new[] { hlen, numberOfNeurons } :
new[] { numberOfNeurons, hlen };
int[] biasesShape = new[] { numberOfNeurons };
this.Initialize(direction, numberOfNeurons, matrixLayout, weightsShape, hiddenShape, biasesShape, random);
this.OutputShape = new Shape(new int[] { shape.GetAxis(0), numberOfNeurons });
}
/// <summary>
/// Initializes a new instance of the <see cref="GRUCell"/> class.
/// </summary>
/// <param name="shape">The shape of the layer's input tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the layer.</param>
/// <param name="matrixLayout">Specifies whether the weight matrices are row-major or column-major.</param>
/// <param name="random">The random numbers generator.</param>
public GRUCell(
Shape shape,
RNNDirection direction,
int numberOfNeurons,
MatrixLayout matrixLayout,
RandomNumberGenerator <float> random)
{
this.Initialize(shape, direction, numberOfNeurons, matrixLayout, random);
}
/// <summary>
/// Initializes a new instance of the <see cref="SRNLayer"/> class.
/// </summary>
/// <param name="shape">The shape of the layer's input tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the hidden and fully connected layers.</param>
/// <param name="matrixLayout">Specifies whether the weight matrices are row-major or column-major.</param>
/// <param name="random">The random numbers generator.</param>
public SRNLayer(
Shape shape,
RNNDirection direction,
IList <int> numberOfNeurons,
MatrixLayout matrixLayout,
RandomNumberGenerator <float> random)
{
this.Initialize(shape, direction, numberOfNeurons, matrixLayout, random);
}
public static Tensor GRU(
this Session session,
Tensor x,
Tensor w,
Tensor u,
Tensor b,
RNNDirection direction,
int numberOfNeurons,
MatrixLayout matrixLayout)
{
const string ActionName = "gru";
// calculate gates = W * x + b
Tensor g = session.FullyConnected(x, w, b, matrixLayout);
int tt = g.Shape.GetAxis(0); // number of vectors in time sequence
return(session.RunOperation(
ActionName,
() =>
{
bool calculateGradient = session.CalculateGradients;
Tensor h = session.AllocateTensor(ActionName, new Shape(new int[] { tt, numberOfNeurons }), calculateGradient);
NativeMethods.gru(
tt,
numberOfNeurons,
u.Weights,
g.Weights,
h.Weights,
direction == RNNDirection.BiDirectional,
matrixLayout == MatrixLayout.RowMajor);
if (calculateGradient)
{
session.Push(
ActionName,
() =>
{
NativeMethods.gru_gradient(
tt,
numberOfNeurons,
u.Weights,
u.Gradient,
g.Weights,
g.Gradient,
h.Weights,
h.Gradient,
direction == RNNDirection.BiDirectional,
matrixLayout == MatrixLayout.RowMajor);
});
}
return h;
}));
}
/// <summary>
/// Initializes the <see cref="GRUCell"/>.
/// </summary>
/// <param name="shape">The dimensions of the layer's input tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the layer.</param>
/// <param name="matrixLayout">Specifies whether the weight matrices are row-major or column-major.</param>
/// <param name="random">The random numbers generator.</param>
private void Initialize(
Shape shape,
RNNDirection direction,
int numberOfNeurons,
MatrixLayout matrixLayout,
RandomNumberGenerator <float> random)
{
if (shape == null)
{
throw new ArgumentNullException(nameof(shape));
}
if (random == null)
{
random = new RandomRangeGenerator(-0.08f, 0.08f);
}
int[] axes = shape.Axes;
// column-major matrix organization - each row contains all weights for one neuron
// row-major matrix organization - each column contains all weights for one neuron
int xlen = axes.Skip(1).Aggregate(1, (total, next) => total * next);
int[] weightsShape = matrixLayout == MatrixLayout.ColumnMajor ?
new[] { xlen, 3 * numberOfNeurons } :
new[] { 3 * numberOfNeurons, xlen };
// keep all weights in single channel
// allocate three matrices (one for each of two gates and one for the state)
int[] biasesShape = new[] { 3 * numberOfNeurons };
int hlen = direction == RNNDirection.ForwardOnly ? numberOfNeurons : numberOfNeurons / 2;
int[] hiddenShape = matrixLayout == MatrixLayout.ColumnMajor ?
new[] { hlen, 3 * numberOfNeurons } :
new[] { 3 * numberOfNeurons, hlen };
this.Initialize(
direction,
numberOfNeurons,
matrixLayout,
weightsShape,
hiddenShape,
biasesShape,
random ?? new RandomRangeGenerator(-0.08f, 0.08f));
this.OutputShape = new Shape(new int[] { shape.GetAxis(0), numberOfNeurons });
// initialize biases for update and reset gates only
Vectors.Set(2 * numberOfNeurons, 1.0f, this.B.Weights, 0);
}
/// <summary>
/// Initializes the <see cref="LSTMCell"/>.
/// </summary>
/// <param name="shape">The dimensions of the layer's input tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the layer.</param>
/// <param name="matrixLayout">Specifies whether the weight matrices are row-major or column-major.</param>
/// <param name="forgetBias">The bias added to forget gates.</param>
/// <param name="random">The random numbers generator.</param>
private void Initialize(
Shape shape,
RNNDirection direction,
int numberOfNeurons,
MatrixLayout matrixLayout,
float forgetBias,
RandomNumberGenerator <float> random)
{
if (shape == null)
{
throw new ArgumentNullException(nameof(shape));
}
int[] axes = shape.Axes;
// column-major matrix organization - each row contains all weights for one neuron
// row-major matrix organization - each column contains all weights for one neuron
int mbsize = axes.Skip(1).Aggregate(1, (total, next) => total * next);
int[] weightsShape = matrixLayout == MatrixLayout.ColumnMajor ?
new[] { mbsize, 4 * numberOfNeurons } :
new[] { 4 * numberOfNeurons, mbsize };
int[] hiddenShape = matrixLayout == MatrixLayout.ColumnMajor ?
new[] { numberOfNeurons, 4 * numberOfNeurons } :
new[] { 4 * numberOfNeurons, numberOfNeurons };
// keep all weights in single channel
// allocate four matrices (one for each of three gates and one for the state)
int[] biasesShape = new[] { 4 * numberOfNeurons };
this.Initialize(
direction,
numberOfNeurons,
matrixLayout,
weightsShape,
hiddenShape,
biasesShape,
random ?? new RandomRangeGenerator(-0.08f, 0.08f));
this.ForgetBias = forgetBias;
this.OutputShape = new Shape(new int[] { shape.GetAxis(0), numberOfNeurons });
}
/// <summary>
/// Initializes the <see cref="GRULayer"/>.
/// </summary>
/// <param name="shape">The shape of the layer's input tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
/// <param name="numberOfNeurons">The number of neurons in the hidden and fully connected layers.</param>
/// <param name="forgetBias">The bias added to forget gates.</param>
/// <param name="matrixLayout">Specifies whether the weight matrices are row-major or column-major.</param>
/// <param name="random">The random numbers generator.</param>
private void Initialize(
Shape shape,
RNNDirection direction,
IList <int> numberOfNeurons,
float forgetBias,
MatrixLayout matrixLayout,
RandomNumberGenerator <float> random)
{
if (shape == null)
{
throw new ArgumentNullException(nameof(shape));
}
if (numberOfNeurons == null)
{
throw new ArgumentNullException(nameof(numberOfNeurons));
}
if (numberOfNeurons.Count < 2)
{
throw new ArgumentException("Recurrent neural network must have at least two layers.");
}
// create layers
List <Layer> layers = new List <Layer>(numberOfNeurons.Count);
for (int i = 0, ii = numberOfNeurons.Count; i < ii; i++)
{
Layer layer = i + 1 < ii ?
new LSTMCell(shape, direction, numberOfNeurons[i], forgetBias, matrixLayout, random) as Layer :
new FullyConnectedLayer(shape, numberOfNeurons[i], matrixLayout, random) as Layer;
layers.Add(layer);
shape = layer.OutputShape;
}
// build LSTM graph
this.Graph.AddEdges(layers);
// output shape is the output shape of the decoder
this.OutputShape = shape;
}
/// <summary>
/// Initializes a new instance of the <see cref="RNNCell"/> class.
/// </summary>
/// <param name="outputShape">The shape of the layer's output tensor.</param>
/// <param name="direction">The cell direction (forward-only or bi-directional).</param>
protected RNNCell(Shape outputShape, RNNDirection direction)
: base(outputShape)
{
this.Direction = direction;
}
