public void ForwardTest3() { const int batchSize = 2; const int inputSize = 3; const int numberOfNeurons = 2; Session session = new Session(); GRUCell layer = new GRUCell(new Shape(new int[] { batchSize, inputSize }), RNNDirection.BiDirectional, numberOfNeurons, MatrixLayout.ColumnMajor, null); layer.W.Set(new float[] { 0.57935405f, -0.2018174f, 0.3719957f, -0.11352646f, 0.23978919f, 0.30809408f, -0.6668052f, 0.0096491f, 0.17214662f, -0.4206545f, 0.65368795f, -0.07057703f, -0.2414839f, -0.08907348f, 0.42851567f, 0.2056688f, -0.4476247f, 0.02284491f, }); layer.U.Set(new float[] { 0.6758169f, 0.08030099f, 0.6760981f, 0.17049837f, 0.6758169f, 0.08030099f, }); layer.B.Set(new float[] { 0.3719957f, -0.11352646f, 0.23978919f, 0.30809408f, 0.5805608f, -0.33490005f }); Tensor x = new Tensor(null, new[] { batchSize, inputSize }); x.Set(new float[] { 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f }); // set expectations Tensor expected = new Tensor(null, new[] { numberOfNeurons, numberOfNeurons }); expected.Set(new float[] { 0.2298902f, -0.2274744f, 0.4774976f, -0.124593f }); // calculate Tensor y = layer.Forward(session, new[] { x })[0]; Helpers.AreTensorsEqual(expected, y); y.SetGradient(new float[] { 0.1f, 0.2f, 0.3f, 0.4f }); session.Unroll(); Helpers.AreArraysEqual( new float[] { 0.0156396851f, 0.00146510126f, 0.0510645546f, -0.0120584471f, -2.38707144E-05f, 0.111364923f, 0.0216912664f, 0.00183137658f, 0.07349292f, -0.01574755f, -4.774143E-05f, 0.146852463f, 0.0277428478f, 0.002197652f, 0.09592129f, -0.0194366574f, -7.161214E-05f, 0.182340011f }, layer.W.Gradient); Helpers.AreArraysEqual( new float[] { 0.0073473705f, 0.000842030859f, 0.009745982f, 0.001120495f, 2.9741248E-05f, -0.007956707f, }, layer.U.Gradient); Helpers.AreArraysEqual( new float[] { 0.0605158024f, 0.00366275315f, 0.224283636f, -0.0368910469f, -0.000238707144f, 0.3548754f, }, layer.B.Gradient); Helpers.AreArraysEqual( new float[] { 0.0968419462f, -0.000431704335f, 0.0488961339f, 0.134377331f, -0.0109592406f, 0.03289969f, }, x.Gradient); }
public void ForwardTest1() { const int batchSize = 2; const int inputSize = 3; const int numberOfNeurons = 2; Session session = new Session(); GRUCell layer = new GRUCell(new Shape(new int[] { batchSize, inputSize }), RNNDirection.ForwardOnly, numberOfNeurons, MatrixLayout.ColumnMajor, null); layer.W.Set(new float[] { 0.57935405f, -0.2018174f, 0.3719957f, -0.11352646f, 0.23978919f, 0.30809408f, -0.6668052f, 0.0096491f, 0.17214662f, -0.4206545f, 0.65368795f, -0.07057703f, -0.2414839f, -0.08907348f, 0.42851567f, 0.2056688f, -0.4476247f, 0.02284491f, }); layer.U.Set(new float[] { 0.6758169f, 0.08030099f, 0.6760981f, 0.17049837f, 0.6758169f, 0.08030099f, -0.33939722f, 0.44897282f, 0.18849522f, 0.399871f, -0.33939722f, 0.44897282f, }); layer.B.Set(new float[] { 0.3719957f, -0.11352646f, 0.23978919f, 0.30809408f, 0.5805608f, -0.33490005f }); Tensor x = new Tensor(null, new[] { batchSize, inputSize }); x.Set(new float[] { 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f }); // set expectations Tensor expected = new Tensor(null, new[] { numberOfNeurons, numberOfNeurons }); expected.Set(new float[] { 0.2988823f, -0.1389049f, 0.5474511f, -0.1845018f }); // calculate Tensor y = layer.Forward(session, new[] { x })[0]; Helpers.AreTensorsEqual(expected, y); y.SetGradient(new float[] { 0.1f, 0.2f, 0.3f, 0.4f }); session.Unroll(); Helpers.AreArraysEqual( new float[] { 0.01607361f, -0.007430535f, 0.001741877f, -0.000598697f, 0.04607954f, 0.08297627f, 0.02303241f, -0.01174307f, 0.002177347f, -0.0007483712f, 0.0663493f, 0.1174305f, 0.02999121f, -0.0160556f, 0.002612816f, -0.0008980455f, 0.08661906f, 0.1518847f }, layer.W.Gradient); Helpers.AreArraysEqual( new float[] { 0.009080857f, -0.003106385f, 0.001301541f, -0.0004473499f, 0.0183215f, 0.03444418f, -0.004220309f, 0.001443686f, -0.0006048883f, 0.0002079049f, -0.006485323f, -0.01219232f, }, layer.U.Gradient); Helpers.AreArraysEqual( new float[] { 0.06958796f, -0.04312533f, 0.004354693f, -0.001496742f, 0.2026976f, 0.3445423f }, layer.B.Gradient); Helpers.AreArraysEqual( new float[] { 0.1136149f, 0.03690277f, -0.05459791f, 0.09195063f, 0.02584295f, -0.03966833f }, x.Gradient); }
public void ForwardTest2() { const int batchSize = 2; const int inputSize = 3; const int numberOfNeurons = 2; GRUCell layer = new GRUCell(new Shape(new int[] { batchSize, inputSize }), RNNDirection.ForwardOnly, numberOfNeurons, MatrixLayout.RowMajor, null); layer.W.Set(new float[] { 0.57935405f, -0.2018174f, 0.3719957f, -0.11352646f, 0.23978919f, 0.30809408f, -0.6668052f, 0.0096491f, 0.17214662f, -0.4206545f, 0.65368795f, -0.07057703f, -0.2414839f, -0.08907348f, 0.42851567f, 0.2056688f, -0.4476247f, 0.02284491f, }); layer.U.Set(new float[] { 0.6758169f, 0.08030099f, 0.6760981f, 0.17049837f, -0.33939722f, 0.44897282f, 0.18849522f, 0.399871f, 0.6758169f, 0.08030099f, -0.33939722f, 0.44897282f, }); layer.B.Set(new float[] { 0.3719957f, -0.11352646f, 0.23978919f, 0.30809408f, 0.5805608f, -0.33490005f }); Tensor x = new Tensor(null, new[] { batchSize, inputSize }); x.Set(new float[] { 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f }); // set expectations Tensor expected = new Tensor(null, new[] { numberOfNeurons, numberOfNeurons }); expected.Set(new float[] { 0.3631181f, -0.1901545f, 0.5813864f, -0.382444f }); // calculate Session session = new Session(); Tensor y = layer.Forward(session, new[] { x })[0]; Helpers.AreTensorsEqual(expected, y); y.SetGradient(new float[] { 0.1f, 0.2f, 0.3f, 0.4f }); session.Unroll(); Helpers.AreArraysEqual( new float[] { 0.009786441f, 0.01411872f, 0.018451f, -0.01654311f, -0.02359727f, -0.03065143f, 0.0008334472f, 0.001041809f, 0.001250171f, -0.001588227f, -0.001985283f, -0.00238234f, 0.05585977f, 0.07548201f, 0.09510425f, 0.08687179f, 0.1219571f, 0.1570424f }, layer.W.Gradient); Helpers.AreArraysEqual( new float[] { 0.006601683f, -0.003457112f, -0.01148537f, 0.006014556f, 0.0007565994f, -0.0003962094f, -0.001441784f, 0.0007550211f, 0.02053534f, -0.01384354f, 0.02934673f, -0.01978358f, }, layer.U.Gradient); Helpers.AreArraysEqual( new float[] { 0.04332279f, -0.07054159f, 0.002083618f, -0.003970566f, 0.1962224f, 0.3508531f }, layer.B.Gradient); Helpers.AreArraysEqual( new float[] { 0.03742515f, -0.1009044f, 0.03375921f, 0.02073827f, -0.1018581f, 0.05336173f }, x.Gradient); }
public override Batch <double[]> Forward(Batch <double[]> inputs, bool isTraining) { var length1 = this.outputs * 3; var length2 = this.inputs * length1; var length3 = this.outputs * length1; var length4 = this.timesteps * this.outputs; var xWeights = new double[length2]; var hWeights = new double[length3]; var outputs = new double[inputs.Size][]; for (int i = 0; i < length2; i++) { xWeights[i] = this.weights[i]; } for (int i = 0, j = length2; i < length3; i++, j++) { hWeights[i] = this.weights[j]; } for (int i = 0; i < inputs.Size; i++) { outputs[i] = new double[length4]; } this.layerList = new List <GRUCell>(); if (!this.stateful || this.h == null) { this.h = new Batch <double[]>(new double[inputs.Size][]); for (int i = 0; i < inputs.Size; i++) { this.h[i] = new double[this.outputs]; for (int j = 0; j < this.outputs; j++) { this.h[i][j] = 0.0; } } } else if (this.h.Size < inputs.Size) { var batch = new Batch <double[]>(new double[inputs.Size][]); for (int i = 0; i < this.h.Size; i++) { batch[i] = this.h[i]; } for (int i = this.h.Size; i < inputs.Size; i++) { batch[i] = new double[this.outputs]; for (int j = 0; j < this.outputs; j++) { batch[i][j] = 0.0; } } this.h = batch; } for (int t = 0; t < this.timesteps; t++) { var layer = new GRUCell(this.inputs, this.outputs, xWeights, hWeights, this.biases, this.tanhActivationFunction, this.sigmoidActivationFunction); var x = new Batch <double[]>(new double[inputs.Size][]); for (int i = 0; i < inputs.Size; i++) { var vector = new double[this.inputs]; for (int j = 0, k = this.inputs * t; j < this.inputs; j++, k++) { vector[j] = inputs[i][k]; } x[i] = vector; } this.h = layer.Forward(x, this.h); for (int i = 0; i < inputs.Size; i++) { for (int j = 0, k = this.outputs * t; j < this.outputs; j++, k++) { outputs[i][k] = this.h[i][j]; } } this.layerList.Add(layer); } return(new Batch <double[]>(outputs)); }