static void YCalc(Real[] di, Real[,] line, Real[] v, int n, int k) { double sum = 0; int j, m; for (int i = 0; i < n; i++) { j = i - k; if (j < 0) { m = k - i; j = 0; } else { m = 0; } for (; m < k; j++, m++) { sum += line[i, m] * v[j]; } v[i] = (v[i] - sum) / di[i]; sum = 0; } }
static void XCalc(Real[] di, Real[,] line, Real[] v, int n, int k) { double sum = 0; int j, m, q, t, h; h = -1; t = k + 1; for (int i = n - 1; i >= 0; i--) { j = n - i; if (n - k > j - 2) // -2 или k-1 { t--; q = n - 1; } else { h--; q = n + h;; } for (m = t; m < k; j--, m++, q--) { sum += line[q, m] * v[q]; } v[i] = (v[i] - sum) / di[i]; sum = 0; } }
public LSTM(int inSize, int outSize, Real[,] initialUpwardW = null, Real[] initialUpwardb = null, Real[,] initialLateralW = null, string name = FUNCTION_NAME, string[] inputNames = null, string[] outputNames = null, bool gpuEnable = false) : base(name, inputNames, outputNames) { this.InputCount = inSize; this.OutputCount = outSize; List <NdArray> functionParameters = new List <NdArray>(); this.upward0 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward0", gpuEnable: gpuEnable); this.upward1 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward1", gpuEnable: gpuEnable); this.upward2 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward2", gpuEnable: gpuEnable); this.upward3 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward3", gpuEnable: gpuEnable); functionParameters.AddRange(this.upward0.Parameters); functionParameters.AddRange(this.upward1.Parameters); functionParameters.AddRange(this.upward2.Parameters); functionParameters.AddRange(this.upward3.Parameters); //lateralはBiasは無し this.lateral0 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral0", gpuEnable: gpuEnable); this.lateral1 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral1", gpuEnable: gpuEnable); this.lateral2 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral2", gpuEnable: gpuEnable); this.lateral3 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral3", gpuEnable: gpuEnable); functionParameters.AddRange(this.lateral0.Parameters); functionParameters.AddRange(this.lateral1.Parameters); functionParameters.AddRange(this.lateral2.Parameters); functionParameters.AddRange(this.lateral3.Parameters); this.Parameters = functionParameters.ToArray(); SingleInputForward = ForwardCpu; SingleOutputBackward = BackwardCpu; }
static void LCalc(Real[] di, Real[,] line, Real[] v, int n, int k) { double sum = 0, diSum = 0; int q = 0; int[] R = new int[n]; R[0] = k; for (int i = 1; i < k; i++) { R[i] = R[i - 1] - 1; } for (int i = 0; i < n; i++) { if (i > k) { q++; } for (int j = R[i]; j < k; j++) { for (int t = 0; t < j - R[i]; t++) { sum += line[i, R[i] + t] * line[j - R[i] + q, R[j - R[i]] + t]; } line[i, j] = (line[i, j] - sum) / di[j - R[i] + q]; sum = 0; diSum += line[i, j] * line[i, j]; } di[i] = Math.Sqrt(di[i] - diSum); diSum = 0; } }
public void EmbedIDRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 30); int outputCount = Mother.Dice.Next(1, 30); int batchCount = Mother.Dice.Next(1, 5); int[,] input = (int[, ])Enumerable.Repeat(0, batchCount * inputCount).ToNdArray(batchCount, inputCount); input[0, 0] = 1; Real[,,] dummyGy = Initializer.GetRandomValues <Real[, , ]>(batchCount, inputCount, outputCount); Real[,] w = Initializer.GetRandomValues <Real[, ]>(inputCount, outputCount); //Chainer NChainer.EmbedID <Real> cEmbedId = new NChainer.EmbedID <Real>(inputCount, outputCount, w); Variable <int> cX = new Variable <int>(input); Variable <Real> cY = cEmbedId.Forward(cX); cY.Grad = dummyGy; cY.Backward(); //KelpNet EmbedID <Real> embedId = new EmbedID <Real>(inputCount, outputCount, w); NdArray <Real> x = new NdArray <Real>(input, asBatch: true); NdArray <Real> y = embedId.Forward(x)[0]; y.Grad = dummyGy.Flatten(); y.Backward(); Real[] cYdata = ((Real[, , ])cY.Data).Flatten(); Real[] cWgrad = ((Real[, ])cEmbedId.W.Grad).Flatten(); //許容範囲を算出 Real delta = 0.00001f; //y Assert.AreEqual(cYdata.Length, y.Data.Length); for (int i = 0; i < y.Data.Length; i++) { Assert.AreEqual(cYdata[i], y.Data[i], delta); } //W.grad Assert.AreEqual(cWgrad.Length, embedId.Weight.Grad.Length); for (int i = 0; i < embedId.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad[i], embedId.Weight.Grad[i], delta); } }
//////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// Initializes a new instance of the KelpNet.Functions.Connections.LSTM class. /// </summary> /// /// <param name="inSize"> Size of the in. </param> /// <param name="outSize"> Size of the out. </param> /// <param name="initialUpwardW"> (Optional) The initial upward w. </param> /// <param name="initialUpwardb"> (Optional) The initial upwardb. </param> /// <param name="initialLateralW"> (Optional) The initial lateral w. </param> /// <param name="name"> (Optional) The name. </param> /// <param name="inputNames"> (Optional) List of names of the inputs. </param> /// <param name="outputNames"> (Optional) List of names of the outputs. </param> /// <param name="gpuEnable"> (Optional) True if GPU enable. </param> //////////////////////////////////////////////////////////////////////////////////////////////////// public LSTM(bool verbose, int inSize, int outSize, [CanBeNull] Real[,] initialUpwardW = null, [CanBeNull] Real[] initialUpwardb = null, [CanBeNull] Real[,] initialLateralW = null, [CanBeNull] string name = FUNCTION_NAME, [CanBeNull] string[] inputNames = null, [CanBeNull] string[] outputNames = null, bool gpuEnable = false) : base(name, inputNames, outputNames) { Verbose = verbose; InputCount = inSize; OutputCount = outSize; List <NdArray> functionParameters = new List <NdArray>(); upward0 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward0", gpuEnable: gpuEnable); upward1 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward1", gpuEnable: gpuEnable); upward2 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward2", gpuEnable: gpuEnable); upward3 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward3", gpuEnable: gpuEnable); functionParameters.AddRange(upward0.Parameters); functionParameters.AddRange(upward1.Parameters); functionParameters.AddRange(upward2.Parameters); functionParameters.AddRange(upward3.Parameters); // lateral does not have Bias lateral0 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral0", gpuEnable: gpuEnable); lateral1 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral1", gpuEnable: gpuEnable); lateral2 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral2", gpuEnable: gpuEnable); lateral3 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral3", gpuEnable: gpuEnable); functionParameters.AddRange(lateral0.Parameters); functionParameters.AddRange(lateral1.Parameters); functionParameters.AddRange(lateral2.Parameters); functionParameters.AddRange(lateral3.Parameters); Parameters = functionParameters.ToArray(); SingleInputForward = ForwardCpu; SingleOutputBackward = BackwardCpu; }
public void EmbedIDRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 30); int outputCount = Mother.Dice.Next(1, 30); int batchCount = Mother.Dice.Next(1, 5); int[,] input = (int[, ])Enumerable.Repeat(0, batchCount * inputCount).ToNdArray(batchCount, inputCount); input[0, 0] = 1; Real[,,] dummyGy = (Real[, , ])Initializer.GetRealNdArray(new[] { batchCount, inputCount, outputCount }); Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { inputCount, outputCount }); //Chainer NChainer.EmbedID <Real> cEmbedId = new NChainer.EmbedID <Real>(inputCount, outputCount, Real.ToBaseNdArray(w)); Variable <int> cX = new Variable <int>(input); Variable <Real> cY = cEmbedId.Forward(cX); cY.Grad = Real.ToBaseNdArray(dummyGy); cY.Backward(); //KelpNet KelpNet.EmbedID embedId = new KelpNet.EmbedID(inputCount, outputCount, w); NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount); NdArray y = embedId.Forward(x)[0]; y.Grad = Real.ToRealArray(dummyGy); y.Backward(); Real[] cYdata = Real.ToRealArray((Real[, , ])cY.Data); Real[] cWgrad = Real.ToRealArray((Real[, ])cEmbedId.W.Grad); //許容範囲を算出 double delta = 0.00001; //y Assert.AreEqual(cYdata.Length, y.Data.Length); for (int i = 0; i < y.Data.Length; i++) { Assert.AreEqual(cYdata[i], y.Data[i], delta); } //W.grad Assert.AreEqual(cWgrad.Length, embedId.Weight.Grad.Length); for (int i = 0; i < embedId.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad[i], embedId.Weight.Grad[i], delta); } }
protected SquareMatrix(Real[,] squareMatrix) : base(squareMatrix) { if (squareMatrix.GetLength(0) != squareMatrix.GetLength(1)) { throw new ArgumentException("Parameter name: squareMatrix", "squareMatrix"); } }
/// <summary> /// Scalar multiplication with a complex number /// </summary> /// <param name="left"></param> /// <param name="right"></param> /// <returns></returns> public static RealMatrix operator *(RealMatrix left, Real right) { Real[,] indices = left.Indices; for (int i = 0; i < left.Height; i++) { for (int j = 0; j < left.Width; j++) { indices[i, j] *= right; } } return(new RealMatrix(indices)); }
static Real[] multiple(Real[] di, Real[,] line, Real[] v, int n, int k, Real[] res) { for (int i = 0; i < k; i++) { for (int j = 0, m = k - i; j < n; j++) { res[j] += line[j, i] * v[j]; if (j < n - m) { res[j] += line[m, i] * v[j]; } } } for (int i = 0; i < n; i++) { res[i] += di[i] * v[i]; } return(res); }
public EmbedID(int inputCount, int outputCount, Real[,] initialW = null, string name = FUNCTION_NAME, string[] inputNames = null, string[] outputNames = null) : base(name, inputNames, outputNames) { this.InputCount = inputCount; this.OutputCount = outputCount; this.Weight = new NdArray(inputCount, outputCount); this.Weight.Name = this.Name + " Weight"; if (initialW == null) { Initializer.InitWeight(this.Weight); } else { //単純に代入しないのはサイズのチェックを兼ねるため this.Weight.Data = Real.ToRealArray(initialW); } this.Parameters = new[] { this.Weight }; }
static Real[] multipleN(Real[] di, Real[,] line, Real[] v, int n, int k, Real[] res) { int m, j = 0; for (int i = 0; i < n; i++) { res[i] += di[i] * v[i]; m = i - k; if (m < 0) { continue; } for (; j < k; j++, m++) { res[i] += line[i, j] * v[m]; res[m] += line[m, j] * v[i]; } j = 0; } return(res); }
public EmbedID(int inputCount, int outputCount, Real[,] initialW = null, string name = FUNCTION_NAME, string[] inputNames = null, string[] outputNames = null) : base(name, inputNames, outputNames) { this.InputCount = inputCount; this.OutputCount = outputCount; this.Weight = new NdArray(inputCount, outputCount); this.Weight.Name = this.Name + " Weight"; if (initialW == null) { Initializer.InitWeight(this.Weight); } else { //Do not simply substitute to check the size this.Weight.Data = Real.GetArray(initialW); } this.Parameters = new[] { this.Weight }; SingleInputForward = NeedPreviousForwardCpu; SingleOutputBackward = NeedPreviousBackwardCpu; }
//////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// Initializes a new instance of the KelpNet.Functions.Connections.EmbedID class. /// </summary> /// /// <param name="inputCount"> Number of inputs. </param> /// <param name="outputCount"> Number of outputs. </param> /// <param name="initialW"> (Optional) The initial w. </param> /// <param name="name"> (Optional) The name. </param> /// <param name="inputNames"> (Optional) List of names of the inputs. </param> /// <param name="outputNames"> (Optional) List of names of the outputs. </param> //////////////////////////////////////////////////////////////////////////////////////////////////// public EmbedID(int inputCount, int outputCount, [CanBeNull] Real[,] initialW = null, [CanBeNull] string name = FUNCTION_NAME, [CanBeNull] string[] inputNames = null, [CanBeNull] string[] outputNames = null) : base(name, inputNames, outputNames) { InputCount = inputCount; OutputCount = outputCount; Weight = new NdArray(inputCount, outputCount); Weight.Name = Name + " Weight"; if (initialW == null) { Initializer.InitWeight(Weight); } else { // Do not simply substitute for checking the size Weight.Data = Real.GetArray(initialW); } Parameters = new[] { Weight }; SingleInputForward = NeedPreviousForwardCpu; SingleOutputBackward = NeedPreviousBackwardCpu; }
public void SGDRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 50); int outputCount = Mother.Dice.Next(2, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, inputCount }); Real[,] dummyGy = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, outputCount }); Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, inputCount }); Real[] b = Initializer.GetRealArray(outputCount); //Chainer NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b)); NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>(); cSgd.Setup(cLinear); Variable <Real> cX = new Variable <Real>(Real.ToBaseNdArray(input)); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = Real.ToBaseNdArray(dummyGy); cY.Backward(); cSgd.Update(); //KelpNet KelpNet.Linear linear = new KelpNet.Linear(inputCount, outputCount, false, w, b); KelpNet.SGD sgd = new SGD(); sgd.SetUp(linear); NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount); NdArray y = linear.Forward(x)[0]; y.Grad = Real.ToRealArray(dummyGy); y.Backward(); sgd.Update(); Real[] cW = Real.ToRealArray((Real[, ])cLinear.W.Data); Real[] cb = (Real[])cLinear.b.Data; //許容範囲を算出 double delta = 0.00001; //W.grad Assert.AreEqual(cW.Length, linear.Weight.Data.Length); for (int i = 0; i < linear.Weight.Data.Length; i++) { Assert.AreEqual(cW[i], linear.Weight.Data[i], delta); } //b.grad Assert.AreEqual(cb.Length, linear.Bias.Data.Length); for (int i = 0; i < linear.Bias.Data.Length; i++) { Assert.AreEqual(cb[i], linear.Bias.Data[i], delta); } }
protected Matrix(Real[,] matrix) { RowCount = (byte)matrix.GetLength(0); ColumnCount = (byte)matrix.GetLength(1); this.matrix = matrix; }
public void SGDRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 50); int outputCount = Mother.Dice.Next(2, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount); Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount); Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount); Real[] b = Initializer.GetRandomValues <Real[]>(outputCount); //Chainer Linear <Real> cLinear = new Linear <Real>(inputCount, outputCount, false, w, b); NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>(); cSgd.Setup(cLinear); Variable <Real> cX = new Variable <Real>(input); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = dummyGy; cY.Backward(); cSgd.Update(); //KelpNet CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b); KelpNet.SGD <Real> sgd = new SGD <Real>(); sgd.SetUp(linear); NdArray <Real> x = new NdArray <Real>(input, asBatch: true); NdArray <Real> y = linear.Forward(x)[0]; y.Grad = dummyGy.Flatten(); y.Backward(); sgd.Update(); Real[] cW = ((Real[, ])cLinear.W.Data).Flatten(); Real[] cb = (Real[])cLinear.b.Data; //許容範囲を算出 Real delta = 0.00001f; //W.grad Assert.AreEqual(cW.Length, linear.Weight.Data.Length); for (int i = 0; i < linear.Weight.Data.Length; i++) { Assert.AreEqual(cW[i], linear.Weight.Data[i], delta); } //b.grad Assert.AreEqual(cb.Length, linear.Bias.Data.Length); for (int i = 0; i < linear.Bias.Data.Length; i++) { Assert.AreEqual(cb[i], linear.Bias.Data[i], delta); } }
public static void Run() { // Describe each initial value Real[,,,] initial_W1 = { { { { 1.0, 0.5, 0.0 }, { 0.5, 0.0, -0.5 }, { 0.0, -0.5, -1.0 } } }, { { { 0.0, -0.1, 0.1 }, { -0.3, 0.4, 0.7 }, { 0.5, -0.2, 0.2 } } } }; Real[] initial_b1 = { 0.5, 1.0 }; Real[,,,] initial_W2 = { { { { -0.1, 0.6 }, { 0.3, -0.9 } }, { { 0.7, 0.9 }, { -0.2, -0.3 } } }, { { { -0.6, -0.1 }, { 0.3, 0.3 } }, { { -0.5, 0.8 }, { 0.9, 0.1 } } } }; Real[] initial_b2 = { 0.1, 0.9 }; Real[,] initial_W3 = { { 0.5, 0.3, 0.4, 0.2, 0.6, 0.1, 0.4, 0.3 }, { 0.6, 0.4, 0.9, 0.1, 0.5, 0.2, 0.3, 0.4 } }; Real[] initial_b3 = { 0.01, 0.02 }; Real[,] initial_W4 = { { 0.8, 0.2 }, { 0.4, 0.6 } }; Real[] initial_b4 = { 0.02, 0.01 }; //Input data NdArray x = new NdArray(new Real[, , ] { { { 0.0, 0.0, 0.0, 0.0, 0.0, 0.2, 0.9, 0.2, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.9, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.1, 0.8, 0.5, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.3, 0.3, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.1, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.4, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.8, 0.4, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.1, 0.8, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 } } }); //teacher signal Real[] t = { 0.0, 1.0 }; // If you want to check the contents of a layer, have an instance as a single layer Convolution2D l2 = new Convolution2D(true, 1, 2, 3, initialW: initial_W1, initialb: initial_b1, name: "l2 Conv2D"); // Write the network configuration in FunctionStack FunctionStack nn = new FunctionStack("Test5", l2, //new Convolution2D(1, 2, 3, initialW: initial_W1, initialb: initial_b1), new ReLU(name: "l2 ReLU"), //new AveragePooling(2, 2, name: "l2 AVGPooling"), new MaxPooling(2, 2, name: "l2 MaxPooling"), new Convolution2D(true, 2, 2, 2, initialW: initial_W2, initialb: initial_b2, name: "l3 Conv2D"), new ReLU(name: "l3 ReLU"), //new AveragePooling(2, 2, name: "l3 AVGPooling"), new MaxPooling(2, 2, name: "l3 MaxPooling"), new Linear(true, 8, 2, initialW: initial_W3, initialb: initial_b3, name: "l4 Linear"), new ReLU(name: "l4 ReLU"), new Linear(true, 2, 2, initialW: initial_W4, initialb: initial_b4, name: "l5 Linear") ); // If you omit the optimizer declaration, the default SGD(0.1) will be used // nn.SetOptimizer(new SGD()); // Training conducted Trainer.Train(nn, x, t, new MeanSquaredError(), false); // If Update is executed, grad is consumed, so output the value first RILogManager.Default?.SendDebug("gw1"); RILogManager.Default?.SendDebug(l2.Weight.ToString("Grad")); RILogManager.Default?.SendDebug("gb1"); RILogManager.Default?.SendDebug(l2.Bias.ToString("Grad")); //update nn.Update(); RILogManager.Default?.SendDebug("w1"); RILogManager.Default?.SendDebug(l2.Weight.ToString()); RILogManager.Default?.SendDebug("b1"); RILogManager.Default?.SendDebug(l2.Bias.ToString()); }
public void RandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = 1 + Mother.Dice.Next() % 49; int outputCount = 1 + Mother.Dice.Next() % 49; Real[,] input = (Real[, ])Initializer.GetRealNdArray(new[] { 1, inputCount }); Real[,] dummyGy = (Real[, ])Initializer.GetRealNdArray(new[] { 1, outputCount }); Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, inputCount }); Real[] b = Initializer.GetRealArray(outputCount); //Chainer NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b)); Variable <Real> cX = new Variable <Real>(Real.ToBaseNdArray(input)); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = Real.ToBaseNdArray(dummyGy); cY.Backward(); //KelpNet KelpNet.Linear linear = new KelpNet.Linear(inputCount, outputCount, false, w, b); NdArray x = new NdArray(input); NdArray y = linear.Forward(x)[0]; y.Grad = Real.ToRealArray(dummyGy); y.Backward(); Real[] cYdata = Real.ToRealArray((Real[, ])cY.Data); Real[] cXgrad = Real.ToRealArray((Real[, ])cX.Grad); Real[] cWgrad = Real.ToRealArray((Real[, ])cLinear.W.Grad); Real[] cbgrad = (Real[])cLinear.b.Grad; //許容範囲を算出 double delta = 0.00001; //y Assert.AreEqual(cYdata.Length, y.Data.Length); for (int i = 0; i < y.Data.Length; i++) { Assert.AreEqual(cYdata[i], y.Data[i], delta); } //x.grad Assert.AreEqual(cXgrad.Length, x.Grad.Length); for (int i = 0; i < x.Grad.Length; i++) { Assert.AreEqual(cXgrad[i], x.Grad[i], delta); } //W.grad Assert.AreEqual(cWgrad.Length, linear.Weight.Grad.Length); for (int i = 0; i < linear.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad[i], linear.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad.Length, linear.Bias.Grad.Length); for (int i = 0; i < linear.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad[i], linear.Bias.Grad[i], delta); } }
public void RandomTest(bool gpuEnable) { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(1, 50); int outputCount = Mother.Dice.Next(1, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount); Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount); Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount); Real[] b = Initializer.GetRandomValues <Real[]>(outputCount); //Chainer NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, w, b); Variable <Real> cX = new Variable <Real>(input); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = dummyGy; cY.Backward(); //KelpNet CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b, gpuEnable: gpuEnable); NdArray <Real> x = new NdArray <Real>(input, asBatch: true); NdArray <Real> y = linear.Forward(x)[0]; y.Grad = dummyGy.Flatten(); y.Backward(); Real[] cYdata = ((Real[, ])cY.Data).Flatten(); Real[] cXgrad = ((Real[, ])cX.Grad).Flatten(); Real[] cWgrad = ((Real[, ])cLinear.W.Grad).Flatten(); Real[] cbgrad = (Real[])cLinear.b.Grad; //許容範囲を算出 Real delta = 0.00001f; //y Assert.AreEqual(cYdata.Length, y.Data.Length); for (int i = 0; i < y.Data.Length; i++) { Assert.AreEqual(cYdata[i], y.Data[i], delta); } //x.grad Assert.AreEqual(cXgrad.Length, x.Grad.Length); for (int i = 0; i < x.Grad.Length; i++) { Assert.AreEqual(cXgrad[i], x.Grad[i], delta); } //W.grad Assert.AreEqual(cWgrad.Length, linear.Weight.Grad.Length); for (int i = 0; i < linear.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad[i], linear.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad.Length, linear.Bias.Grad.Length); for (int i = 0; i < linear.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad[i], linear.Bias.Grad[i], delta); } }
public static void Run() { //Align Weight before and after splitting Real[,] testWeightValues = { { -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }, { -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }, { -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }, { -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 } }, new Real[, ] { { -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 } }, new Real[, ] { { -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 } }, new Real[, ] { { -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } } }; Linear l0 = new Linear(true, 5, 20, initialW: testWeightValues, name: "l0"); Linear l1 = new Linear(true, 5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear l2 = new Linear(true, 5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear l3 = new Linear(true, 5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear l4 = new Linear(true, 5, 5, initialW: testJaggWeightValues[3], name: "l4"); l0.SetOptimizer(new SGD()); SGD sgd = new SGD(); l1.SetOptimizer(sgd); l2.SetOptimizer(sgd); l3.SetOptimizer(sgd); l4.SetOptimizer(sgd); //Input is equivalent, but Grad is added and it is divided Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 }; NdArray testInputValuesA = new NdArray(testValue); NdArray testInputValuesB = new NdArray(testValue); RILogManager.Default?.SendDebug("l0 for"); NdArray[] l0Result = l0.Forward(true, testInputValuesA); RILogManager.Default?.SendDebug(l0Result.ToString()); RILogManager.Default?.SendDebug("l1 for"); NdArray[] l1Result = l1.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l1Result.ToString()); RILogManager.Default?.SendDebug("l2 for"); NdArray[] l2Result = l2.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l2Result.ToString()); RILogManager.Default?.SendDebug("l3 for"); NdArray[] l3Result = l3.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l3Result.ToString()); RILogManager.Default?.SendDebug("l4 for"); NdArray[] l4Result = l4.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l4Result.ToString()); //Create an appropriate Grad value l0Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03, 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04, -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04, -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; l1Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 }; l2Result[0].Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 }; l3Result[0].Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 }; l4Result[0].Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; //Backward l0.Backward(true, l0Result); l1.Backward(true, l1Result); l2.Backward(true, l2Result); l3.Backward(true, l3Result); l4.Backward(true, l4Result); RILogManager.Default?.SendDebug("l0 back"); RILogManager.Default?.SendDebug(testInputValuesA.ToString("Grad")); RILogManager.Default?.SendDebug("l1-l4 sum back"); RILogManager.Default?.SendDebug(testInputValuesB.ToString("Grad")); l0.Update(); //Although the format is irregular, since 10 contains SGD sgd.Update(); // Use stochastic gradient descent as the optimizer RILogManager.Default?.SendDebug("l0 Weight"); RILogManager.Default?.SendDebug(l0.Weight.ToString()); RILogManager.Default?.SendDebug("l1 Weight"); RILogManager.Default?.SendDebug(l1.Weight.ToString()); RILogManager.Default?.SendDebug("l0 Bias"); RILogManager.Default?.SendDebug(l0.Bias.ToString()); RILogManager.Default?.SendDebug("l1 Bias"); RILogManager.Default?.SendDebug(l1.Bias.ToString()); }
public void SwishRandomTest() { Python.Initialize(); Chainer.Initialize(); int ioCount = Mother.Dice.Next(1, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, ioCount }); Real[,] dummyGy = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, ioCount }); Real beta = Mother.Dice.NextDouble(); //Chainer NChainer.Swish<Real> cSwish = new NChainer.Swish<Real>(new[] { ioCount }, beta); Variable<Real> cX = new Variable<Real>(Real.ToBaseNdArray(input)); Variable<Real> cY = cSwish.Forward(cX); cY.Grad = Real.ToBaseNdArray(dummyGy); cY.Backward(); //KelpNet KelpNet.Swish swish = new KelpNet.Swish(new[] { ioCount }, beta); NdArray x = new NdArray(Real.ToRealArray(input), new[] { ioCount }, batchCount); NdArray y = swish.Forward(x)[0]; y.Grad = Real.ToRealArray(dummyGy); y.Backward(); Real[] cYdata = Real.ToRealArray((Real[,])cY.Data); Real[] cXgrad = Real.ToRealArray((Real[,])cX.Grad); Real[] cbgrad = (Real[])cSwish.beta.Grad; //許容範囲を算出 double delta = 0.00001; //y Assert.AreEqual(cYdata.Length, y.Data.Length); for (int i = 0; i < y.Data.Length; i++) { Assert.AreEqual(cYdata[i], y.Data[i], delta); } //x.grad Assert.AreEqual(cXgrad.Length, x.Grad.Length); for (int i = 0; i < x.Grad.Length; i++) { Assert.AreEqual(cXgrad[i], x.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad.Length, swish.Beta.Grad.Length); for (int i = 0; i < swish.Beta.Grad.Length; i++) { Assert.AreEqual(cbgrad[i], swish.Beta.Grad[i], delta); } }
public void LSTMRandomTest() { Python.Initialize(); Chainer.Initialize(); int batchCount = Mother.Dice.Next(1, 5); int inputCount = Mother.Dice.Next(1, 5); int outputCount = Mother.Dice.Next(1, 5); Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount); Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount); Real[,] upwardInit = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount); Real[,] lateralInit = Initializer.GetRandomValues <Real[, ]>(outputCount, outputCount); Real[,,] biasInit = Initializer.GetRandomValues <Real[, , ]>(1, outputCount, 1); Real[,,] forgetBiasInit = Initializer.GetRandomValues <Real[, , ]>(1, outputCount, 1); //Chainer NChainer.LSTM <Real> clstm = new NChainer.LSTM <Real>(inputCount, outputCount, lateralInit, upwardInit, biasInit, forgetBiasInit); Variable <Real> cX = new Variable <Real>(input); Variable <Real> cY = clstm.Forward(cX); cY.Grad = dummyGy; cY.Backward(); //KelpNet LSTM <Real> lstm = new LSTM <Real>(inputCount, outputCount, lateralInit, upwardInit, biasInit, forgetBiasInit); NdArray <Real> x = new NdArray <Real>(input, asBatch: true); NdArray <Real> y = lstm.Forward(x)[0]; y.Grad = dummyGy.Flatten(); y.Backward(); //許容範囲を算出 Real delta = 0.00001f; Real[] cYdata = ((Real[, ])cY.Data).Flatten(); Real[] cXgrad = ((Real[, ])cX.Grad).Flatten(); Real[] cupwardWGrad = ((Real[, ])clstm.upward.W.Grad).Flatten(); Real[] cupwardbGrad = (Real[])clstm.upward.b.Grad; //y Assert.AreEqual(cYdata.Length, y.Data.Length); for (int i = 0; i < cYdata.Length; i++) { Assert.AreEqual(cYdata[i], y.Data[i], delta); } //x.Grad Assert.AreEqual(cXgrad.Length, x.Grad.Length); for (int i = 0; i < cXgrad.Length; i++) { Assert.AreEqual(cXgrad[i], x.Grad[i], delta); } //W.grad int wLen = lstm.upward.Weight.Grad.Length; Assert.AreEqual(cupwardWGrad.Length, lstm.upward.Weight.Grad.Length); for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad int bLen = lstm.upward.Bias.Length; Assert.AreEqual(cupwardbGrad.Length, lstm.upward.Bias.Grad.Length); for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } ////////////////////////////////////////////////////////////////////////////////////////// // 1度目はlateralに値はない // ////////////////////////////////////////////////////////////////////////////////////////// //Real[] clateralWGrad = Real.ToRealArray((Real[,])clstm.lateral.W.Grad); //Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length); //for (int i = 0; i < wLen; i++) //{ // Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta); //} ////////////////////////////////////////////////////////////////////////////////////////// /////////// //2周目 // /////////// Real[,] input2 = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount); Real[,] dummyGy2 = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount); //Chainer Variable <Real> cX2 = new Variable <Real>(input2); Variable <Real> cY2 = clstm.Forward(cX2); cY2.Grad = dummyGy2; cY2.Backward(); //KelpNet NdArray <Real> x2 = new NdArray <Real>(input2, asBatch: true); NdArray <Real> y2 = lstm.Forward(x2)[0]; y2.Grad = dummyGy2.Flatten(); y2.Backward(); Real[] cYdata2 = ((Real[, ])cY2.Data).Flatten(); Real[] cXgrad2 = ((Real[, ])cX2.Grad).Flatten(); Real[] cupwardWGrad2 = ((Real[, ])clstm.upward.W.Grad).Flatten(); Real[] cupwardbGrad2 = (Real[])clstm.upward.b.Grad; Real[] clateralWGrad = ((Real[, ])clstm.lateral.W.Grad).Flatten(); //y Assert.AreEqual(cYdata2.Length, y2.Data.Length); for (int i = 0; i < cYdata2.Length; i++) { Assert.AreEqual(cYdata2[i], y2.Data[i], delta); } //x.Grad Assert.AreEqual(cXgrad2.Length, x2.Grad.Length); for (int i = 0; i < cXgrad2.Length; i++) { Assert.AreEqual(cXgrad2[i], x2.Grad[i], delta); } //W.grad Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length); for (int i = 0; i < clateralWGrad.Length; i++) { Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta); } //経由が多いため誤差が大きい delta = 1.0f; for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad2[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad2[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } }
public void RnnLSTMRandomTest() { Python.Initialize(); Chainer.Initialize(); Real[,] input = { { 1.0 }, { 3.0 }, { 5.0 }, { 7.0 }, { 9.0 } }; Real[,] teach = { { 3.0 }, { 5.0 }, { 7.0 }, { 9.0 }, { 11.0 } }; Real[,] input2 = { { 3.0 }, { 5.0 }, { 7.0 }, { 9.0 }, { 11.0 } }; Real[,] teach2 = { { 5.0 }, { 7.0 }, { 9.0 }, { 11.0 }, { 13.0 } }; int outputCount = 1; int inputCount = 1; int hiddenCount = 2; Real[,] upwardInit = (Real[, ])Initializer.GetRealNdArray(new[] { hiddenCount, hiddenCount }); Real[,] lateralInit = (Real[, ])Initializer.GetRealNdArray(new[] { hiddenCount, hiddenCount }); Real[,,] biasInit = (Real[, , ])Initializer.GetRealNdArray(new[] { 1, hiddenCount, 1 }); Real[,,] forgetBiasInit = (Real[, , ])Initializer.GetRealNdArray(new[] { 1, hiddenCount, 1 }); //Chainer Real[,] w1 = (Real[, ])Initializer.GetRealNdArray(new[] { hiddenCount, inputCount }); Real[] b1 = Initializer.GetRealArray(hiddenCount); //Chainer NChainer.Linear <Real> cLinear1 = new NChainer.Linear <Real>(inputCount, hiddenCount, false, Real.ToBaseNdArray(w1), Real.ToBaseArray(b1)); NChainer.LSTM <Real> cLstm = new NChainer.LSTM <Real>(hiddenCount, hiddenCount, Real.ToBaseNdArray(lateralInit), Real.ToBaseNdArray(upwardInit), Real.ToBaseNdArray(biasInit), Real.ToBaseNdArray(forgetBiasInit)); Real[,] w2 = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, hiddenCount }); Real[] b2 = Initializer.GetRealArray(outputCount); NChainer.Linear <Real> cLinear2 = new NChainer.Linear <Real>(hiddenCount, outputCount, false, Real.ToBaseNdArray(w2), Real.ToBaseArray(b2)); Variable <Real> cX1 = new Variable <Real>(Real.ToBaseNdArray(input)); Variable <Real> cY11 = cLinear1.Forward(cX1); Variable <Real> cY12 = cLstm.Forward(cY11); Variable <Real> cY13 = cLinear2.Forward(cY12); Variable <Real> cT = new Variable <Real>(Real.ToBaseNdArray(teach)); Variable <Real> cLoss = new NChainer.MeanSquaredError <Real>().Forward(cY13, cT); cLoss.Backward(); //KelpNet KelpNet.CL.Linear linear1 = new KelpNet.CL.Linear(inputCount, hiddenCount, false, w1, b1); KelpNet.LSTM lstm = new KelpNet.LSTM(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit); KelpNet.CL.Linear linear2 = new KelpNet.CL.Linear(hiddenCount, outputCount, false, w2, b2); NdArray x1 = new NdArray(Real.ToRealArray(input), new[] { 1 }, 5); NdArray y11 = linear1.Forward(x1)[0]; NdArray y12 = lstm.Forward(y11)[0]; NdArray y13 = linear2.Forward(y12)[0]; NdArray t = new NdArray(Real.ToRealArray(teach), new[] { 1 }, 5); NdArray loss = new KelpNet.MeanSquaredError().Evaluate(y13, t); y13.Backward(); Real[] cY11data = Real.ToRealArray((Real[, ])cY11.Data); Real[] cY12data = Real.ToRealArray((Real[, ])cY12.Data); Real[] cY13data = Real.ToRealArray((Real[, ])cY13.Data); Real[] cXgrad = Real.ToRealArray((Real[, ])cX1.Grad); Real[] cupwardWGrad = Real.ToRealArray((Real[, ])cLstm.upward.W.Grad); Real[] cupwardbGrad = (Real[])cLstm.upward.b.Grad; //許容範囲を設定 double delta = 0.00001; //y11 Assert.AreEqual(cY11data.Length, y11.Data.Length); for (int i = 0; i < cY11data.Length; i++) { Assert.AreEqual(cY11data[i], y11.Data[i], delta); } //y12 Assert.AreEqual(cY12data.Length, y12.Data.Length); for (int i = 0; i < cY12data.Length; i++) { Assert.AreEqual(cY12data[i], y12.Data[i], delta); } //y13 Assert.AreEqual(cY13data.Length, y13.Data.Length); for (int i = 0; i < cY13data.Length; i++) { Assert.AreEqual(cY13data[i], y13.Data[i], delta); } //許容範囲を設定 delta = 0.0001; //loss Assert.AreEqual(cLoss.Data[0], loss.Data[0], delta); //x.Grad Assert.AreEqual(cXgrad.Length, x1.Grad.Length); for (int i = 0; i < cXgrad.Length; i++) { Assert.AreEqual(cXgrad[i], x1.Grad[i], delta); } Real[] cWgrad11 = Real.ToRealArray((Real[, ])cLinear1.W.Grad); Real[] cbgrad11 = (Real[])cLinear1.b.Grad; //W.grad Assert.AreEqual(cWgrad11.Length, linear1.Weight.Grad.Length); for (int i = 0; i < linear1.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad11[i], linear1.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad11.Length, linear1.Bias.Grad.Length); for (int i = 0; i < linear1.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad11[i], linear1.Bias.Grad[i], delta); } Real[] cWgrad12 = Real.ToRealArray((Real[, ])cLinear2.W.Grad); Real[] cbgrad12 = (Real[])cLinear2.b.Grad; //W.grad Assert.AreEqual(cWgrad12.Length, linear2.Weight.Grad.Length); for (int i = 0; i < linear2.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad12[i], linear2.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad12.Length, linear2.Bias.Grad.Length); for (int i = 0; i < linear2.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad12[i], linear2.Bias.Grad[i], delta); } //W.grad int wLen = lstm.upward.Weight.Grad.Length; Assert.AreEqual(cupwardWGrad.Length, lstm.upward.Weight.Grad.Length); for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad int bLen = lstm.upward.Bias.Length; Assert.AreEqual(cupwardbGrad.Length, lstm.upward.Bias.Grad.Length); for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } //2周目 Variable <Real> cX2 = new Variable <Real>(Real.ToBaseNdArray(input2)); Variable <Real> cY21 = cLinear1.Forward(cX2); Variable <Real> cY22 = cLstm.Forward(cY21); Variable <Real> cY23 = cLinear2.Forward(cY22); Variable <Real> cT2 = new Variable <Real>(Real.ToBaseNdArray(teach2)); Variable <Real> cLoss2 = new NChainer.MeanSquaredError <Real>().Forward(cY23, cT2); //KelpNet NdArray x2 = new NdArray(Real.ToRealArray(input2), new[] { 1 }, 5); NdArray y21 = linear1.Forward(x2)[0]; NdArray y22 = lstm.Forward(y21)[0]; NdArray y23 = linear2.Forward(y22)[0]; NdArray t2 = new NdArray(Real.ToRealArray(teach2), new[] { 1 }, 5); NdArray loss2 = new KelpNet.MeanSquaredError().Evaluate(y23, t2); Assert.AreEqual(cLoss2.Data[0], loss2.Data[0], delta); //Backwardを実行 cLoss2.Backward(); y23.Backward(); Real[] cYdata21 = Real.ToRealArray((Real[, ])cY21.Data); Real[] cYdata22 = Real.ToRealArray((Real[, ])cY22.Data); Real[] cYdata23 = Real.ToRealArray((Real[, ])cY23.Data); Real[] cXgrad2 = Real.ToRealArray((Real[, ])cX2.Grad); Real[] cupwardWGrad2 = Real.ToRealArray((Real[, ])cLstm.upward.W.Grad); Real[] cupwardbGrad2 = (Real[])cLstm.upward.b.Grad; Real[] clateralWGrad = Real.ToRealArray((Real[, ])cLstm.lateral.W.Grad); //y21 Assert.AreEqual(cYdata21.Length, y21.Data.Length); for (int i = 0; i < cYdata21.Length; i++) { Assert.AreEqual(cYdata21[i], y21.Data[i], delta); } //y22 Assert.AreEqual(cYdata22.Length, y22.Data.Length); for (int i = 0; i < cYdata22.Length; i++) { Assert.AreEqual(cYdata22[i], y22.Data[i], delta); } //y23 Assert.AreEqual(cYdata23.Length, y23.Data.Length); for (int i = 0; i < cYdata23.Length; i++) { Assert.AreEqual(cYdata23[i], y23.Data[i], delta); } //x.Grad Assert.AreEqual(cXgrad2.Length, x2.Grad.Length); for (int i = 0; i < cXgrad2.Length; i++) { Assert.AreEqual(cXgrad2[i], x2.Grad[i], delta); } //経由が多くかなり誤差が大きい為 delta = 1.0; Real[] cWgrad22 = Real.ToRealArray((Real[, ])cLinear2.W.Grad); Real[] cbgrad22 = (Real[])cLinear2.b.Grad; //W.grad Assert.AreEqual(cWgrad22.Length, linear2.Weight.Grad.Length); for (int i = 0; i < linear2.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad22[i], linear2.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad22.Length, linear2.Bias.Grad.Length); for (int i = 0; i < linear2.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad22[i], linear2.Bias.Grad[i], delta); } delta = 2.0; //W.grad Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length); for (int i = 0; i < clateralWGrad.Length; i++) { Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta); } for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad2[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad2[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } delta = 20.0; Real[] cWgrad21 = Real.ToRealArray((Real[, ])cLinear1.W.Grad); Real[] cbgrad21 = (Real[])cLinear1.b.Grad; //W.grad Assert.AreEqual(cWgrad21.Length, linear1.Weight.Grad.Length); for (int i = 0; i < linear1.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad21[i], linear1.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad21.Length, linear1.Bias.Grad.Length); for (int i = 0; i < linear1.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad21[i], linear1.Bias.Grad[i], delta); } }
public static void Run() { //各初期値を記述 Real[,,,] initial_W1 = { { { { 1.0, 0.5, 0.0 }, { 0.5, 0.0, -0.5 }, { 0.0, -0.5, -1.0 } } }, { { { 0.0, -0.1, 0.1 }, { -0.3, 0.4, 0.7 }, { 0.5, -0.2, 0.2 } } } }; Real[] initial_b1 = { 0.5, 1.0 }; Real[,,,] initial_W2 = { { { { -0.1, 0.6 }, { 0.3, -0.9 } }, { { 0.7, 0.9 }, { -0.2, -0.3 } } }, { { { -0.6, -0.1 }, { 0.3, 0.3 } }, { { -0.5, 0.8 }, { 0.9, 0.1 } } } }; Real[] initial_b2 = { 0.1, 0.9 }; Real[,] initial_W3 = { { 0.5, 0.3, 0.4, 0.2, 0.6, 0.1, 0.4, 0.3 }, { 0.6, 0.4, 0.9, 0.1, 0.5, 0.2, 0.3, 0.4 } }; Real[] initial_b3 = { 0.01, 0.02 }; Real[,] initial_W4 = { { 0.8, 0.2 }, { 0.4, 0.6 } }; Real[] initial_b4 = { 0.02, 0.01 }; //入力データ NdArray x = new NdArray(new Real[, , ] { { { 0.0, 0.0, 0.0, 0.0, 0.0, 0.2, 0.9, 0.2, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.9, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.1, 0.8, 0.5, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.3, 0.3, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.1, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.4, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.8, 0.4, 0.1, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.1, 0.8, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 } } }); //教師信号 Real[] t = { 0.0, 1.0 }; //層の中身をチェックしたい場合は、層単体でインスタンスを持つ Convolution2D l2 = new Convolution2D(1, 2, 3, initialW: initial_W1, initialb: initial_b1, name: "l2 Conv2D"); //ネットワークの構成を FunctionStack に書き連ねる FunctionStack nn = new FunctionStack( l2, //new Convolution2D(1, 2, 3, initialW: initial_W1, initialb: initial_b1), new ReLU(name: "l2 ReLU"), //new AveragePooling(2, 2, name: "l2 AVGPooling"), new MaxPooling2D(2, 2, name: "l2 MaxPooling"), new Convolution2D(2, 2, 2, initialW: initial_W2, initialb: initial_b2, name: "l3 Conv2D"), new ReLU(name: "l3 ReLU"), //new AveragePooling(2, 2, name: "l3 AVGPooling"), new MaxPooling2D(2, 2, name: "l3 MaxPooling"), new Linear(8, 2, initialW: initial_W3, initialb: initial_b3, name: "l4 Linear"), new ReLU(name: "l4 ReLU"), new Linear(2, 2, initialW: initial_W4, initialb: initial_b4, name: "l5 Linear") ); nn.SetOptimizer(new SGD(0.1)); //訓練を実施 Trainer.Train(nn, x, t, new MeanSquaredError(), false); //Updateを実行するとgradが消費されてしまうため値を先に出力 Console.WriteLine("gw1"); Console.WriteLine(l2.Weight.ToString("Grad")); Console.WriteLine("gb1"); Console.WriteLine(l2.Bias.ToString("Grad")); //更新 nn.Update(); Console.WriteLine("w1"); Console.WriteLine(l2.Weight); Console.WriteLine("b1"); Console.WriteLine(l2.Bias); }
public void AdamRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 50); int outputCount = Mother.Dice.Next(2, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount); Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount); Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount); Real[] b = Initializer.GetRandomValues <Real[]>(outputCount); float alpha = (float)Mother.Dice.NextDouble(); //0.001f float beta1 = (float)Mother.Dice.NextDouble(); //0.9f; float beta2 = (float)Mother.Dice.NextDouble(); //0.999f; float eps = (float)Mother.Dice.NextDouble(); //1e-08f; float eta = (float)Mother.Dice.NextDouble(); //1.0f; //Chainer NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, w, b); NChainer.Adam <Real> cAdam = new NChainer.Adam <Real>(alpha, beta1, beta2, eps, eta); cAdam.Setup(cLinear); Variable <Real> cX = new Variable <Real>(input); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = dummyGy; cY.Backward(); cAdam.Update(); //KelpNet KelpNet.CL.Linear <Real> linear = new KelpNet.CL.Linear <Real>(inputCount, outputCount, false, w, b); KelpNet.Adam <Real> adam = new Adam <Real>(alpha, beta1, beta2, eps, eta); adam.SetUp(linear); NdArray <Real> x = new NdArray <Real>(input, asBatch: true); NdArray <Real> y = linear.Forward(x)[0]; y.Grad = dummyGy.Flatten(); y.Backward(); adam.Update(); Real[] cW = ((Real[, ])cLinear.W.Data).Flatten(); Real[] cb = (Real[])cLinear.b.Data; //許容範囲を算出 Real delta = 0.00001f; //W.grad Assert.AreEqual(cW.Length, linear.Weight.Data.Length); for (int i = 0; i < linear.Weight.Data.Length; i++) { Assert.AreEqual(cW[i], linear.Weight.Data[i], delta); } //b.grad Assert.AreEqual(cb.Length, linear.Bias.Data.Length); for (int i = 0; i < linear.Bias.Data.Length; i++) { Assert.AreEqual(cb[i], linear.Bias.Data[i], delta); } }
public static void Run() { //Weightを分割の前と後で揃える Real[,] testWeightValues = { { -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }, { -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }, { -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }, { -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 } }, new Real[, ] { { -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 } }, new Real[, ] { { -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 } }, new Real[, ] { { -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } } }; Linear l0 = new Linear(5, 20, initialW: testWeightValues, name: "l0"); Linear l1 = new Linear(5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear l2 = new Linear(5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear l3 = new Linear(5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear l4 = new Linear(5, 5, initialW: testJaggWeightValues[3], name: "l4"); //FunctionにOptimizerを設定 l0.SetOptimizer(new SGD()); //OptimiserにFunctionを登録 SGD sgd = new SGD(); l1.SetOptimizer(sgd); l2.SetOptimizer(sgd); l3.SetOptimizer(sgd); l4.SetOptimizer(sgd); //入力は同値だがGradが加算されてしまうため分ける Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 }; NdArray testInputValuesA = new NdArray(testValue); NdArray testInputValuesB = new NdArray(testValue); Console.WriteLine("l0 for"); NdArray l0Result = l0.Forward(testInputValuesA)[0]; Console.WriteLine(l0Result); Console.WriteLine("\nl1 for"); NdArray l1Result = l1.Forward(testInputValuesB)[0]; Console.WriteLine(l1Result); Console.WriteLine("\nl2 for"); NdArray l2Result = l2.Forward(testInputValuesB)[0]; Console.WriteLine(l2Result); Console.WriteLine("\nl3 for"); NdArray l3Result = l3.Forward(testInputValuesB)[0]; Console.WriteLine(l3Result); Console.WriteLine("\nl4 for"); NdArray l4Result = l4.Forward(testInputValuesB)[0]; Console.WriteLine(l4Result); Console.WriteLine(); //適当なGrad値をでっち上げる l0Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03, 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04, -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04, -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; l1Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 }; l2Result.Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 }; l3Result.Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 }; l4Result.Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; //Backwardを実行 l0.Backward(l0Result); l1.Backward(l1Result); l2.Backward(l2Result); l3.Backward(l3Result); l4.Backward(l4Result); Console.WriteLine("\nl0 back"); Console.WriteLine(testInputValuesA.ToString("Grad")); Console.WriteLine("\nl1-l4 sum back"); Console.WriteLine(testInputValuesB.ToString("Grad")); l0.Update(); //書式が変則的だがl0はSGDを内包しているため sgd.Update(); //こちらはOptimizerに関数を登録して使用している Console.WriteLine("\nl0 Weight"); Console.WriteLine(l0.Weight); Console.WriteLine("\nl1 Weight"); Console.WriteLine(l1.Weight); Console.WriteLine("\nl0 Bias"); Console.WriteLine(l0.Bias); Console.WriteLine("\nl1 Bias"); Console.WriteLine(l1.Bias); }
public void AdamRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 50); int outputCount = Mother.Dice.Next(2, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, inputCount }); Real[,] dummyGy = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, outputCount }); Real[,] w = (Real[,])Initializer.GetRealNdArray(new[] { outputCount, inputCount }); Real[] b = Initializer.GetRealArray(outputCount); float alpha = (float)Mother.Dice.NextDouble(); //0.001f float beta1 = (float)Mother.Dice.NextDouble(); //0.9f; float beta2 = (float)Mother.Dice.NextDouble(); //0.999f; float eps = (float)Mother.Dice.NextDouble(); //1e-08f; float eta = (float)Mother.Dice.NextDouble(); //1.0f; //Chainer NChainer.Linear<Real> cLinear = new NChainer.Linear<Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b)); NChainer.Adam<Real> cAdam = new NChainer.Adam<Real>(alpha, beta1, beta2, eps, eta); cAdam.Setup(cLinear); Variable<Real> cX = new Variable<Real>(Real.ToBaseNdArray(input)); Variable<Real> cY = cLinear.Forward(cX); cY.Grad = Real.ToBaseNdArray(dummyGy); cY.Backward(); cAdam.Update(); //KelpNet KelpNet.CL.Linear linear = new KelpNet.CL.Linear(inputCount, outputCount, false, w, b); KelpNet.Adam adam = new Adam(alpha, beta1, beta2, eps, eta); adam.SetUp(linear); NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount); NdArray y = linear.Forward(x)[0]; y.Grad = Real.ToRealArray(dummyGy); y.Backward(); adam.Update(); Real[] cW = Real.ToRealArray((Real[,])cLinear.W.Data); Real[] cb = (Real[])cLinear.b.Data; //許容範囲を算出 double delta = 0.00001; //W.grad Assert.AreEqual(cW.Length, linear.Weight.Data.Length); for (int i = 0; i < linear.Weight.Data.Length; i++) { Assert.AreEqual(cW[i], linear.Weight.Data[i], delta); } //b.grad Assert.AreEqual(cb.Length, linear.Bias.Data.Length); for (int i = 0; i < linear.Bias.Data.Length; i++) { Assert.AreEqual(cb[i], linear.Bias.Data[i], delta); } }
public FilteringMatrix(Real[,] filteringMatrix) : base(filteringMatrix) { Factor = DefaultFactor; Bias = DefaultBias; }
public void RnnLSTMRandomTest() { Python.Initialize(); Chainer.Initialize(); Real[,] input = { { 1.0f }, { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f } }; Real[,] teach = { { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f } }; Real[,] input2 = { { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f } }; Real[,] teach2 = { { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f }, { 13.0f } }; int outputCount = 1; int inputCount = 1; int hiddenCount = 2; Real[,] upwardInit = Initializer.GetRandomValues <Real[, ]>(hiddenCount, hiddenCount); Real[,] lateralInit = Initializer.GetRandomValues <Real[, ]>(hiddenCount, hiddenCount); Real[,,] biasInit = Initializer.GetRandomValues <Real[, , ]>(1, hiddenCount, 1); Real[,,] forgetBiasInit = Initializer.GetRandomValues <Real[, , ]>(1, hiddenCount, 1); //Chainer Real[,] w1 = Initializer.GetRandomValues <Real[, ]>(hiddenCount, inputCount); Real[] b1 = Initializer.GetRandomValues <Real[]>(hiddenCount); //Chainer Linear <Real> cLinear1 = new Linear <Real>(inputCount, hiddenCount, false, w1, b1); NChainer.LSTM <Real> cLstm = new NChainer.LSTM <Real>(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit); Real[,] w2 = Initializer.GetRandomValues <Real[, ]>(outputCount, hiddenCount); Real[] b2 = Initializer.GetRandomValues <Real[]>(outputCount); Linear <Real> cLinear2 = new Linear <Real>(hiddenCount, outputCount, false, w2, b2); Variable <Real> cX1 = new Variable <Real>(input); Variable <Real> cY11 = cLinear1.Forward(cX1); Variable <Real> cY12 = cLstm.Forward(cY11); Variable <Real> cY13 = cLinear2.Forward(cY12); Variable <Real> cT = new Variable <Real>(teach); Variable <Real> cLoss = new NChainer.MeanSquaredError <Real>().Forward(cY13, cT); cLoss.Backward(); //KelpNet CL.Linear <Real> linear1 = new CL.Linear <Real>(inputCount, hiddenCount, false, w1, b1); LSTM <Real> lstm = new LSTM <Real>(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit); CL.Linear <Real> linear2 = new CL.Linear <Real>(hiddenCount, outputCount, false, w2, b2); NdArray <Real> x1 = new NdArray <Real>(input, asBatch: true); NdArray <Real> y11 = linear1.Forward(x1)[0]; NdArray <Real> y12 = lstm.Forward(y11)[0]; NdArray <Real> y13 = linear2.Forward(y12)[0]; NdArray <Real> t = new NdArray <Real>(teach, asBatch: true); NdArray <Real> loss = new MeanSquaredError <Real>().Evaluate(y13, t); y13.Backward(); Real[] cY11data = ((Real[, ])cY11.Data).Flatten(); Real[] cY12data = ((Real[, ])cY12.Data).Flatten(); Real[] cY13data = ((Real[, ])cY13.Data).Flatten(); Real[] cXgrad = ((Real[, ])cX1.Grad).Flatten(); Real[] cupwardWGrad = ((Real[, ])cLstm.upward.W.Grad).Flatten(); Real[] cupwardbGrad = (Real[])cLstm.upward.b.Grad; //許容範囲を設定 Real delta = 0.00001f; //y11 Assert.AreEqual(cY11data.Length, y11.Data.Length); for (int i = 0; i < cY11data.Length; i++) { Assert.AreEqual(cY11data[i], y11.Data[i], delta); } //y12 Assert.AreEqual(cY12data.Length, y12.Data.Length); for (int i = 0; i < cY12data.Length; i++) { Assert.AreEqual(cY12data[i], y12.Data[i], delta); } //y13 Assert.AreEqual(cY13data.Length, y13.Data.Length); for (int i = 0; i < cY13data.Length; i++) { Assert.AreEqual(cY13data[i], y13.Data[i], delta); } //許容範囲を設定 delta = 0.0001f; //loss Assert.AreEqual(cLoss.Data[0], loss.Data[0], delta); //x.Grad Assert.AreEqual(cXgrad.Length, x1.Grad.Length); for (int i = 0; i < cXgrad.Length; i++) { Assert.AreEqual(cXgrad[i], x1.Grad[i], delta); } Real[] cWgrad11 = ((Real[, ])cLinear1.W.Grad).Flatten(); Real[] cbgrad11 = (Real[])cLinear1.b.Grad; //W.grad Assert.AreEqual(cWgrad11.Length, linear1.Weight.Grad.Length); for (int i = 0; i < linear1.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad11[i], linear1.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad11.Length, linear1.Bias.Grad.Length); for (int i = 0; i < linear1.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad11[i], linear1.Bias.Grad[i], delta); } Real[] cWgrad12 = ((Real[, ])cLinear2.W.Grad).Flatten(); Real[] cbgrad12 = (Real[])cLinear2.b.Grad; //W.grad Assert.AreEqual(cWgrad12.Length, linear2.Weight.Grad.Length); for (int i = 0; i < linear2.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad12[i], linear2.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad12.Length, linear2.Bias.Grad.Length); for (int i = 0; i < linear2.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad12[i], linear2.Bias.Grad[i], delta); } //W.grad int wLen = lstm.upward.Weight.Grad.Length; Assert.AreEqual(cupwardWGrad.Length, lstm.upward.Weight.Grad.Length); for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad int bLen = lstm.upward.Bias.Length; Assert.AreEqual(cupwardbGrad.Length, lstm.upward.Bias.Grad.Length); for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } //2周目 Variable <Real> cX2 = new Variable <Real>(input2); Variable <Real> cY21 = cLinear1.Forward(cX2); Variable <Real> cY22 = cLstm.Forward(cY21); Variable <Real> cY23 = cLinear2.Forward(cY22); Variable <Real> cT2 = new Variable <Real>(teach2); Variable <Real> cLoss2 = new NChainer.MeanSquaredError <Real>().Forward(cY23, cT2); //KelpNet NdArray <Real> x2 = new NdArray <Real>(input2, asBatch: true); NdArray <Real> y21 = linear1.Forward(x2)[0]; NdArray <Real> y22 = lstm.Forward(y21)[0]; NdArray <Real> y23 = linear2.Forward(y22)[0]; NdArray <Real> t2 = new NdArray <Real>(teach2, asBatch: true); NdArray <Real> loss2 = new MeanSquaredError <Real>().Evaluate(y23, t2); Assert.AreEqual(cLoss2.Data[0], loss2.Data[0], delta); //Backwardを実行 cLoss2.Backward(); y23.Backward(); Real[] cYdata21 = ((Real[, ])cY21.Data).Flatten(); Real[] cYdata22 = ((Real[, ])cY22.Data).Flatten(); Real[] cYdata23 = ((Real[, ])cY23.Data).Flatten(); Real[] cXgrad2 = ((Real[, ])cX2.Grad).Flatten(); Real[] cupwardWGrad2 = ((Real[, ])cLstm.upward.W.Grad).Flatten(); Real[] cupwardbGrad2 = (Real[])cLstm.upward.b.Grad; Real[] clateralWGrad = ((Real[, ])cLstm.lateral.W.Grad).Flatten(); //y21 Assert.AreEqual(cYdata21.Length, y21.Data.Length); for (int i = 0; i < cYdata21.Length; i++) { Assert.AreEqual(cYdata21[i], y21.Data[i], delta); } //y22 Assert.AreEqual(cYdata22.Length, y22.Data.Length); for (int i = 0; i < cYdata22.Length; i++) { Assert.AreEqual(cYdata22[i], y22.Data[i], delta); } //y23 Assert.AreEqual(cYdata23.Length, y23.Data.Length); for (int i = 0; i < cYdata23.Length; i++) { Assert.AreEqual(cYdata23[i], y23.Data[i], delta); } //x.Grad Assert.AreEqual(cXgrad2.Length, x2.Grad.Length); for (int i = 0; i < cXgrad2.Length; i++) { Assert.AreEqual(cXgrad2[i], x2.Grad[i], delta); } //経由が多くかなり誤差が大きい為 delta = 1.0f; Real[] cWgrad22 = ((Real[, ])cLinear2.W.Grad).Flatten(); Real[] cbgrad22 = (Real[])cLinear2.b.Grad; //W.grad Assert.AreEqual(cWgrad22.Length, linear2.Weight.Grad.Length); for (int i = 0; i < linear2.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad22[i], linear2.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad22.Length, linear2.Bias.Grad.Length); for (int i = 0; i < linear2.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad22[i], linear2.Bias.Grad[i], delta); } delta = 2.0f; //W.grad Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length); for (int i = 0; i < clateralWGrad.Length; i++) { Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta); } for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad2[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad2[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } delta = 20.0f; Real[] cWgrad21 = ((Real[, ])cLinear1.W.Grad).Flatten(); Real[] cbgrad21 = (Real[])cLinear1.b.Grad; //W.grad Assert.AreEqual(cWgrad21.Length, linear1.Weight.Grad.Length); for (int i = 0; i < linear1.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad21[i], linear1.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad21.Length, linear1.Bias.Grad.Length); for (int i = 0; i < linear1.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad21[i], linear1.Bias.Grad[i], delta); } }