static void YCalc(Real[] di, Real[,] line, Real[] v, int n, int k)
{
double sum = 0;
int j, m;
for (int i = 0; i < n; i++)
{
j = i - k;
if (j < 0)
{
m = k - i;
j = 0;
}
else
{
m = 0;
}
for (; m < k; j++, m++)
{
sum += line[i, m] * v[j];
}
v[i] = (v[i] - sum) / di[i];
sum = 0;
}
}
static void XCalc(Real[] di, Real[,] line, Real[] v, int n, int k)
{
double sum = 0;
int j, m, q, t, h;
h = -1;
t = k + 1;
for (int i = n - 1; i >= 0; i--)
{
j = n - i;
if (n - k > j - 2) // -2 или k-1
{
t--;
q = n - 1;
}
else
{
h--;
q = n + h;;
}
for (m = t; m < k; j--, m++, q--)
{
sum += line[q, m] * v[q];
}
v[i] = (v[i] - sum) / di[i];
sum = 0;
}
}
public LSTM(int inSize, int outSize, Real[,] initialUpwardW = null, Real[] initialUpwardb = null, Real[,] initialLateralW = null, string name = FUNCTION_NAME, string[] inputNames = null, string[] outputNames = null, bool gpuEnable = false) : base(name, inputNames, outputNames)
{
this.InputCount = inSize;
this.OutputCount = outSize;
List <NdArray> functionParameters = new List <NdArray>();
this.upward0 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward0", gpuEnable: gpuEnable);
this.upward1 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward1", gpuEnable: gpuEnable);
this.upward2 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward2", gpuEnable: gpuEnable);
this.upward3 = new Linear(inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward3", gpuEnable: gpuEnable);
functionParameters.AddRange(this.upward0.Parameters);
functionParameters.AddRange(this.upward1.Parameters);
functionParameters.AddRange(this.upward2.Parameters);
functionParameters.AddRange(this.upward3.Parameters);
//lateralはBiasは無し
this.lateral0 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral0", gpuEnable: gpuEnable);
this.lateral1 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral1", gpuEnable: gpuEnable);
this.lateral2 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral2", gpuEnable: gpuEnable);
this.lateral3 = new Linear(outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral3", gpuEnable: gpuEnable);
functionParameters.AddRange(this.lateral0.Parameters);
functionParameters.AddRange(this.lateral1.Parameters);
functionParameters.AddRange(this.lateral2.Parameters);
functionParameters.AddRange(this.lateral3.Parameters);
this.Parameters = functionParameters.ToArray();
SingleInputForward = ForwardCpu;
SingleOutputBackward = BackwardCpu;
}
static void LCalc(Real[] di, Real[,] line, Real[] v, int n, int k)
{
double sum = 0, diSum = 0;
int q = 0;
int[] R = new int[n];
R[0] = k;
for (int i = 1; i < k; i++)
{
R[i] = R[i - 1] - 1;
}
for (int i = 0; i < n; i++)
{
if (i > k)
{
q++;
}
for (int j = R[i]; j < k; j++)
{
for (int t = 0; t < j - R[i]; t++)
{
sum += line[i, R[i] + t] * line[j - R[i] + q, R[j - R[i]] + t];
}
line[i, j] = (line[i, j] - sum) / di[j - R[i] + q];
sum = 0;
diSum += line[i, j] * line[i, j];
}
di[i] = Math.Sqrt(di[i] - diSum);
diSum = 0;
}
}
public void EmbedIDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 30);
int outputCount = Mother.Dice.Next(1, 30);
int batchCount = Mother.Dice.Next(1, 5);
int[,] input = (int[, ])Enumerable.Repeat(0, batchCount * inputCount).ToNdArray(batchCount, inputCount);
input[0, 0] = 1;
Real[,,] dummyGy = Initializer.GetRandomValues <Real[, , ]>(batchCount, inputCount, outputCount);
Real[,] w = Initializer.GetRandomValues <Real[, ]>(inputCount, outputCount);
//Chainer
NChainer.EmbedID <Real> cEmbedId = new NChainer.EmbedID <Real>(inputCount, outputCount, w);
Variable <int> cX = new Variable <int>(input);
Variable <Real> cY = cEmbedId.Forward(cX);
cY.Grad = dummyGy;
cY.Backward();
//KelpNet
EmbedID <Real> embedId = new EmbedID <Real>(inputCount, outputCount, w);
NdArray <Real> x = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y = embedId.Forward(x)[0];
y.Grad = dummyGy.Flatten();
y.Backward();
Real[] cYdata = ((Real[, , ])cY.Data).Flatten();
Real[] cWgrad = ((Real[, ])cEmbedId.W.Grad).Flatten();
//許容範囲を算出
Real delta = 0.00001f;
//y
Assert.AreEqual(cYdata.Length, y.Data.Length);
for (int i = 0; i < y.Data.Length; i++)
{
Assert.AreEqual(cYdata[i], y.Data[i], delta);
}
//W.grad
Assert.AreEqual(cWgrad.Length, embedId.Weight.Grad.Length);
for (int i = 0; i < embedId.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad[i], embedId.Weight.Grad[i], delta);
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Initializes a new instance of the KelpNet.Functions.Connections.LSTM class.
/// </summary>
///
/// <param name="inSize"> Size of the in. </param>
/// <param name="outSize"> Size of the out. </param>
/// <param name="initialUpwardW"> (Optional) The initial upward w. </param>
/// <param name="initialUpwardb"> (Optional) The initial upwardb. </param>
/// <param name="initialLateralW"> (Optional) The initial lateral w. </param>
/// <param name="name"> (Optional) The name. </param>
/// <param name="inputNames"> (Optional) List of names of the inputs. </param>
/// <param name="outputNames"> (Optional) List of names of the outputs. </param>
/// <param name="gpuEnable"> (Optional) True if GPU enable. </param>
////////////////////////////////////////////////////////////////////////////////////////////////////
public LSTM(bool verbose, int inSize, int outSize, [CanBeNull] Real[,] initialUpwardW = null, [CanBeNull] Real[] initialUpwardb = null, [CanBeNull] Real[,] initialLateralW = null, [CanBeNull] string name = FUNCTION_NAME, [CanBeNull] string[] inputNames = null, [CanBeNull] string[] outputNames = null, bool gpuEnable = false) : base(name, inputNames, outputNames)
{
Verbose = verbose;
InputCount = inSize;
OutputCount = outSize;
List <NdArray> functionParameters = new List <NdArray>();
upward0 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward0", gpuEnable: gpuEnable);
upward1 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward1", gpuEnable: gpuEnable);
upward2 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward2", gpuEnable: gpuEnable);
upward3 = new Linear(verbose, inSize, outSize, noBias: false, initialW: initialUpwardW, initialb: initialUpwardb, name: "upward3", gpuEnable: gpuEnable);
functionParameters.AddRange(upward0.Parameters);
functionParameters.AddRange(upward1.Parameters);
functionParameters.AddRange(upward2.Parameters);
functionParameters.AddRange(upward3.Parameters);
// lateral does not have Bias
lateral0 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral0", gpuEnable: gpuEnable);
lateral1 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral1", gpuEnable: gpuEnable);
lateral2 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral2", gpuEnable: gpuEnable);
lateral3 = new Linear(verbose, outSize, outSize, noBias: true, initialW: initialLateralW, name: "lateral3", gpuEnable: gpuEnable);
functionParameters.AddRange(lateral0.Parameters);
functionParameters.AddRange(lateral1.Parameters);
functionParameters.AddRange(lateral2.Parameters);
functionParameters.AddRange(lateral3.Parameters);
Parameters = functionParameters.ToArray();
SingleInputForward = ForwardCpu;
SingleOutputBackward = BackwardCpu;
}
public void EmbedIDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 30);
int outputCount = Mother.Dice.Next(1, 30);
int batchCount = Mother.Dice.Next(1, 5);
int[,] input = (int[, ])Enumerable.Repeat(0, batchCount * inputCount).ToNdArray(batchCount, inputCount);
input[0, 0] = 1;
Real[,,] dummyGy = (Real[, , ])Initializer.GetRealNdArray(new[] { batchCount, inputCount, outputCount });
Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { inputCount, outputCount });
//Chainer
NChainer.EmbedID <Real> cEmbedId = new NChainer.EmbedID <Real>(inputCount, outputCount, Real.ToBaseNdArray(w));
Variable <int> cX = new Variable <int>(input);
Variable <Real> cY = cEmbedId.Forward(cX);
cY.Grad = Real.ToBaseNdArray(dummyGy);
cY.Backward();
//KelpNet
KelpNet.EmbedID embedId = new KelpNet.EmbedID(inputCount, outputCount, w);
NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount);
NdArray y = embedId.Forward(x)[0];
y.Grad = Real.ToRealArray(dummyGy);
y.Backward();
Real[] cYdata = Real.ToRealArray((Real[, , ])cY.Data);
Real[] cWgrad = Real.ToRealArray((Real[, ])cEmbedId.W.Grad);
//許容範囲を算出
double delta = 0.00001;
//y
Assert.AreEqual(cYdata.Length, y.Data.Length);
for (int i = 0; i < y.Data.Length; i++)
{
Assert.AreEqual(cYdata[i], y.Data[i], delta);
}
//W.grad
Assert.AreEqual(cWgrad.Length, embedId.Weight.Grad.Length);
for (int i = 0; i < embedId.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad[i], embedId.Weight.Grad[i], delta);
}
}
protected SquareMatrix(Real[,] squareMatrix)
: base(squareMatrix)
{
if (squareMatrix.GetLength(0) != squareMatrix.GetLength(1))
{
throw new ArgumentException("Parameter name: squareMatrix", "squareMatrix");
}
}
/// <summary>
/// Scalar multiplication with a complex number
/// </summary>
/// <param name="left"></param>
/// <param name="right"></param>
/// <returns></returns>
public static RealMatrix operator *(RealMatrix left, Real right)
{
Real[,] indices = left.Indices;
for (int i = 0; i < left.Height; i++)
{
for (int j = 0; j < left.Width; j++)
{
indices[i, j] *= right;
}
}
return(new RealMatrix(indices));
}
static Real[] multiple(Real[] di, Real[,] line, Real[] v, int n, int k, Real[] res)
{
for (int i = 0; i < k; i++)
{
for (int j = 0, m = k - i; j < n; j++)
{
res[j] += line[j, i] * v[j];
if (j < n - m)
{
res[j] += line[m, i] * v[j];
}
}
}
for (int i = 0; i < n; i++)
{
res[i] += di[i] * v[i];
}
return(res);
}
public EmbedID(int inputCount, int outputCount, Real[,] initialW = null, string name = FUNCTION_NAME, string[] inputNames = null, string[] outputNames = null) : base(name, inputNames, outputNames)
{
this.InputCount = inputCount;
this.OutputCount = outputCount;
this.Weight = new NdArray(inputCount, outputCount);
this.Weight.Name = this.Name + " Weight";
if (initialW == null)
{
Initializer.InitWeight(this.Weight);
}
else
{
//単純に代入しないのはサイズのチェックを兼ねるため
this.Weight.Data = Real.ToRealArray(initialW);
}
this.Parameters = new[] { this.Weight };
}
static Real[] multipleN(Real[] di, Real[,] line, Real[] v, int n, int k, Real[] res)
{
int m, j = 0;
for (int i = 0; i < n; i++)
{
res[i] += di[i] * v[i];
m = i - k;
if (m < 0)
{
continue;
}
for (; j < k; j++, m++)
{
res[i] += line[i, j] * v[m];
res[m] += line[m, j] * v[i];
}
j = 0;
}
return(res);
}
public EmbedID(int inputCount, int outputCount, Real[,] initialW = null, string name = FUNCTION_NAME, string[] inputNames = null, string[] outputNames = null) : base(name, inputNames, outputNames)
{
this.InputCount = inputCount;
this.OutputCount = outputCount;
this.Weight = new NdArray(inputCount, outputCount);
this.Weight.Name = this.Name + " Weight";
if (initialW == null)
{
Initializer.InitWeight(this.Weight);
}
else
{
//Do not simply substitute to check the size
this.Weight.Data = Real.GetArray(initialW);
}
this.Parameters = new[] { this.Weight };
SingleInputForward = NeedPreviousForwardCpu;
SingleOutputBackward = NeedPreviousBackwardCpu;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Initializes a new instance of the KelpNet.Functions.Connections.EmbedID class.
/// </summary>
///
/// <param name="inputCount"> Number of inputs. </param>
/// <param name="outputCount"> Number of outputs. </param>
/// <param name="initialW"> (Optional) The initial w. </param>
/// <param name="name"> (Optional) The name. </param>
/// <param name="inputNames"> (Optional) List of names of the inputs. </param>
/// <param name="outputNames"> (Optional) List of names of the outputs. </param>
////////////////////////////////////////////////////////////////////////////////////////////////////
public EmbedID(int inputCount, int outputCount, [CanBeNull] Real[,] initialW = null, [CanBeNull] string name = FUNCTION_NAME, [CanBeNull] string[] inputNames = null, [CanBeNull] string[] outputNames = null) : base(name, inputNames, outputNames)
{
InputCount = inputCount;
OutputCount = outputCount;
Weight = new NdArray(inputCount, outputCount);
Weight.Name = Name + " Weight";
if (initialW == null)
{
Initializer.InitWeight(Weight);
}
else
{
// Do not simply substitute for checking the size
Weight.Data = Real.GetArray(initialW);
}
Parameters = new[] { Weight };
SingleInputForward = NeedPreviousForwardCpu;
SingleOutputBackward = NeedPreviousBackwardCpu;
}
public void SGDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, inputCount });
Real[,] dummyGy = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, outputCount });
Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, inputCount });
Real[] b = Initializer.GetRealArray(outputCount);
//Chainer
NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b));
NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>();
cSgd.Setup(cLinear);
Variable <Real> cX = new Variable <Real>(Real.ToBaseNdArray(input));
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = Real.ToBaseNdArray(dummyGy);
cY.Backward();
cSgd.Update();
//KelpNet
KelpNet.Linear linear = new KelpNet.Linear(inputCount, outputCount, false, w, b);
KelpNet.SGD sgd = new SGD();
sgd.SetUp(linear);
NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount);
NdArray y = linear.Forward(x)[0];
y.Grad = Real.ToRealArray(dummyGy);
y.Backward();
sgd.Update();
Real[] cW = Real.ToRealArray((Real[, ])cLinear.W.Data);
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
double delta = 0.00001;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
protected Matrix(Real[,] matrix)
{
RowCount = (byte)matrix.GetLength(0);
ColumnCount = (byte)matrix.GetLength(1);
this.matrix = matrix;
}
public void SGDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount);
Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount);
Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount);
Real[] b = Initializer.GetRandomValues <Real[]>(outputCount);
//Chainer
Linear <Real> cLinear = new Linear <Real>(inputCount, outputCount, false, w, b);
NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>();
cSgd.Setup(cLinear);
Variable <Real> cX = new Variable <Real>(input);
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = dummyGy;
cY.Backward();
cSgd.Update();
//KelpNet
CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b);
KelpNet.SGD <Real> sgd = new SGD <Real>();
sgd.SetUp(linear);
NdArray <Real> x = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y = linear.Forward(x)[0];
y.Grad = dummyGy.Flatten();
y.Backward();
sgd.Update();
Real[] cW = ((Real[, ])cLinear.W.Data).Flatten();
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
Real delta = 0.00001f;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
public static void Run()
{
// Describe each initial value
Real[,,,] initial_W1 =
{
{ { { 1.0, 0.5, 0.0 }, { 0.5, 0.0, -0.5 }, { 0.0, -0.5, -1.0 } } },
{ { { 0.0, -0.1, 0.1 }, { -0.3, 0.4, 0.7 }, { 0.5, -0.2, 0.2 } } }
};
Real[] initial_b1 = { 0.5, 1.0 };
Real[,,,] initial_W2 =
{
{ { { -0.1, 0.6 }, { 0.3, -0.9 } }, { { 0.7, 0.9 }, { -0.2, -0.3 } } },
{ { { -0.6, -0.1 }, { 0.3, 0.3 } }, { { -0.5, 0.8 }, { 0.9, 0.1 } } }
};
Real[] initial_b2 = { 0.1, 0.9 };
Real[,] initial_W3 =
{
{ 0.5, 0.3, 0.4, 0.2, 0.6, 0.1, 0.4, 0.3 },
{ 0.6, 0.4, 0.9, 0.1, 0.5, 0.2, 0.3, 0.4 }
};
Real[] initial_b3 = { 0.01, 0.02 };
Real[,] initial_W4 = { { 0.8, 0.2 }, { 0.4, 0.6 } };
Real[] initial_b4 = { 0.02, 0.01 };
//Input data
NdArray x = new NdArray(new Real[, , ] {
{
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.2, 0.9, 0.2, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.9, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.1, 0.8, 0.5, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.3, 0.3, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.1, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.4, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.8, 0.4, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.1, 0.8, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }
}
});
//teacher signal
Real[] t = { 0.0, 1.0 };
// If you want to check the contents of a layer, have an instance as a single layer
Convolution2D l2 = new Convolution2D(true, 1, 2, 3, initialW: initial_W1, initialb: initial_b1, name: "l2 Conv2D");
// Write the network configuration in FunctionStack
FunctionStack nn = new FunctionStack("Test5",
l2, //new Convolution2D(1, 2, 3, initialW: initial_W1, initialb: initial_b1),
new ReLU(name: "l2 ReLU"),
//new AveragePooling(2, 2, name: "l2 AVGPooling"),
new MaxPooling(2, 2, name: "l2 MaxPooling"),
new Convolution2D(true, 2, 2, 2, initialW: initial_W2, initialb: initial_b2, name: "l3 Conv2D"),
new ReLU(name: "l3 ReLU"),
//new AveragePooling(2, 2, name: "l3 AVGPooling"),
new MaxPooling(2, 2, name: "l3 MaxPooling"),
new Linear(true, 8, 2, initialW: initial_W3, initialb: initial_b3, name: "l4 Linear"),
new ReLU(name: "l4 ReLU"),
new Linear(true, 2, 2, initialW: initial_W4, initialb: initial_b4, name: "l5 Linear")
);
// If you omit the optimizer declaration, the default SGD(0.1) will be used
// nn.SetOptimizer(new SGD());
// Training conducted
Trainer.Train(nn, x, t, new MeanSquaredError(), false);
// If Update is executed, grad is consumed, so output the value first
RILogManager.Default?.SendDebug("gw1");
RILogManager.Default?.SendDebug(l2.Weight.ToString("Grad"));
RILogManager.Default?.SendDebug("gb1");
RILogManager.Default?.SendDebug(l2.Bias.ToString("Grad"));
//update
nn.Update();
RILogManager.Default?.SendDebug("w1");
RILogManager.Default?.SendDebug(l2.Weight.ToString());
RILogManager.Default?.SendDebug("b1");
RILogManager.Default?.SendDebug(l2.Bias.ToString());
}
public void RandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = 1 + Mother.Dice.Next() % 49;
int outputCount = 1 + Mother.Dice.Next() % 49;
Real[,] input = (Real[, ])Initializer.GetRealNdArray(new[] { 1, inputCount });
Real[,] dummyGy = (Real[, ])Initializer.GetRealNdArray(new[] { 1, outputCount });
Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, inputCount });
Real[] b = Initializer.GetRealArray(outputCount);
//Chainer
NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b));
Variable <Real> cX = new Variable <Real>(Real.ToBaseNdArray(input));
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = Real.ToBaseNdArray(dummyGy);
cY.Backward();
//KelpNet
KelpNet.Linear linear = new KelpNet.Linear(inputCount, outputCount, false, w, b);
NdArray x = new NdArray(input);
NdArray y = linear.Forward(x)[0];
y.Grad = Real.ToRealArray(dummyGy);
y.Backward();
Real[] cYdata = Real.ToRealArray((Real[, ])cY.Data);
Real[] cXgrad = Real.ToRealArray((Real[, ])cX.Grad);
Real[] cWgrad = Real.ToRealArray((Real[, ])cLinear.W.Grad);
Real[] cbgrad = (Real[])cLinear.b.Grad;
//許容範囲を算出
double delta = 0.00001;
//y
Assert.AreEqual(cYdata.Length, y.Data.Length);
for (int i = 0; i < y.Data.Length; i++)
{
Assert.AreEqual(cYdata[i], y.Data[i], delta);
}
//x.grad
Assert.AreEqual(cXgrad.Length, x.Grad.Length);
for (int i = 0; i < x.Grad.Length; i++)
{
Assert.AreEqual(cXgrad[i], x.Grad[i], delta);
}
//W.grad
Assert.AreEqual(cWgrad.Length, linear.Weight.Grad.Length);
for (int i = 0; i < linear.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad[i], linear.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad.Length, linear.Bias.Grad.Length);
for (int i = 0; i < linear.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad[i], linear.Bias.Grad[i], delta);
}
}
public void RandomTest(bool gpuEnable)
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(1, 50);
int outputCount = Mother.Dice.Next(1, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount);
Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount);
Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount);
Real[] b = Initializer.GetRandomValues <Real[]>(outputCount);
//Chainer
NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, w, b);
Variable <Real> cX = new Variable <Real>(input);
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = dummyGy;
cY.Backward();
//KelpNet
CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b, gpuEnable: gpuEnable);
NdArray <Real> x = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y = linear.Forward(x)[0];
y.Grad = dummyGy.Flatten();
y.Backward();
Real[] cYdata = ((Real[, ])cY.Data).Flatten();
Real[] cXgrad = ((Real[, ])cX.Grad).Flatten();
Real[] cWgrad = ((Real[, ])cLinear.W.Grad).Flatten();
Real[] cbgrad = (Real[])cLinear.b.Grad;
//許容範囲を算出
Real delta = 0.00001f;
//y
Assert.AreEqual(cYdata.Length, y.Data.Length);
for (int i = 0; i < y.Data.Length; i++)
{
Assert.AreEqual(cYdata[i], y.Data[i], delta);
}
//x.grad
Assert.AreEqual(cXgrad.Length, x.Grad.Length);
for (int i = 0; i < x.Grad.Length; i++)
{
Assert.AreEqual(cXgrad[i], x.Grad[i], delta);
}
//W.grad
Assert.AreEqual(cWgrad.Length, linear.Weight.Grad.Length);
for (int i = 0; i < linear.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad[i], linear.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad.Length, linear.Bias.Grad.Length);
for (int i = 0; i < linear.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad[i], linear.Bias.Grad[i], delta);
}
}
public static void Run()
{
//Align Weight before and after splitting
Real[,] testWeightValues =
{
{ -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 },
{ -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 },
{ -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 },
{ -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
};
Real[][,] testJaggWeightValues =
{
new Real[, ] {
{ -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }
},
new Real[, ] {
{ -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }
},
new Real[, ] {
{ -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }
},
new Real[, ] {
{ -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
}
};
Linear l0 = new Linear(true, 5, 20, initialW: testWeightValues, name: "l0");
Linear l1 = new Linear(true, 5, 5, initialW: testJaggWeightValues[0], name: "l1");
Linear l2 = new Linear(true, 5, 5, initialW: testJaggWeightValues[1], name: "l2");
Linear l3 = new Linear(true, 5, 5, initialW: testJaggWeightValues[2], name: "l3");
Linear l4 = new Linear(true, 5, 5, initialW: testJaggWeightValues[3], name: "l4");
l0.SetOptimizer(new SGD());
SGD sgd = new SGD();
l1.SetOptimizer(sgd);
l2.SetOptimizer(sgd);
l3.SetOptimizer(sgd);
l4.SetOptimizer(sgd);
//Input is equivalent, but Grad is added and it is divided
Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 };
NdArray testInputValuesA = new NdArray(testValue);
NdArray testInputValuesB = new NdArray(testValue);
RILogManager.Default?.SendDebug("l0 for");
NdArray[] l0Result = l0.Forward(true, testInputValuesA);
RILogManager.Default?.SendDebug(l0Result.ToString());
RILogManager.Default?.SendDebug("l1 for");
NdArray[] l1Result = l1.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l1Result.ToString());
RILogManager.Default?.SendDebug("l2 for");
NdArray[] l2Result = l2.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l2Result.ToString());
RILogManager.Default?.SendDebug("l3 for");
NdArray[] l3Result = l3.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l3Result.ToString());
RILogManager.Default?.SendDebug("l4 for");
NdArray[] l4Result = l4.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l4Result.ToString());
//Create an appropriate Grad value
l0Result[0].Grad = new Real[]
{
-2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03,
1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04,
-4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04,
-4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03
};
l1Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 };
l2Result[0].Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 };
l3Result[0].Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 };
l4Result[0].Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 };
//Backward
l0.Backward(true, l0Result);
l1.Backward(true, l1Result);
l2.Backward(true, l2Result);
l3.Backward(true, l3Result);
l4.Backward(true, l4Result);
RILogManager.Default?.SendDebug("l0 back");
RILogManager.Default?.SendDebug(testInputValuesA.ToString("Grad"));
RILogManager.Default?.SendDebug("l1-l4 sum back");
RILogManager.Default?.SendDebug(testInputValuesB.ToString("Grad"));
l0.Update(); //Although the format is irregular, since 10 contains SGD
sgd.Update(); // Use stochastic gradient descent as the optimizer
RILogManager.Default?.SendDebug("l0 Weight");
RILogManager.Default?.SendDebug(l0.Weight.ToString());
RILogManager.Default?.SendDebug("l1 Weight");
RILogManager.Default?.SendDebug(l1.Weight.ToString());
RILogManager.Default?.SendDebug("l0 Bias");
RILogManager.Default?.SendDebug(l0.Bias.ToString());
RILogManager.Default?.SendDebug("l1 Bias");
RILogManager.Default?.SendDebug(l1.Bias.ToString());
}
public void SwishRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int ioCount = Mother.Dice.Next(1, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, ioCount });
Real[,] dummyGy = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, ioCount });
Real beta = Mother.Dice.NextDouble();
//Chainer
NChainer.Swish<Real> cSwish = new NChainer.Swish<Real>(new[] { ioCount }, beta);
Variable<Real> cX = new Variable<Real>(Real.ToBaseNdArray(input));
Variable<Real> cY = cSwish.Forward(cX);
cY.Grad = Real.ToBaseNdArray(dummyGy);
cY.Backward();
//KelpNet
KelpNet.Swish swish = new KelpNet.Swish(new[] { ioCount }, beta);
NdArray x = new NdArray(Real.ToRealArray(input), new[] { ioCount }, batchCount);
NdArray y = swish.Forward(x)[0];
y.Grad = Real.ToRealArray(dummyGy);
y.Backward();
Real[] cYdata = Real.ToRealArray((Real[,])cY.Data);
Real[] cXgrad = Real.ToRealArray((Real[,])cX.Grad);
Real[] cbgrad = (Real[])cSwish.beta.Grad;
//許容範囲を算出
double delta = 0.00001;
//y
Assert.AreEqual(cYdata.Length, y.Data.Length);
for (int i = 0; i < y.Data.Length; i++)
{
Assert.AreEqual(cYdata[i], y.Data[i], delta);
}
//x.grad
Assert.AreEqual(cXgrad.Length, x.Grad.Length);
for (int i = 0; i < x.Grad.Length; i++)
{
Assert.AreEqual(cXgrad[i], x.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad.Length, swish.Beta.Grad.Length);
for (int i = 0; i < swish.Beta.Grad.Length; i++)
{
Assert.AreEqual(cbgrad[i], swish.Beta.Grad[i], delta);
}
}
public void LSTMRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int batchCount = Mother.Dice.Next(1, 5);
int inputCount = Mother.Dice.Next(1, 5);
int outputCount = Mother.Dice.Next(1, 5);
Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount);
Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount);
Real[,] upwardInit = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount);
Real[,] lateralInit = Initializer.GetRandomValues <Real[, ]>(outputCount, outputCount);
Real[,,] biasInit = Initializer.GetRandomValues <Real[, , ]>(1, outputCount, 1);
Real[,,] forgetBiasInit = Initializer.GetRandomValues <Real[, , ]>(1, outputCount, 1);
//Chainer
NChainer.LSTM <Real> clstm = new NChainer.LSTM <Real>(inputCount, outputCount, lateralInit, upwardInit, biasInit, forgetBiasInit);
Variable <Real> cX = new Variable <Real>(input);
Variable <Real> cY = clstm.Forward(cX);
cY.Grad = dummyGy;
cY.Backward();
//KelpNet
LSTM <Real> lstm = new LSTM <Real>(inputCount, outputCount, lateralInit, upwardInit, biasInit, forgetBiasInit);
NdArray <Real> x = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y = lstm.Forward(x)[0];
y.Grad = dummyGy.Flatten();
y.Backward();
//許容範囲を算出
Real delta = 0.00001f;
Real[] cYdata = ((Real[, ])cY.Data).Flatten();
Real[] cXgrad = ((Real[, ])cX.Grad).Flatten();
Real[] cupwardWGrad = ((Real[, ])clstm.upward.W.Grad).Flatten();
Real[] cupwardbGrad = (Real[])clstm.upward.b.Grad;
//y
Assert.AreEqual(cYdata.Length, y.Data.Length);
for (int i = 0; i < cYdata.Length; i++)
{
Assert.AreEqual(cYdata[i], y.Data[i], delta);
}
//x.Grad
Assert.AreEqual(cXgrad.Length, x.Grad.Length);
for (int i = 0; i < cXgrad.Length; i++)
{
Assert.AreEqual(cXgrad[i], x.Grad[i], delta);
}
//W.grad
int wLen = lstm.upward.Weight.Grad.Length;
Assert.AreEqual(cupwardWGrad.Length, lstm.upward.Weight.Grad.Length);
for (int i = 0; i < wLen; i++)
{
Assert.AreEqual(cupwardWGrad[i + wLen * 0], lstm.upward.Weight.Grad[i], delta);
}
//b.grad
int bLen = lstm.upward.Bias.Length;
Assert.AreEqual(cupwardbGrad.Length, lstm.upward.Bias.Grad.Length);
for (int i = 0; i < bLen; i++)
{
Assert.AreEqual(cupwardbGrad[i + wLen * 0], lstm.upward.Bias.Grad[i], delta);
}
//////////////////////////////////////////////////////////////////////////////////////////
// 1度目はlateralに値はない //
//////////////////////////////////////////////////////////////////////////////////////////
//Real[] clateralWGrad = Real.ToRealArray((Real[,])clstm.lateral.W.Grad);
//Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length);
//for (int i = 0; i < wLen; i++)
//{
// Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta);
//}
//////////////////////////////////////////////////////////////////////////////////////////
///////////
//２周目 //
///////////
Real[,] input2 = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount);
Real[,] dummyGy2 = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount);
//Chainer
Variable <Real> cX2 = new Variable <Real>(input2);
Variable <Real> cY2 = clstm.Forward(cX2);
cY2.Grad = dummyGy2;
cY2.Backward();
//KelpNet
NdArray <Real> x2 = new NdArray <Real>(input2, asBatch: true);
NdArray <Real> y2 = lstm.Forward(x2)[0];
y2.Grad = dummyGy2.Flatten();
y2.Backward();
Real[] cYdata2 = ((Real[, ])cY2.Data).Flatten();
Real[] cXgrad2 = ((Real[, ])cX2.Grad).Flatten();
Real[] cupwardWGrad2 = ((Real[, ])clstm.upward.W.Grad).Flatten();
Real[] cupwardbGrad2 = (Real[])clstm.upward.b.Grad;
Real[] clateralWGrad = ((Real[, ])clstm.lateral.W.Grad).Flatten();
//y
Assert.AreEqual(cYdata2.Length, y2.Data.Length);
for (int i = 0; i < cYdata2.Length; i++)
{
Assert.AreEqual(cYdata2[i], y2.Data[i], delta);
}
//x.Grad
Assert.AreEqual(cXgrad2.Length, x2.Grad.Length);
for (int i = 0; i < cXgrad2.Length; i++)
{
Assert.AreEqual(cXgrad2[i], x2.Grad[i], delta);
}
//W.grad
Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length);
for (int i = 0; i < clateralWGrad.Length; i++)
{
Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta);
}
//経由が多いため誤差が大きい
delta = 1.0f;
for (int i = 0; i < wLen; i++)
{
Assert.AreEqual(cupwardWGrad2[i + wLen * 0], lstm.upward.Weight.Grad[i], delta);
}
//b.grad
for (int i = 0; i < bLen; i++)
{
Assert.AreEqual(cupwardbGrad2[i + wLen * 0], lstm.upward.Bias.Grad[i], delta);
}
}
public void RnnLSTMRandomTest()
{
Python.Initialize();
Chainer.Initialize();
Real[,] input = { { 1.0 }, { 3.0 }, { 5.0 }, { 7.0 }, { 9.0 } };
Real[,] teach = { { 3.0 }, { 5.0 }, { 7.0 }, { 9.0 }, { 11.0 } };
Real[,] input2 = { { 3.0 }, { 5.0 }, { 7.0 }, { 9.0 }, { 11.0 } };
Real[,] teach2 = { { 5.0 }, { 7.0 }, { 9.0 }, { 11.0 }, { 13.0 } };
int outputCount = 1;
int inputCount = 1;
int hiddenCount = 2;
Real[,] upwardInit = (Real[, ])Initializer.GetRealNdArray(new[] { hiddenCount, hiddenCount });
Real[,] lateralInit = (Real[, ])Initializer.GetRealNdArray(new[] { hiddenCount, hiddenCount });
Real[,,] biasInit = (Real[, , ])Initializer.GetRealNdArray(new[] { 1, hiddenCount, 1 });
Real[,,] forgetBiasInit = (Real[, , ])Initializer.GetRealNdArray(new[] { 1, hiddenCount, 1 });
//Chainer
Real[,] w1 = (Real[, ])Initializer.GetRealNdArray(new[] { hiddenCount, inputCount });
Real[] b1 = Initializer.GetRealArray(hiddenCount);
//Chainer
NChainer.Linear <Real> cLinear1 = new NChainer.Linear <Real>(inputCount, hiddenCount, false, Real.ToBaseNdArray(w1), Real.ToBaseArray(b1));
NChainer.LSTM <Real> cLstm = new NChainer.LSTM <Real>(hiddenCount, hiddenCount, Real.ToBaseNdArray(lateralInit), Real.ToBaseNdArray(upwardInit), Real.ToBaseNdArray(biasInit), Real.ToBaseNdArray(forgetBiasInit));
Real[,] w2 = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, hiddenCount });
Real[] b2 = Initializer.GetRealArray(outputCount);
NChainer.Linear <Real> cLinear2 = new NChainer.Linear <Real>(hiddenCount, outputCount, false, Real.ToBaseNdArray(w2), Real.ToBaseArray(b2));
Variable <Real> cX1 = new Variable <Real>(Real.ToBaseNdArray(input));
Variable <Real> cY11 = cLinear1.Forward(cX1);
Variable <Real> cY12 = cLstm.Forward(cY11);
Variable <Real> cY13 = cLinear2.Forward(cY12);
Variable <Real> cT = new Variable <Real>(Real.ToBaseNdArray(teach));
Variable <Real> cLoss = new NChainer.MeanSquaredError <Real>().Forward(cY13, cT);
cLoss.Backward();
//KelpNet
KelpNet.CL.Linear linear1 = new KelpNet.CL.Linear(inputCount, hiddenCount, false, w1, b1);
KelpNet.LSTM lstm = new KelpNet.LSTM(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit);
KelpNet.CL.Linear linear2 = new KelpNet.CL.Linear(hiddenCount, outputCount, false, w2, b2);
NdArray x1 = new NdArray(Real.ToRealArray(input), new[] { 1 }, 5);
NdArray y11 = linear1.Forward(x1)[0];
NdArray y12 = lstm.Forward(y11)[0];
NdArray y13 = linear2.Forward(y12)[0];
NdArray t = new NdArray(Real.ToRealArray(teach), new[] { 1 }, 5);
NdArray loss = new KelpNet.MeanSquaredError().Evaluate(y13, t);
y13.Backward();
Real[] cY11data = Real.ToRealArray((Real[, ])cY11.Data);
Real[] cY12data = Real.ToRealArray((Real[, ])cY12.Data);
Real[] cY13data = Real.ToRealArray((Real[, ])cY13.Data);
Real[] cXgrad = Real.ToRealArray((Real[, ])cX1.Grad);
Real[] cupwardWGrad = Real.ToRealArray((Real[, ])cLstm.upward.W.Grad);
Real[] cupwardbGrad = (Real[])cLstm.upward.b.Grad;
//許容範囲を設定
double delta = 0.00001;
//y11
Assert.AreEqual(cY11data.Length, y11.Data.Length);
for (int i = 0; i < cY11data.Length; i++)
{
Assert.AreEqual(cY11data[i], y11.Data[i], delta);
}
//y12
Assert.AreEqual(cY12data.Length, y12.Data.Length);
for (int i = 0; i < cY12data.Length; i++)
{
Assert.AreEqual(cY12data[i], y12.Data[i], delta);
}
//y13
Assert.AreEqual(cY13data.Length, y13.Data.Length);
for (int i = 0; i < cY13data.Length; i++)
{
Assert.AreEqual(cY13data[i], y13.Data[i], delta);
}
//許容範囲を設定
delta = 0.0001;
//loss
Assert.AreEqual(cLoss.Data[0], loss.Data[0], delta);
//x.Grad
Assert.AreEqual(cXgrad.Length, x1.Grad.Length);
for (int i = 0; i < cXgrad.Length; i++)
{
Assert.AreEqual(cXgrad[i], x1.Grad[i], delta);
}
Real[] cWgrad11 = Real.ToRealArray((Real[, ])cLinear1.W.Grad);
Real[] cbgrad11 = (Real[])cLinear1.b.Grad;
//W.grad
Assert.AreEqual(cWgrad11.Length, linear1.Weight.Grad.Length);
for (int i = 0; i < linear1.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad11[i], linear1.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad11.Length, linear1.Bias.Grad.Length);
for (int i = 0; i < linear1.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad11[i], linear1.Bias.Grad[i], delta);
}
Real[] cWgrad12 = Real.ToRealArray((Real[, ])cLinear2.W.Grad);
Real[] cbgrad12 = (Real[])cLinear2.b.Grad;
//W.grad
Assert.AreEqual(cWgrad12.Length, linear2.Weight.Grad.Length);
for (int i = 0; i < linear2.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad12[i], linear2.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad12.Length, linear2.Bias.Grad.Length);
for (int i = 0; i < linear2.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad12[i], linear2.Bias.Grad[i], delta);
}
//W.grad
int wLen = lstm.upward.Weight.Grad.Length;
Assert.AreEqual(cupwardWGrad.Length, lstm.upward.Weight.Grad.Length);
for (int i = 0; i < wLen; i++)
{
Assert.AreEqual(cupwardWGrad[i + wLen * 0], lstm.upward.Weight.Grad[i], delta);
}
//b.grad
int bLen = lstm.upward.Bias.Length;
Assert.AreEqual(cupwardbGrad.Length, lstm.upward.Bias.Grad.Length);
for (int i = 0; i < bLen; i++)
{
Assert.AreEqual(cupwardbGrad[i + wLen * 0], lstm.upward.Bias.Grad[i], delta);
}
//２周目
Variable <Real> cX2 = new Variable <Real>(Real.ToBaseNdArray(input2));
Variable <Real> cY21 = cLinear1.Forward(cX2);
Variable <Real> cY22 = cLstm.Forward(cY21);
Variable <Real> cY23 = cLinear2.Forward(cY22);
Variable <Real> cT2 = new Variable <Real>(Real.ToBaseNdArray(teach2));
Variable <Real> cLoss2 = new NChainer.MeanSquaredError <Real>().Forward(cY23, cT2);
//KelpNet
NdArray x2 = new NdArray(Real.ToRealArray(input2), new[] { 1 }, 5);
NdArray y21 = linear1.Forward(x2)[0];
NdArray y22 = lstm.Forward(y21)[0];
NdArray y23 = linear2.Forward(y22)[0];
NdArray t2 = new NdArray(Real.ToRealArray(teach2), new[] { 1 }, 5);
NdArray loss2 = new KelpNet.MeanSquaredError().Evaluate(y23, t2);
Assert.AreEqual(cLoss2.Data[0], loss2.Data[0], delta);
//Backwardを実行
cLoss2.Backward();
y23.Backward();
Real[] cYdata21 = Real.ToRealArray((Real[, ])cY21.Data);
Real[] cYdata22 = Real.ToRealArray((Real[, ])cY22.Data);
Real[] cYdata23 = Real.ToRealArray((Real[, ])cY23.Data);
Real[] cXgrad2 = Real.ToRealArray((Real[, ])cX2.Grad);
Real[] cupwardWGrad2 = Real.ToRealArray((Real[, ])cLstm.upward.W.Grad);
Real[] cupwardbGrad2 = (Real[])cLstm.upward.b.Grad;
Real[] clateralWGrad = Real.ToRealArray((Real[, ])cLstm.lateral.W.Grad);
//y21
Assert.AreEqual(cYdata21.Length, y21.Data.Length);
for (int i = 0; i < cYdata21.Length; i++)
{
Assert.AreEqual(cYdata21[i], y21.Data[i], delta);
}
//y22
Assert.AreEqual(cYdata22.Length, y22.Data.Length);
for (int i = 0; i < cYdata22.Length; i++)
{
Assert.AreEqual(cYdata22[i], y22.Data[i], delta);
}
//y23
Assert.AreEqual(cYdata23.Length, y23.Data.Length);
for (int i = 0; i < cYdata23.Length; i++)
{
Assert.AreEqual(cYdata23[i], y23.Data[i], delta);
}
//x.Grad
Assert.AreEqual(cXgrad2.Length, x2.Grad.Length);
for (int i = 0; i < cXgrad2.Length; i++)
{
Assert.AreEqual(cXgrad2[i], x2.Grad[i], delta);
}
//経由が多くかなり誤差が大きい為
delta = 1.0;
Real[] cWgrad22 = Real.ToRealArray((Real[, ])cLinear2.W.Grad);
Real[] cbgrad22 = (Real[])cLinear2.b.Grad;
//W.grad
Assert.AreEqual(cWgrad22.Length, linear2.Weight.Grad.Length);
for (int i = 0; i < linear2.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad22[i], linear2.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad22.Length, linear2.Bias.Grad.Length);
for (int i = 0; i < linear2.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad22[i], linear2.Bias.Grad[i], delta);
}
delta = 2.0;
//W.grad
Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length);
for (int i = 0; i < clateralWGrad.Length; i++)
{
Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta);
}
for (int i = 0; i < wLen; i++)
{
Assert.AreEqual(cupwardWGrad2[i + wLen * 0], lstm.upward.Weight.Grad[i], delta);
}
//b.grad
for (int i = 0; i < bLen; i++)
{
Assert.AreEqual(cupwardbGrad2[i + wLen * 0], lstm.upward.Bias.Grad[i], delta);
}
delta = 20.0;
Real[] cWgrad21 = Real.ToRealArray((Real[, ])cLinear1.W.Grad);
Real[] cbgrad21 = (Real[])cLinear1.b.Grad;
//W.grad
Assert.AreEqual(cWgrad21.Length, linear1.Weight.Grad.Length);
for (int i = 0; i < linear1.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad21[i], linear1.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad21.Length, linear1.Bias.Grad.Length);
for (int i = 0; i < linear1.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad21[i], linear1.Bias.Grad[i], delta);
}
}
public static void Run()
{
//各初期値を記述
Real[,,,] initial_W1 =
{
{ { { 1.0, 0.5, 0.0 }, { 0.5, 0.0, -0.5 }, { 0.0, -0.5, -1.0 } } },
{ { { 0.0, -0.1, 0.1 }, { -0.3, 0.4, 0.7 }, { 0.5, -0.2, 0.2 } } }
};
Real[] initial_b1 = { 0.5, 1.0 };
Real[,,,] initial_W2 =
{
{ { { -0.1, 0.6 }, { 0.3, -0.9 } }, { { 0.7, 0.9 }, { -0.2, -0.3 } } },
{ { { -0.6, -0.1 }, { 0.3, 0.3 } }, { { -0.5, 0.8 }, { 0.9, 0.1 } } }
};
Real[] initial_b2 = { 0.1, 0.9 };
Real[,] initial_W3 =
{
{ 0.5, 0.3, 0.4, 0.2, 0.6, 0.1, 0.4, 0.3 },
{ 0.6, 0.4, 0.9, 0.1, 0.5, 0.2, 0.3, 0.4 }
};
Real[] initial_b3 = { 0.01, 0.02 };
Real[,] initial_W4 = { { 0.8, 0.2 }, { 0.4, 0.6 } };
Real[] initial_b4 = { 0.02, 0.01 };
//入力データ
NdArray x = new NdArray(new Real[, , ] {
{
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.2, 0.9, 0.2, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.9, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.1, 0.8, 0.5, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.3, 0.3, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.1, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.4, 0.8, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.8, 0.4, 0.1, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.2, 0.8, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.1, 0.8, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.1, 0.7, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0 },
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }
}
});
//教師信号
Real[] t = { 0.0, 1.0 };
//層の中身をチェックしたい場合は、層単体でインスタンスを持つ
Convolution2D l2 = new Convolution2D(1, 2, 3, initialW: initial_W1, initialb: initial_b1, name: "l2 Conv2D");
//ネットワークの構成を FunctionStack に書き連ねる
FunctionStack nn = new FunctionStack(
l2, //new Convolution2D(1, 2, 3, initialW: initial_W1, initialb: initial_b1),
new ReLU(name: "l2 ReLU"),
//new AveragePooling(2, 2, name: "l2 AVGPooling"),
new MaxPooling2D(2, 2, name: "l2 MaxPooling"),
new Convolution2D(2, 2, 2, initialW: initial_W2, initialb: initial_b2, name: "l3 Conv2D"),
new ReLU(name: "l3 ReLU"),
//new AveragePooling(2, 2, name: "l3 AVGPooling"),
new MaxPooling2D(2, 2, name: "l3 MaxPooling"),
new Linear(8, 2, initialW: initial_W3, initialb: initial_b3, name: "l4 Linear"),
new ReLU(name: "l4 ReLU"),
new Linear(2, 2, initialW: initial_W4, initialb: initial_b4, name: "l5 Linear")
);
nn.SetOptimizer(new SGD(0.1));
//訓練を実施
Trainer.Train(nn, x, t, new MeanSquaredError(), false);
//Updateを実行するとgradが消費されてしまうため値を先に出力
Console.WriteLine("gw1");
Console.WriteLine(l2.Weight.ToString("Grad"));
Console.WriteLine("gb1");
Console.WriteLine(l2.Bias.ToString("Grad"));
//更新
nn.Update();
Console.WriteLine("w1");
Console.WriteLine(l2.Weight);
Console.WriteLine("b1");
Console.WriteLine(l2.Bias);
}
public void AdamRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount);
Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount);
Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount);
Real[] b = Initializer.GetRandomValues <Real[]>(outputCount);
float alpha = (float)Mother.Dice.NextDouble(); //0.001f
float beta1 = (float)Mother.Dice.NextDouble(); //0.9f;
float beta2 = (float)Mother.Dice.NextDouble(); //0.999f;
float eps = (float)Mother.Dice.NextDouble(); //1e-08f;
float eta = (float)Mother.Dice.NextDouble(); //1.0f;
//Chainer
NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, w, b);
NChainer.Adam <Real> cAdam = new NChainer.Adam <Real>(alpha, beta1, beta2, eps, eta);
cAdam.Setup(cLinear);
Variable <Real> cX = new Variable <Real>(input);
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = dummyGy;
cY.Backward();
cAdam.Update();
//KelpNet
KelpNet.CL.Linear <Real> linear = new KelpNet.CL.Linear <Real>(inputCount, outputCount, false, w, b);
KelpNet.Adam <Real> adam = new Adam <Real>(alpha, beta1, beta2, eps, eta);
adam.SetUp(linear);
NdArray <Real> x = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y = linear.Forward(x)[0];
y.Grad = dummyGy.Flatten();
y.Backward();
adam.Update();
Real[] cW = ((Real[, ])cLinear.W.Data).Flatten();
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
Real delta = 0.00001f;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
public static void Run()
{
//Weightを分割の前と後で揃える
Real[,] testWeightValues =
{
{ -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 },
{ -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 },
{ -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 },
{ -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
};
Real[][,] testJaggWeightValues =
{
new Real[, ] {
{ -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }
},
new Real[, ] {
{ -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }
},
new Real[, ] {
{ -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }
},
new Real[, ] {
{ -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
}
};
Linear l0 = new Linear(5, 20, initialW: testWeightValues, name: "l0");
Linear l1 = new Linear(5, 5, initialW: testJaggWeightValues[0], name: "l1");
Linear l2 = new Linear(5, 5, initialW: testJaggWeightValues[1], name: "l2");
Linear l3 = new Linear(5, 5, initialW: testJaggWeightValues[2], name: "l3");
Linear l4 = new Linear(5, 5, initialW: testJaggWeightValues[3], name: "l4");
//FunctionにOptimizerを設定
l0.SetOptimizer(new SGD());
//OptimiserにFunctionを登録
SGD sgd = new SGD();
l1.SetOptimizer(sgd);
l2.SetOptimizer(sgd);
l3.SetOptimizer(sgd);
l4.SetOptimizer(sgd);
//入力は同値だがGradが加算されてしまうため分ける
Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 };
NdArray testInputValuesA = new NdArray(testValue);
NdArray testInputValuesB = new NdArray(testValue);
Console.WriteLine("l0 for");
NdArray l0Result = l0.Forward(testInputValuesA)[0];
Console.WriteLine(l0Result);
Console.WriteLine("\nl1 for");
NdArray l1Result = l1.Forward(testInputValuesB)[0];
Console.WriteLine(l1Result);
Console.WriteLine("\nl2 for");
NdArray l2Result = l2.Forward(testInputValuesB)[0];
Console.WriteLine(l2Result);
Console.WriteLine("\nl3 for");
NdArray l3Result = l3.Forward(testInputValuesB)[0];
Console.WriteLine(l3Result);
Console.WriteLine("\nl4 for");
NdArray l4Result = l4.Forward(testInputValuesB)[0];
Console.WriteLine(l4Result);
Console.WriteLine();
//適当なGrad値をでっち上げる
l0Result.Grad = new Real[]
{
-2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03,
1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04,
-4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04,
-4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03
};
l1Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 };
l2Result.Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 };
l3Result.Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 };
l4Result.Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 };
//Backwardを実行
l0.Backward(l0Result);
l1.Backward(l1Result);
l2.Backward(l2Result);
l3.Backward(l3Result);
l4.Backward(l4Result);
Console.WriteLine("\nl0 back");
Console.WriteLine(testInputValuesA.ToString("Grad"));
Console.WriteLine("\nl1-l4 sum back");
Console.WriteLine(testInputValuesB.ToString("Grad"));
l0.Update(); //書式が変則的だがl0はSGDを内包しているため
sgd.Update(); //こちらはOptimizerに関数を登録して使用している
Console.WriteLine("\nl0 Weight");
Console.WriteLine(l0.Weight);
Console.WriteLine("\nl1 Weight");
Console.WriteLine(l1.Weight);
Console.WriteLine("\nl0 Bias");
Console.WriteLine(l0.Bias);
Console.WriteLine("\nl1 Bias");
Console.WriteLine(l1.Bias);
}
public void AdamRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, inputCount });
Real[,] dummyGy = (Real[,])Initializer.GetRealNdArray(new[] { batchCount, outputCount });
Real[,] w = (Real[,])Initializer.GetRealNdArray(new[] { outputCount, inputCount });
Real[] b = Initializer.GetRealArray(outputCount);
float alpha = (float)Mother.Dice.NextDouble(); //0.001f
float beta1 = (float)Mother.Dice.NextDouble(); //0.9f;
float beta2 = (float)Mother.Dice.NextDouble(); //0.999f;
float eps = (float)Mother.Dice.NextDouble(); //1e-08f;
float eta = (float)Mother.Dice.NextDouble(); //1.0f;
//Chainer
NChainer.Linear<Real> cLinear = new NChainer.Linear<Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b));
NChainer.Adam<Real> cAdam = new NChainer.Adam<Real>(alpha, beta1, beta2, eps, eta);
cAdam.Setup(cLinear);
Variable<Real> cX = new Variable<Real>(Real.ToBaseNdArray(input));
Variable<Real> cY = cLinear.Forward(cX);
cY.Grad = Real.ToBaseNdArray(dummyGy);
cY.Backward();
cAdam.Update();
//KelpNet
KelpNet.CL.Linear linear = new KelpNet.CL.Linear(inputCount, outputCount, false, w, b);
KelpNet.Adam adam = new Adam(alpha, beta1, beta2, eps, eta);
adam.SetUp(linear);
NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount);
NdArray y = linear.Forward(x)[0];
y.Grad = Real.ToRealArray(dummyGy);
y.Backward();
adam.Update();
Real[] cW = Real.ToRealArray((Real[,])cLinear.W.Data);
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
double delta = 0.00001;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
public FilteringMatrix(Real[,] filteringMatrix)
: base(filteringMatrix)
{
Factor = DefaultFactor;
Bias = DefaultBias;
}
public void RnnLSTMRandomTest()
{
Python.Initialize();
Chainer.Initialize();
Real[,] input = { { 1.0f }, { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f } };
Real[,] teach = { { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f } };
Real[,] input2 = { { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f } };
Real[,] teach2 = { { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f }, { 13.0f } };
int outputCount = 1;
int inputCount = 1;
int hiddenCount = 2;
Real[,] upwardInit = Initializer.GetRandomValues <Real[, ]>(hiddenCount, hiddenCount);
Real[,] lateralInit = Initializer.GetRandomValues <Real[, ]>(hiddenCount, hiddenCount);
Real[,,] biasInit = Initializer.GetRandomValues <Real[, , ]>(1, hiddenCount, 1);
Real[,,] forgetBiasInit = Initializer.GetRandomValues <Real[, , ]>(1, hiddenCount, 1);
//Chainer
Real[,] w1 = Initializer.GetRandomValues <Real[, ]>(hiddenCount, inputCount);
Real[] b1 = Initializer.GetRandomValues <Real[]>(hiddenCount);
//Chainer
Linear <Real> cLinear1 = new Linear <Real>(inputCount, hiddenCount, false, w1, b1);
NChainer.LSTM <Real> cLstm = new NChainer.LSTM <Real>(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit);
Real[,] w2 = Initializer.GetRandomValues <Real[, ]>(outputCount, hiddenCount);
Real[] b2 = Initializer.GetRandomValues <Real[]>(outputCount);
Linear <Real> cLinear2 = new Linear <Real>(hiddenCount, outputCount, false, w2, b2);
Variable <Real> cX1 = new Variable <Real>(input);
Variable <Real> cY11 = cLinear1.Forward(cX1);
Variable <Real> cY12 = cLstm.Forward(cY11);
Variable <Real> cY13 = cLinear2.Forward(cY12);
Variable <Real> cT = new Variable <Real>(teach);
Variable <Real> cLoss = new NChainer.MeanSquaredError <Real>().Forward(cY13, cT);
cLoss.Backward();
//KelpNet
CL.Linear <Real> linear1 = new CL.Linear <Real>(inputCount, hiddenCount, false, w1, b1);
LSTM <Real> lstm = new LSTM <Real>(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit);
CL.Linear <Real> linear2 = new CL.Linear <Real>(hiddenCount, outputCount, false, w2, b2);
NdArray <Real> x1 = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y11 = linear1.Forward(x1)[0];
NdArray <Real> y12 = lstm.Forward(y11)[0];
NdArray <Real> y13 = linear2.Forward(y12)[0];
NdArray <Real> t = new NdArray <Real>(teach, asBatch: true);
NdArray <Real> loss = new MeanSquaredError <Real>().Evaluate(y13, t);
y13.Backward();
Real[] cY11data = ((Real[, ])cY11.Data).Flatten();
Real[] cY12data = ((Real[, ])cY12.Data).Flatten();
Real[] cY13data = ((Real[, ])cY13.Data).Flatten();
Real[] cXgrad = ((Real[, ])cX1.Grad).Flatten();
Real[] cupwardWGrad = ((Real[, ])cLstm.upward.W.Grad).Flatten();
Real[] cupwardbGrad = (Real[])cLstm.upward.b.Grad;
//許容範囲を設定
Real delta = 0.00001f;
//y11
Assert.AreEqual(cY11data.Length, y11.Data.Length);
for (int i = 0; i < cY11data.Length; i++)
{
Assert.AreEqual(cY11data[i], y11.Data[i], delta);
}
//y12
Assert.AreEqual(cY12data.Length, y12.Data.Length);
for (int i = 0; i < cY12data.Length; i++)
{
Assert.AreEqual(cY12data[i], y12.Data[i], delta);
}
//y13
Assert.AreEqual(cY13data.Length, y13.Data.Length);
for (int i = 0; i < cY13data.Length; i++)
{
Assert.AreEqual(cY13data[i], y13.Data[i], delta);
}
//許容範囲を設定
delta = 0.0001f;
//loss
Assert.AreEqual(cLoss.Data[0], loss.Data[0], delta);
//x.Grad
Assert.AreEqual(cXgrad.Length, x1.Grad.Length);
for (int i = 0; i < cXgrad.Length; i++)
{
Assert.AreEqual(cXgrad[i], x1.Grad[i], delta);
}
Real[] cWgrad11 = ((Real[, ])cLinear1.W.Grad).Flatten();
Real[] cbgrad11 = (Real[])cLinear1.b.Grad;
//W.grad
Assert.AreEqual(cWgrad11.Length, linear1.Weight.Grad.Length);
for (int i = 0; i < linear1.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad11[i], linear1.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad11.Length, linear1.Bias.Grad.Length);
for (int i = 0; i < linear1.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad11[i], linear1.Bias.Grad[i], delta);
}
Real[] cWgrad12 = ((Real[, ])cLinear2.W.Grad).Flatten();
Real[] cbgrad12 = (Real[])cLinear2.b.Grad;
//W.grad
Assert.AreEqual(cWgrad12.Length, linear2.Weight.Grad.Length);
for (int i = 0; i < linear2.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad12[i], linear2.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad12.Length, linear2.Bias.Grad.Length);
for (int i = 0; i < linear2.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad12[i], linear2.Bias.Grad[i], delta);
}
//W.grad
int wLen = lstm.upward.Weight.Grad.Length;
Assert.AreEqual(cupwardWGrad.Length, lstm.upward.Weight.Grad.Length);
for (int i = 0; i < wLen; i++)
{
Assert.AreEqual(cupwardWGrad[i + wLen * 0], lstm.upward.Weight.Grad[i], delta);
}
//b.grad
int bLen = lstm.upward.Bias.Length;
Assert.AreEqual(cupwardbGrad.Length, lstm.upward.Bias.Grad.Length);
for (int i = 0; i < bLen; i++)
{
Assert.AreEqual(cupwardbGrad[i + wLen * 0], lstm.upward.Bias.Grad[i], delta);
}
//２周目
Variable <Real> cX2 = new Variable <Real>(input2);
Variable <Real> cY21 = cLinear1.Forward(cX2);
Variable <Real> cY22 = cLstm.Forward(cY21);
Variable <Real> cY23 = cLinear2.Forward(cY22);
Variable <Real> cT2 = new Variable <Real>(teach2);
Variable <Real> cLoss2 = new NChainer.MeanSquaredError <Real>().Forward(cY23, cT2);
//KelpNet
NdArray <Real> x2 = new NdArray <Real>(input2, asBatch: true);
NdArray <Real> y21 = linear1.Forward(x2)[0];
NdArray <Real> y22 = lstm.Forward(y21)[0];
NdArray <Real> y23 = linear2.Forward(y22)[0];
NdArray <Real> t2 = new NdArray <Real>(teach2, asBatch: true);
NdArray <Real> loss2 = new MeanSquaredError <Real>().Evaluate(y23, t2);
Assert.AreEqual(cLoss2.Data[0], loss2.Data[0], delta);
//Backwardを実行
cLoss2.Backward();
y23.Backward();
Real[] cYdata21 = ((Real[, ])cY21.Data).Flatten();
Real[] cYdata22 = ((Real[, ])cY22.Data).Flatten();
Real[] cYdata23 = ((Real[, ])cY23.Data).Flatten();
Real[] cXgrad2 = ((Real[, ])cX2.Grad).Flatten();
Real[] cupwardWGrad2 = ((Real[, ])cLstm.upward.W.Grad).Flatten();
Real[] cupwardbGrad2 = (Real[])cLstm.upward.b.Grad;
Real[] clateralWGrad = ((Real[, ])cLstm.lateral.W.Grad).Flatten();
//y21
Assert.AreEqual(cYdata21.Length, y21.Data.Length);
for (int i = 0; i < cYdata21.Length; i++)
{
Assert.AreEqual(cYdata21[i], y21.Data[i], delta);
}
//y22
Assert.AreEqual(cYdata22.Length, y22.Data.Length);
for (int i = 0; i < cYdata22.Length; i++)
{
Assert.AreEqual(cYdata22[i], y22.Data[i], delta);
}
//y23
Assert.AreEqual(cYdata23.Length, y23.Data.Length);
for (int i = 0; i < cYdata23.Length; i++)
{
Assert.AreEqual(cYdata23[i], y23.Data[i], delta);
}
//x.Grad
Assert.AreEqual(cXgrad2.Length, x2.Grad.Length);
for (int i = 0; i < cXgrad2.Length; i++)
{
Assert.AreEqual(cXgrad2[i], x2.Grad[i], delta);
}
//経由が多くかなり誤差が大きい為
delta = 1.0f;
Real[] cWgrad22 = ((Real[, ])cLinear2.W.Grad).Flatten();
Real[] cbgrad22 = (Real[])cLinear2.b.Grad;
//W.grad
Assert.AreEqual(cWgrad22.Length, linear2.Weight.Grad.Length);
for (int i = 0; i < linear2.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad22[i], linear2.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad22.Length, linear2.Bias.Grad.Length);
for (int i = 0; i < linear2.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad22[i], linear2.Bias.Grad[i], delta);
}
delta = 2.0f;
//W.grad
Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length);
for (int i = 0; i < clateralWGrad.Length; i++)
{
Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta);
}
for (int i = 0; i < wLen; i++)
{
Assert.AreEqual(cupwardWGrad2[i + wLen * 0], lstm.upward.Weight.Grad[i], delta);
}
//b.grad
for (int i = 0; i < bLen; i++)
{
Assert.AreEqual(cupwardbGrad2[i + wLen * 0], lstm.upward.Bias.Grad[i], delta);
}
delta = 20.0f;
Real[] cWgrad21 = ((Real[, ])cLinear1.W.Grad).Flatten();
Real[] cbgrad21 = (Real[])cLinear1.b.Grad;
//W.grad
Assert.AreEqual(cWgrad21.Length, linear1.Weight.Grad.Length);
for (int i = 0; i < linear1.Weight.Grad.Length; i++)
{
Assert.AreEqual(cWgrad21[i], linear1.Weight.Grad[i], delta);
}
//b.grad
Assert.AreEqual(cbgrad21.Length, linear1.Bias.Grad.Length);
for (int i = 0; i < linear1.Bias.Grad.Length; i++)
{
Assert.AreEqual(cbgrad21[i], linear1.Bias.Grad[i], delta);
}
}