private void DrawSGD(Graphics graphics, DrawF draw)
{
ArrayList history = new ArrayList();
PointF2D point = new PointF2D(-4.0f, 2.0f);
Function2 fun = new Function2();
SGD optimizer = new SGD(0.95f);
for (int index = 0; index < 30; index++)
{
PointF2D xyPoint = draw.getBlockPoint(point.X, point.Y);
history.Add(xyPoint);
PointF2D diff = fun.DiffFormula(point.X, point.Y);
optimizer.Update(point, diff);
}
PointF2D prePoint = ((PointF2D)history[0]);
for (int index = 0; index < 30; index++)
{
draw.drawPoint(graphics, Brushes.Blue, ((PointF2D)history[index]));
draw.drawLine(graphics, prePoint, ((PointF2D)history[index]));
prePoint = ((PointF2D)history[index]);
}
}
public async void TestTwoLayerNet()
{
// TrainingData data = TrainingData.Load("sampleWidget.td");
List <Matrix> matrices = await LoadMNISTAsync();
Matrix x_train = matrices[0];
Matrix t_train = matrices[1];
// Matrix x_test = matrices[2];
// Matrix y_test = matrices[3];
int itersNum = 3000;
int batch_size = 100;
IOptimize optimizer = new SGD();// SGD();//AdaGrad();// Momentum();// SGD();
net = new TwoLayerNet(784, 50, 10);
net.LossUpdated += loss => { this.LossUpdated?.Invoke(loss); };
// net.print += Net_print;
Matrix x_batch = new Matrix(batch_size, 784);
Matrix t_batch = new Matrix(batch_size, 10);
int[] indexs = new int[batch_size];
for (int i = 0; i < itersNum; i++)
{
for (int j = 0; j < batch_size; j++)
{
indexs[j] = random.Next(TrainDataCount);
}
for (int row = 0; row < indexs.Length; row++)
{
int index = indexs[row];
// Console.WriteLine(index);
for (int col = 0; col < x_batch.Y; col++)
{
x_batch[row, col] = x_train[index, col];
}
for (int col = 0; col < t_batch.Y; col++)
{
t_batch[row, col] = t_train[index, col];//
}
}
// Matrix[] grads = net.NumericalGradient(x_batch, t_batch);
Matrix[] grads = net.Gradient(x_batch, t_batch);
optimizer.Update(net.Params, grads);
net.Update();
Console.WriteLine($"损失值:{net.Loss(x_batch, t_batch)} ");
if (i % 300 == 0)
{
Console.WriteLine("识别精度计算中...");
double accuracy = net.Accuracy(x_train, t_train);
AccuracyUpdated?.Invoke(accuracy);
accuracies.Add(accuracy);
Console.WriteLine($"识别精度：{accuracy}");
}
}
net.Save("sampleWidget.td");
}
public static void Run()
{
//Align Weight before and after splitting
Real[,] testWeightValues =
{
{ -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 },
{ -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 },
{ -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 },
{ -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
};
Real[][,] testJaggWeightValues =
{
new Real[, ] {
{ -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }
},
new Real[, ] {
{ -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }
},
new Real[, ] {
{ -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }
},
new Real[, ] {
{ -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
}
};
Linear l0 = new Linear(true, 5, 20, initialW: testWeightValues, name: "l0");
Linear l1 = new Linear(true, 5, 5, initialW: testJaggWeightValues[0], name: "l1");
Linear l2 = new Linear(true, 5, 5, initialW: testJaggWeightValues[1], name: "l2");
Linear l3 = new Linear(true, 5, 5, initialW: testJaggWeightValues[2], name: "l3");
Linear l4 = new Linear(true, 5, 5, initialW: testJaggWeightValues[3], name: "l4");
l0.SetOptimizer(new SGD());
SGD sgd = new SGD();
l1.SetOptimizer(sgd);
l2.SetOptimizer(sgd);
l3.SetOptimizer(sgd);
l4.SetOptimizer(sgd);
//Input is equivalent, but Grad is added and it is divided
Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 };
NdArray testInputValuesA = new NdArray(testValue);
NdArray testInputValuesB = new NdArray(testValue);
RILogManager.Default?.SendDebug("l0 for");
NdArray[] l0Result = l0.Forward(true, testInputValuesA);
RILogManager.Default?.SendDebug(l0Result.ToString());
RILogManager.Default?.SendDebug("l1 for");
NdArray[] l1Result = l1.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l1Result.ToString());
RILogManager.Default?.SendDebug("l2 for");
NdArray[] l2Result = l2.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l2Result.ToString());
RILogManager.Default?.SendDebug("l3 for");
NdArray[] l3Result = l3.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l3Result.ToString());
RILogManager.Default?.SendDebug("l4 for");
NdArray[] l4Result = l4.Forward(true, testInputValuesB);
RILogManager.Default?.SendDebug(l4Result.ToString());
//Create an appropriate Grad value
l0Result[0].Grad = new Real[]
{
-2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03,
1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04,
-4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04,
-4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03
};
l1Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 };
l2Result[0].Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 };
l3Result[0].Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 };
l4Result[0].Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 };
//Backward
l0.Backward(true, l0Result);
l1.Backward(true, l1Result);
l2.Backward(true, l2Result);
l3.Backward(true, l3Result);
l4.Backward(true, l4Result);
RILogManager.Default?.SendDebug("l0 back");
RILogManager.Default?.SendDebug(testInputValuesA.ToString("Grad"));
RILogManager.Default?.SendDebug("l1-l4 sum back");
RILogManager.Default?.SendDebug(testInputValuesB.ToString("Grad"));
l0.Update(); //Although the format is irregular, since 10 contains SGD
sgd.Update(); // Use stochastic gradient descent as the optimizer
RILogManager.Default?.SendDebug("l0 Weight");
RILogManager.Default?.SendDebug(l0.Weight.ToString());
RILogManager.Default?.SendDebug("l1 Weight");
RILogManager.Default?.SendDebug(l1.Weight.ToString());
RILogManager.Default?.SendDebug("l0 Bias");
RILogManager.Default?.SendDebug(l0.Bias.ToString());
RILogManager.Default?.SendDebug("l1 Bias");
RILogManager.Default?.SendDebug(l1.Bias.ToString());
}
public static void Run()
{
//読み込みたいネットワークの構成を FunctionStack に書き連ね、各 Function のパラメータを合わせる
//ここで必ず name を Chainer の変数名に合わせておくこと
FunctionStack <Real> nn = new FunctionStack <Real>(
new Convolution2D <Real>(1, 2, 3, name: "conv1", gpuEnable: true),//必要であればGPUフラグも忘れずに
new ReLU <Real>(),
new MaxPooling2D <Real>(2, 2),
new Convolution2D <Real>(2, 2, 2, name: "conv2", gpuEnable: true),
new ReLU <Real>(),
new MaxPooling2D <Real>(2, 2),
new Linear <Real>(8, 2, name: "fl3"),
new ReLU <Real>(),
new Linear <Real>(2, 2, name: "fl4")
);
/* Chainerでの宣言
* class NN(chainer.Chain):
* def __init__(self):
* super(NN, self).__init__(
* conv1 = L.Convolution2D(1,2,3),
* conv2 = L.Convolution2D(2,2,2),
* fl3 = L.Linear(8,2),
* fl4 = L.Linear(2,2)
* )
*
* def __call__(self, x):
* h_conv1 = F.relu(self.conv1(x))
* h_pool1 = F.max_pooling_2d(h_conv1, 2)
* h_conv2 = F.relu(self.conv2(h_pool1))
* h_pool2 = F.max_pooling_2d(h_conv2, 2)
* h_fc1 = F.relu(self.fl3(h_pool2))
* y = self.fl4(h_fc1)
* return y
*/
//パラメータを読み込み
ChainerModelDataLoader.ModelLoad(MODEL_FILE_PATH, nn);
//あとは通常通り使用する
SGD <Real> sgd = new SGD <Real>(0.1f);
sgd.SetUp(nn);
//入力データ
NdArray <Real> x = new NdArray <Real>(new Real[, , ] {
{
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.9f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.8f, 0.9f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.1f, 0.8f, 0.5f, 0.8f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.3f, 0.3f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.1f, 0.0f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.7f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.4f, 0.8f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.8f, 0.4f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.8f, 0.3f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.8f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.3f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }
}
});
//教師信号
NdArray <Real> t = new NdArray <Real>(new Real[] { 0.0f, 1.0f });
//訓練を実施
Trainer.Train(nn, x, t, new MeanSquaredError <Real>());
//結果表示用に退避
Convolution2D <Real> l2 = (Convolution2D <Real>)nn.Functions[0];
//Updateを実行するとgradが消費されてしまうため値を先に出力
Console.WriteLine("gw1");
Console.WriteLine(l2.Weight.ToString("Grad"));
Console.WriteLine("gb1");
Console.WriteLine(l2.Bias.ToString("Grad"));
//更新
sgd.Update();
Console.WriteLine("w1");
Console.WriteLine(l2.Weight);
Console.WriteLine("b1");
Console.WriteLine(l2.Bias);
}
public static void Run()
{
//Weightを分割の前と後で揃える
Real[,] testWeightValues =
{
{ -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 },
{ -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 },
{ -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 },
{ -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
};
Real[][,] testJaggWeightValues =
{
new Real[, ] {
{ -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 },
{ -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 },
{ -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 },
{ 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 },
{ 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }
},
new Real[, ] {
{ -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 },
{ -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 },
{ -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 },
{ -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 },
{ -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }
},
new Real[, ] {
{ -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 },
{ -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 },
{ -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 },
{ -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 },
{ 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }
},
new Real[, ] {
{ -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 },
{ 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 },
{ 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 },
{ -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 },
{ 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 }
}
};
Linear l0 = new Linear(5, 20, initialW: testWeightValues, name: "l0");
Linear l1 = new Linear(5, 5, initialW: testJaggWeightValues[0], name: "l1");
Linear l2 = new Linear(5, 5, initialW: testJaggWeightValues[1], name: "l2");
Linear l3 = new Linear(5, 5, initialW: testJaggWeightValues[2], name: "l3");
Linear l4 = new Linear(5, 5, initialW: testJaggWeightValues[3], name: "l4");
//FunctionにOptimizerを設定
l0.SetOptimizer(new SGD());
//OptimiserにFunctionを登録
SGD sgd = new SGD();
l1.SetOptimizer(sgd);
l2.SetOptimizer(sgd);
l3.SetOptimizer(sgd);
l4.SetOptimizer(sgd);
//入力は同値だがGradが加算されてしまうため分ける
Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 };
NdArray testInputValuesA = new NdArray(testValue);
NdArray testInputValuesB = new NdArray(testValue);
Console.WriteLine("l0 for");
NdArray l0Result = l0.Forward(testInputValuesA)[0];
Console.WriteLine(l0Result);
Console.WriteLine("\nl1 for");
NdArray l1Result = l1.Forward(testInputValuesB)[0];
Console.WriteLine(l1Result);
Console.WriteLine("\nl2 for");
NdArray l2Result = l2.Forward(testInputValuesB)[0];
Console.WriteLine(l2Result);
Console.WriteLine("\nl3 for");
NdArray l3Result = l3.Forward(testInputValuesB)[0];
Console.WriteLine(l3Result);
Console.WriteLine("\nl4 for");
NdArray l4Result = l4.Forward(testInputValuesB)[0];
Console.WriteLine(l4Result);
Console.WriteLine();
//適当なGrad値をでっち上げる
l0Result.Grad = new Real[]
{
-2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03,
1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04,
-4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04,
-4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03
};
l1Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 };
l2Result.Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 };
l3Result.Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 };
l4Result.Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 };
//Backwardを実行
l0.Backward(l0Result);
l1.Backward(l1Result);
l2.Backward(l2Result);
l3.Backward(l3Result);
l4.Backward(l4Result);
Console.WriteLine("\nl0 back");
Console.WriteLine(testInputValuesA.ToString("Grad"));
Console.WriteLine("\nl1-l4 sum back");
Console.WriteLine(testInputValuesB.ToString("Grad"));
l0.Update(); //書式が変則的だがl0はSGDを内包しているため
sgd.Update(); //こちらはOptimizerに関数を登録して使用している
Console.WriteLine("\nl0 Weight");
Console.WriteLine(l0.Weight);
Console.WriteLine("\nl1 Weight");
Console.WriteLine(l1.Weight);
Console.WriteLine("\nl0 Bias");
Console.WriteLine(l0.Bias);
Console.WriteLine("\nl1 Bias");
Console.WriteLine(l1.Bias);
}
public void SGDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, inputCount });
Real[,] dummyGy = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, outputCount });
Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, inputCount });
Real[] b = Initializer.GetRealArray(outputCount);
//Chainer
NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b));
NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>();
cSgd.Setup(cLinear);
Variable <Real> cX = new Variable <Real>(Real.ToBaseNdArray(input));
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = Real.ToBaseNdArray(dummyGy);
cY.Backward();
cSgd.Update();
//KelpNet
KelpNet.Linear linear = new KelpNet.Linear(inputCount, outputCount, false, w, b);
KelpNet.SGD sgd = new SGD();
sgd.SetUp(linear);
NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount);
NdArray y = linear.Forward(x)[0];
y.Grad = Real.ToRealArray(dummyGy);
y.Backward();
sgd.Update();
Real[] cW = Real.ToRealArray((Real[, ])cLinear.W.Data);
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
double delta = 0.00001;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
public static void Run()
{
//Weightを分割の前と後で揃える
Real[,] testWeightValues = new Real[, ] {
{ -0.02690255f, 0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f },
{ -0.06963444f, -0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f },
{ -0.04966073f, -0.04697048f, -0.02235234f, -0.09396666f, 0.073189f },
{ 0.06563969f, 0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f },
{ 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f },
{ -0.05200623f, -0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f },
{ -0.0254951f, 0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f },
{ -0.0732494f, 0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f },
{ -0.06818873f, -0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f },
{ -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f },
{ -0.0454098f, 0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f },
{ -0.00866754f, -0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f },
{ -0.01789357f, -0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f },
{ -0.02421552f, 0.05533903f, -0.08627617f, 0.094624f, 0.03319318f },
{ 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f },
{ -0.07128382f, 0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f },
{ 0.09431138f, 0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f },
{ 0.01844446f, -0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f },
{ -0.0261334f, -0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f },
{ 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f }
};
Real[][,] testJaggWeightValues =
{
new Real[, ] {
{ -0.02690255f,0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f },
{ -0.06963444f,-0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f },
{ -0.04966073f,-0.04697048f, -0.02235234f, -0.09396666f, 0.073189f },
{ 0.06563969f,0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f },
{ 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f }
},
new Real[, ] {
{ -0.05200623f,-0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f },
{ -0.0254951f,0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f },
{ -0.0732494f,0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f },
{ -0.06818873f,-0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f },
{ -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f }
},
new Real[, ] {
{ -0.0454098f,0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f },
{ -0.00866754f,-0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f },
{ -0.01789357f,-0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f },
{ -0.02421552f,0.05533903f, -0.08627617f, 0.094624f, 0.03319318f },
{ 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f }
},
new Real[, ] {
{ -0.07128382f,0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f },
{ 0.09431138f,0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f },
{ 0.01844446f,-0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f },
{ -0.0261334f,-0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f },
{ 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f }
}
};
Linear <Real> l0 = new Linear <Real>(5, 20, initialW: testWeightValues, name: "l0");
Linear <Real> l1 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[0], name: "l1");
Linear <Real> l2 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[1], name: "l2");
Linear <Real> l3 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[2], name: "l3");
Linear <Real> l4 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[3], name: "l4");
//FunctionにOptimizerを設定
SGD <Real> sgd = new SGD <Real>();
sgd.SetUp(l0);
//OptimiserにFunctionを登録
SGD <Real> sgdSplit = new SGD <Real>();
sgdSplit.SetUp(l1);
sgdSplit.SetUp(l2);
sgdSplit.SetUp(l3);
sgdSplit.SetUp(l4);
//入力は同値だがGradが加算されてしまうため分ける
Real[] testValue = new Real[] { 0.01618112f, -0.08296648f, -0.05545357f, 0.00389254f, -0.05727582f };
NdArray <Real> testInputValuesA = new NdArray <Real>(testValue);
NdArray <Real> testInputValuesB = new NdArray <Real>(testValue);
Console.WriteLine("l0 for");
NdArray <Real> l0Result = l0.Forward(testInputValuesA)[0];
Console.WriteLine(l0Result);
Console.WriteLine("\nl1 for");
NdArray <Real> l1Result = l1.Forward(testInputValuesB)[0];
Console.WriteLine(l1Result);
Console.WriteLine("\nl2 for");
NdArray <Real> l2Result = l2.Forward(testInputValuesB)[0];
Console.WriteLine(l2Result);
Console.WriteLine("\nl3 for");
NdArray <Real> l3Result = l3.Forward(testInputValuesB)[0];
Console.WriteLine(l3Result);
Console.WriteLine("\nl4 for");
NdArray <Real> l4Result = l4.Forward(testInputValuesB)[0];
Console.WriteLine(l4Result);
Console.WriteLine();
//適当なGrad値をでっち上げる
l0Result.Grad = new Real[]
{
-2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f,
1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f,
-4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f,
-4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f
};
l1Result.Grad = new Real[] { -2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f };
l2Result.Grad = new Real[] { 1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f };
l3Result.Grad = new Real[] { -4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f };
l4Result.Grad = new Real[] { -4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f };
//Backwardを実行
l0.Backward(l0Result);
l1.Backward(l1Result);
l2.Backward(l2Result);
l3.Backward(l3Result);
l4.Backward(l4Result);
Console.WriteLine("\nl0 back");
Console.WriteLine(testInputValuesA.ToString("Grad"));
Console.WriteLine("\nl1-l4 sum back");
Console.WriteLine(testInputValuesB.ToString("Grad"));
sgd.Update();
sgdSplit.Update();
Console.WriteLine("\nl0 Weight");
Console.WriteLine(l0.Weight);
Console.WriteLine("\nl1 Weight");
Console.WriteLine(l1.Weight);
Console.WriteLine("\nl0 Bias");
Console.WriteLine(l0.Bias);
Console.WriteLine("\nl1 Bias");
Console.WriteLine(l1.Bias);
}
public void SGDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount);
Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount);
Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount);
Real[] b = Initializer.GetRandomValues <Real[]>(outputCount);
//Chainer
Linear <Real> cLinear = new Linear <Real>(inputCount, outputCount, false, w, b);
NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>();
cSgd.Setup(cLinear);
Variable <Real> cX = new Variable <Real>(input);
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = dummyGy;
cY.Backward();
cSgd.Update();
//KelpNet
CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b);
KelpNet.SGD <Real> sgd = new SGD <Real>();
sgd.SetUp(linear);
NdArray <Real> x = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y = linear.Forward(x)[0];
y.Grad = dummyGy.Flatten();
y.Backward();
sgd.Update();
Real[] cW = ((Real[, ])cLinear.W.Data).Flatten();
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
Real delta = 0.00001f;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
public static void Run()
{
Console.WriteLine("Build Vocabulary.");
Vocabulary vocabulary = new Vocabulary();
string trainPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE, TRAIN_FILE_HASH);
string validPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + VALID_FILE, VALID_FILE, VALID_FILE_HASH);
string testPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TEST_FILE, TEST_FILE, TEST_FILE_HASH);
int[] trainData = vocabulary.LoadData(trainPath);
int[] validData = vocabulary.LoadData(validPath);
int[] testData = vocabulary.LoadData(testPath);
int nVocab = vocabulary.Length;
Console.WriteLine("Network Initilizing.");
FunctionStack <Real> model = new FunctionStack <Real>(
new EmbedID <Real>(nVocab, N_UNITS, name: "l1 EmbedID"),
new Dropout <Real>(),
new LSTM <Real>(N_UNITS, N_UNITS, name: "l2 LSTM"),
new Dropout <Real>(),
new LSTM <Real>(N_UNITS, N_UNITS, name: "l3 LSTM"),
new Dropout <Real>(),
new Linear <Real>(N_UNITS, nVocab, name: "l4 Linear")
);
for (int i = 0; i < model.Functions.Length; i++)
{
for (int j = 0; j < model.Functions[i].Parameters.Length; j++)
{
for (int k = 0; k < model.Functions[i].Parameters[j].Data.Length; k++)
{
model.Functions[i].Parameters[j].Data[k] = ((Real)Mother.Dice.NextDouble() * 2.0f - 1.0f) / 10.0f;
}
}
}
//与えられたthresholdで頭打ちではなく、全パラメータのL2Normからレートを取り補正を行う
GradientClipping <Real> gradientClipping = new GradientClipping <Real>(threshold: GRAD_CLIP);
SGD <Real> sgd = new SGD <Real>(learningRate: 0.1f);
gradientClipping.SetUp(model);
sgd.SetUp(model);
Real wholeLen = trainData.Length;
int jump = (int)Math.Floor(wholeLen / BATCH_SIZE);
int epoch = 0;
Console.WriteLine("Train Start.");
for (int i = 0; i < jump * N_EPOCH; i++)
{
NdArray <Real> x = new NdArray <Real>(new[] { 1 }, BATCH_SIZE);
NdArray <int> t = new NdArray <int>(new[] { 1 }, BATCH_SIZE);
for (int j = 0; j < BATCH_SIZE; j++)
{
x.Data[j] = trainData[(int)((jump * j + i) % wholeLen)];
t.Data[j] = trainData[(int)((jump * j + i + 1) % wholeLen)];
}
NdArray <Real> result = model.Forward(x)[0];
Real sumLoss = new SoftmaxCrossEntropy <Real>().Evaluate(result, t);
Console.WriteLine("[{0}/{1}] Loss: {2}", i + 1, jump, sumLoss);
model.Backward(result);
//Run truncated BPTT
if ((i + 1) % BPROP_LEN == 0)
{
gradientClipping.Update();
sgd.Update();
model.ResetState();
}
if ((i + 1) % jump == 0)
{
epoch++;
Console.WriteLine("evaluate");
Console.WriteLine("validation perplexity: {0}", Evaluate(model, validData));
if (epoch >= 6)
{
sgd.LearningRate /= 1.2f;
Console.WriteLine("learning rate =" + sgd.LearningRate);
}
}
}
Console.WriteLine("test start");
Console.WriteLine("test perplexity:" + Evaluate(model, testData));
}
