private void DrawSGD(Graphics graphics, DrawF draw) { ArrayList history = new ArrayList(); PointF2D point = new PointF2D(-4.0f, 2.0f); Function2 fun = new Function2(); SGD optimizer = new SGD(0.95f); for (int index = 0; index < 30; index++) { PointF2D xyPoint = draw.getBlockPoint(point.X, point.Y); history.Add(xyPoint); PointF2D diff = fun.DiffFormula(point.X, point.Y); optimizer.Update(point, diff); } PointF2D prePoint = ((PointF2D)history[0]); for (int index = 0; index < 30; index++) { draw.drawPoint(graphics, Brushes.Blue, ((PointF2D)history[index])); draw.drawLine(graphics, prePoint, ((PointF2D)history[index])); prePoint = ((PointF2D)history[index]); } }
public async void TestTwoLayerNet() { // TrainingData data = TrainingData.Load("sampleWidget.td"); List <Matrix> matrices = await LoadMNISTAsync(); Matrix x_train = matrices[0]; Matrix t_train = matrices[1]; // Matrix x_test = matrices[2]; // Matrix y_test = matrices[3]; int itersNum = 3000; int batch_size = 100; IOptimize optimizer = new SGD();// SGD();//AdaGrad();// Momentum();// SGD(); net = new TwoLayerNet(784, 50, 10); net.LossUpdated += loss => { this.LossUpdated?.Invoke(loss); }; // net.print += Net_print; Matrix x_batch = new Matrix(batch_size, 784); Matrix t_batch = new Matrix(batch_size, 10); int[] indexs = new int[batch_size]; for (int i = 0; i < itersNum; i++) { for (int j = 0; j < batch_size; j++) { indexs[j] = random.Next(TrainDataCount); } for (int row = 0; row < indexs.Length; row++) { int index = indexs[row]; // Console.WriteLine(index); for (int col = 0; col < x_batch.Y; col++) { x_batch[row, col] = x_train[index, col]; } for (int col = 0; col < t_batch.Y; col++) { t_batch[row, col] = t_train[index, col];// } } // Matrix[] grads = net.NumericalGradient(x_batch, t_batch); Matrix[] grads = net.Gradient(x_batch, t_batch); optimizer.Update(net.Params, grads); net.Update(); Console.WriteLine($"损失值:{net.Loss(x_batch, t_batch)} "); if (i % 300 == 0) { Console.WriteLine("识别精度计算中..."); double accuracy = net.Accuracy(x_train, t_train); AccuracyUpdated?.Invoke(accuracy); accuracies.Add(accuracy); Console.WriteLine($"识别精度:{accuracy}"); } } net.Save("sampleWidget.td"); }
public static void Run() { //Align Weight before and after splitting Real[,] testWeightValues = { { -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }, { -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }, { -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }, { -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 } }, new Real[, ] { { -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 } }, new Real[, ] { { -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 } }, new Real[, ] { { -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } } }; Linear l0 = new Linear(true, 5, 20, initialW: testWeightValues, name: "l0"); Linear l1 = new Linear(true, 5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear l2 = new Linear(true, 5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear l3 = new Linear(true, 5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear l4 = new Linear(true, 5, 5, initialW: testJaggWeightValues[3], name: "l4"); l0.SetOptimizer(new SGD()); SGD sgd = new SGD(); l1.SetOptimizer(sgd); l2.SetOptimizer(sgd); l3.SetOptimizer(sgd); l4.SetOptimizer(sgd); //Input is equivalent, but Grad is added and it is divided Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 }; NdArray testInputValuesA = new NdArray(testValue); NdArray testInputValuesB = new NdArray(testValue); RILogManager.Default?.SendDebug("l0 for"); NdArray[] l0Result = l0.Forward(true, testInputValuesA); RILogManager.Default?.SendDebug(l0Result.ToString()); RILogManager.Default?.SendDebug("l1 for"); NdArray[] l1Result = l1.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l1Result.ToString()); RILogManager.Default?.SendDebug("l2 for"); NdArray[] l2Result = l2.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l2Result.ToString()); RILogManager.Default?.SendDebug("l3 for"); NdArray[] l3Result = l3.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l3Result.ToString()); RILogManager.Default?.SendDebug("l4 for"); NdArray[] l4Result = l4.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l4Result.ToString()); //Create an appropriate Grad value l0Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03, 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04, -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04, -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; l1Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 }; l2Result[0].Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 }; l3Result[0].Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 }; l4Result[0].Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; //Backward l0.Backward(true, l0Result); l1.Backward(true, l1Result); l2.Backward(true, l2Result); l3.Backward(true, l3Result); l4.Backward(true, l4Result); RILogManager.Default?.SendDebug("l0 back"); RILogManager.Default?.SendDebug(testInputValuesA.ToString("Grad")); RILogManager.Default?.SendDebug("l1-l4 sum back"); RILogManager.Default?.SendDebug(testInputValuesB.ToString("Grad")); l0.Update(); //Although the format is irregular, since 10 contains SGD sgd.Update(); // Use stochastic gradient descent as the optimizer RILogManager.Default?.SendDebug("l0 Weight"); RILogManager.Default?.SendDebug(l0.Weight.ToString()); RILogManager.Default?.SendDebug("l1 Weight"); RILogManager.Default?.SendDebug(l1.Weight.ToString()); RILogManager.Default?.SendDebug("l0 Bias"); RILogManager.Default?.SendDebug(l0.Bias.ToString()); RILogManager.Default?.SendDebug("l1 Bias"); RILogManager.Default?.SendDebug(l1.Bias.ToString()); }
public static void Run() { //読み込みたいネットワークの構成を FunctionStack に書き連ね、各 Function のパラメータを合わせる //ここで必ず name を Chainer の変数名に合わせておくこと FunctionStack <Real> nn = new FunctionStack <Real>( new Convolution2D <Real>(1, 2, 3, name: "conv1", gpuEnable: true),//必要であればGPUフラグも忘れずに new ReLU <Real>(), new MaxPooling2D <Real>(2, 2), new Convolution2D <Real>(2, 2, 2, name: "conv2", gpuEnable: true), new ReLU <Real>(), new MaxPooling2D <Real>(2, 2), new Linear <Real>(8, 2, name: "fl3"), new ReLU <Real>(), new Linear <Real>(2, 2, name: "fl4") ); /* Chainerでの宣言 * class NN(chainer.Chain): * def __init__(self): * super(NN, self).__init__( * conv1 = L.Convolution2D(1,2,3), * conv2 = L.Convolution2D(2,2,2), * fl3 = L.Linear(8,2), * fl4 = L.Linear(2,2) * ) * * def __call__(self, x): * h_conv1 = F.relu(self.conv1(x)) * h_pool1 = F.max_pooling_2d(h_conv1, 2) * h_conv2 = F.relu(self.conv2(h_pool1)) * h_pool2 = F.max_pooling_2d(h_conv2, 2) * h_fc1 = F.relu(self.fl3(h_pool2)) * y = self.fl4(h_fc1) * return y */ //パラメータを読み込み ChainerModelDataLoader.ModelLoad(MODEL_FILE_PATH, nn); //あとは通常通り使用する SGD <Real> sgd = new SGD <Real>(0.1f); sgd.SetUp(nn); //入力データ NdArray <Real> x = new NdArray <Real>(new Real[, , ] { { { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.9f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.8f, 0.9f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.1f, 0.8f, 0.5f, 0.8f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.3f, 0.3f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.1f, 0.0f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.7f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.4f, 0.8f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.8f, 0.4f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.8f, 0.3f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.8f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.3f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f } } }); //教師信号 NdArray <Real> t = new NdArray <Real>(new Real[] { 0.0f, 1.0f }); //訓練を実施 Trainer.Train(nn, x, t, new MeanSquaredError <Real>()); //結果表示用に退避 Convolution2D <Real> l2 = (Convolution2D <Real>)nn.Functions[0]; //Updateを実行するとgradが消費されてしまうため値を先に出力 Console.WriteLine("gw1"); Console.WriteLine(l2.Weight.ToString("Grad")); Console.WriteLine("gb1"); Console.WriteLine(l2.Bias.ToString("Grad")); //更新 sgd.Update(); Console.WriteLine("w1"); Console.WriteLine(l2.Weight); Console.WriteLine("b1"); Console.WriteLine(l2.Bias); }
public static void Run() { //Weightを分割の前と後で揃える Real[,] testWeightValues = { { -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }, { -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }, { -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }, { -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 } }, new Real[, ] { { -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 } }, new Real[, ] { { -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 } }, new Real[, ] { { -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } } }; Linear l0 = new Linear(5, 20, initialW: testWeightValues, name: "l0"); Linear l1 = new Linear(5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear l2 = new Linear(5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear l3 = new Linear(5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear l4 = new Linear(5, 5, initialW: testJaggWeightValues[3], name: "l4"); //FunctionにOptimizerを設定 l0.SetOptimizer(new SGD()); //OptimiserにFunctionを登録 SGD sgd = new SGD(); l1.SetOptimizer(sgd); l2.SetOptimizer(sgd); l3.SetOptimizer(sgd); l4.SetOptimizer(sgd); //入力は同値だがGradが加算されてしまうため分ける Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 }; NdArray testInputValuesA = new NdArray(testValue); NdArray testInputValuesB = new NdArray(testValue); Console.WriteLine("l0 for"); NdArray l0Result = l0.Forward(testInputValuesA)[0]; Console.WriteLine(l0Result); Console.WriteLine("\nl1 for"); NdArray l1Result = l1.Forward(testInputValuesB)[0]; Console.WriteLine(l1Result); Console.WriteLine("\nl2 for"); NdArray l2Result = l2.Forward(testInputValuesB)[0]; Console.WriteLine(l2Result); Console.WriteLine("\nl3 for"); NdArray l3Result = l3.Forward(testInputValuesB)[0]; Console.WriteLine(l3Result); Console.WriteLine("\nl4 for"); NdArray l4Result = l4.Forward(testInputValuesB)[0]; Console.WriteLine(l4Result); Console.WriteLine(); //適当なGrad値をでっち上げる l0Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03, 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04, -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04, -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; l1Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 }; l2Result.Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 }; l3Result.Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 }; l4Result.Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; //Backwardを実行 l0.Backward(l0Result); l1.Backward(l1Result); l2.Backward(l2Result); l3.Backward(l3Result); l4.Backward(l4Result); Console.WriteLine("\nl0 back"); Console.WriteLine(testInputValuesA.ToString("Grad")); Console.WriteLine("\nl1-l4 sum back"); Console.WriteLine(testInputValuesB.ToString("Grad")); l0.Update(); //書式が変則的だがl0はSGDを内包しているため sgd.Update(); //こちらはOptimizerに関数を登録して使用している Console.WriteLine("\nl0 Weight"); Console.WriteLine(l0.Weight); Console.WriteLine("\nl1 Weight"); Console.WriteLine(l1.Weight); Console.WriteLine("\nl0 Bias"); Console.WriteLine(l0.Bias); Console.WriteLine("\nl1 Bias"); Console.WriteLine(l1.Bias); }
public void SGDRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 50); int outputCount = Mother.Dice.Next(2, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, inputCount }); Real[,] dummyGy = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, outputCount }); Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, inputCount }); Real[] b = Initializer.GetRealArray(outputCount); //Chainer NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b)); NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>(); cSgd.Setup(cLinear); Variable <Real> cX = new Variable <Real>(Real.ToBaseNdArray(input)); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = Real.ToBaseNdArray(dummyGy); cY.Backward(); cSgd.Update(); //KelpNet KelpNet.Linear linear = new KelpNet.Linear(inputCount, outputCount, false, w, b); KelpNet.SGD sgd = new SGD(); sgd.SetUp(linear); NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount); NdArray y = linear.Forward(x)[0]; y.Grad = Real.ToRealArray(dummyGy); y.Backward(); sgd.Update(); Real[] cW = Real.ToRealArray((Real[, ])cLinear.W.Data); Real[] cb = (Real[])cLinear.b.Data; //許容範囲を算出 double delta = 0.00001; //W.grad Assert.AreEqual(cW.Length, linear.Weight.Data.Length); for (int i = 0; i < linear.Weight.Data.Length; i++) { Assert.AreEqual(cW[i], linear.Weight.Data[i], delta); } //b.grad Assert.AreEqual(cb.Length, linear.Bias.Data.Length); for (int i = 0; i < linear.Bias.Data.Length; i++) { Assert.AreEqual(cb[i], linear.Bias.Data[i], delta); } }
public static void Run() { //Weightを分割の前と後で揃える Real[,] testWeightValues = new Real[, ] { { -0.02690255f, 0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f }, { -0.06963444f, -0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f }, { -0.04966073f, -0.04697048f, -0.02235234f, -0.09396666f, 0.073189f }, { 0.06563969f, 0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f }, { 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f }, { -0.05200623f, -0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f }, { -0.0254951f, 0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f }, { -0.0732494f, 0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f }, { -0.06818873f, -0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f }, { -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f }, { -0.0454098f, 0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f }, { -0.00866754f, -0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f }, { -0.01789357f, -0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f }, { -0.02421552f, 0.05533903f, -0.08627617f, 0.094624f, 0.03319318f }, { 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f }, { -0.07128382f, 0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f }, { 0.09431138f, 0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f }, { 0.01844446f, -0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f }, { -0.0261334f, -0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f }, { 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255f,0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f }, { -0.06963444f,-0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f }, { -0.04966073f,-0.04697048f, -0.02235234f, -0.09396666f, 0.073189f }, { 0.06563969f,0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f }, { 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f } }, new Real[, ] { { -0.05200623f,-0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f }, { -0.0254951f,0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f }, { -0.0732494f,0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f }, { -0.06818873f,-0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f }, { -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f } }, new Real[, ] { { -0.0454098f,0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f }, { -0.00866754f,-0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f }, { -0.01789357f,-0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f }, { -0.02421552f,0.05533903f, -0.08627617f, 0.094624f, 0.03319318f }, { 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f } }, new Real[, ] { { -0.07128382f,0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f }, { 0.09431138f,0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f }, { 0.01844446f,-0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f }, { -0.0261334f,-0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f }, { 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f } } }; Linear <Real> l0 = new Linear <Real>(5, 20, initialW: testWeightValues, name: "l0"); Linear <Real> l1 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear <Real> l2 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear <Real> l3 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear <Real> l4 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[3], name: "l4"); //FunctionにOptimizerを設定 SGD <Real> sgd = new SGD <Real>(); sgd.SetUp(l0); //OptimiserにFunctionを登録 SGD <Real> sgdSplit = new SGD <Real>(); sgdSplit.SetUp(l1); sgdSplit.SetUp(l2); sgdSplit.SetUp(l3); sgdSplit.SetUp(l4); //入力は同値だがGradが加算されてしまうため分ける Real[] testValue = new Real[] { 0.01618112f, -0.08296648f, -0.05545357f, 0.00389254f, -0.05727582f }; NdArray <Real> testInputValuesA = new NdArray <Real>(testValue); NdArray <Real> testInputValuesB = new NdArray <Real>(testValue); Console.WriteLine("l0 for"); NdArray <Real> l0Result = l0.Forward(testInputValuesA)[0]; Console.WriteLine(l0Result); Console.WriteLine("\nl1 for"); NdArray <Real> l1Result = l1.Forward(testInputValuesB)[0]; Console.WriteLine(l1Result); Console.WriteLine("\nl2 for"); NdArray <Real> l2Result = l2.Forward(testInputValuesB)[0]; Console.WriteLine(l2Result); Console.WriteLine("\nl3 for"); NdArray <Real> l3Result = l3.Forward(testInputValuesB)[0]; Console.WriteLine(l3Result); Console.WriteLine("\nl4 for"); NdArray <Real> l4Result = l4.Forward(testInputValuesB)[0]; Console.WriteLine(l4Result); Console.WriteLine(); //適当なGrad値をでっち上げる l0Result.Grad = new Real[] { -2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f, 1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f, -4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f, -4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f }; l1Result.Grad = new Real[] { -2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f }; l2Result.Grad = new Real[] { 1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f }; l3Result.Grad = new Real[] { -4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f }; l4Result.Grad = new Real[] { -4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f }; //Backwardを実行 l0.Backward(l0Result); l1.Backward(l1Result); l2.Backward(l2Result); l3.Backward(l3Result); l4.Backward(l4Result); Console.WriteLine("\nl0 back"); Console.WriteLine(testInputValuesA.ToString("Grad")); Console.WriteLine("\nl1-l4 sum back"); Console.WriteLine(testInputValuesB.ToString("Grad")); sgd.Update(); sgdSplit.Update(); Console.WriteLine("\nl0 Weight"); Console.WriteLine(l0.Weight); Console.WriteLine("\nl1 Weight"); Console.WriteLine(l1.Weight); Console.WriteLine("\nl0 Bias"); Console.WriteLine(l0.Bias); Console.WriteLine("\nl1 Bias"); Console.WriteLine(l1.Bias); }
public void SGDRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 50); int outputCount = Mother.Dice.Next(2, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount); Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount); Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount); Real[] b = Initializer.GetRandomValues <Real[]>(outputCount); //Chainer Linear <Real> cLinear = new Linear <Real>(inputCount, outputCount, false, w, b); NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>(); cSgd.Setup(cLinear); Variable <Real> cX = new Variable <Real>(input); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = dummyGy; cY.Backward(); cSgd.Update(); //KelpNet CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b); KelpNet.SGD <Real> sgd = new SGD <Real>(); sgd.SetUp(linear); NdArray <Real> x = new NdArray <Real>(input, asBatch: true); NdArray <Real> y = linear.Forward(x)[0]; y.Grad = dummyGy.Flatten(); y.Backward(); sgd.Update(); Real[] cW = ((Real[, ])cLinear.W.Data).Flatten(); Real[] cb = (Real[])cLinear.b.Data; //許容範囲を算出 Real delta = 0.00001f; //W.grad Assert.AreEqual(cW.Length, linear.Weight.Data.Length); for (int i = 0; i < linear.Weight.Data.Length; i++) { Assert.AreEqual(cW[i], linear.Weight.Data[i], delta); } //b.grad Assert.AreEqual(cb.Length, linear.Bias.Data.Length); for (int i = 0; i < linear.Bias.Data.Length; i++) { Assert.AreEqual(cb[i], linear.Bias.Data[i], delta); } }
public static void Run() { Console.WriteLine("Build Vocabulary."); Vocabulary vocabulary = new Vocabulary(); string trainPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE, TRAIN_FILE_HASH); string validPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + VALID_FILE, VALID_FILE, VALID_FILE_HASH); string testPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TEST_FILE, TEST_FILE, TEST_FILE_HASH); int[] trainData = vocabulary.LoadData(trainPath); int[] validData = vocabulary.LoadData(validPath); int[] testData = vocabulary.LoadData(testPath); int nVocab = vocabulary.Length; Console.WriteLine("Network Initilizing."); FunctionStack <Real> model = new FunctionStack <Real>( new EmbedID <Real>(nVocab, N_UNITS, name: "l1 EmbedID"), new Dropout <Real>(), new LSTM <Real>(N_UNITS, N_UNITS, name: "l2 LSTM"), new Dropout <Real>(), new LSTM <Real>(N_UNITS, N_UNITS, name: "l3 LSTM"), new Dropout <Real>(), new Linear <Real>(N_UNITS, nVocab, name: "l4 Linear") ); for (int i = 0; i < model.Functions.Length; i++) { for (int j = 0; j < model.Functions[i].Parameters.Length; j++) { for (int k = 0; k < model.Functions[i].Parameters[j].Data.Length; k++) { model.Functions[i].Parameters[j].Data[k] = ((Real)Mother.Dice.NextDouble() * 2.0f - 1.0f) / 10.0f; } } } //与えられたthresholdで頭打ちではなく、全パラメータのL2Normからレートを取り補正を行う GradientClipping <Real> gradientClipping = new GradientClipping <Real>(threshold: GRAD_CLIP); SGD <Real> sgd = new SGD <Real>(learningRate: 0.1f); gradientClipping.SetUp(model); sgd.SetUp(model); Real wholeLen = trainData.Length; int jump = (int)Math.Floor(wholeLen / BATCH_SIZE); int epoch = 0; Console.WriteLine("Train Start."); for (int i = 0; i < jump * N_EPOCH; i++) { NdArray <Real> x = new NdArray <Real>(new[] { 1 }, BATCH_SIZE); NdArray <int> t = new NdArray <int>(new[] { 1 }, BATCH_SIZE); for (int j = 0; j < BATCH_SIZE; j++) { x.Data[j] = trainData[(int)((jump * j + i) % wholeLen)]; t.Data[j] = trainData[(int)((jump * j + i + 1) % wholeLen)]; } NdArray <Real> result = model.Forward(x)[0]; Real sumLoss = new SoftmaxCrossEntropy <Real>().Evaluate(result, t); Console.WriteLine("[{0}/{1}] Loss: {2}", i + 1, jump, sumLoss); model.Backward(result); //Run truncated BPTT if ((i + 1) % BPROP_LEN == 0) { gradientClipping.Update(); sgd.Update(); model.ResetState(); } if ((i + 1) % jump == 0) { epoch++; Console.WriteLine("evaluate"); Console.WriteLine("validation perplexity: {0}", Evaluate(model, validData)); if (epoch >= 6) { sgd.LearningRate /= 1.2f; Console.WriteLine("learning rate =" + sgd.LearningRate); } } } Console.WriteLine("test start"); Console.WriteLine("test perplexity:" + Evaluate(model, testData)); }