public Sample13WinForm()
{
this.InitializeComponent();
ClientSize = new Size(128 * 4, 128 * 4);
//目標とするフィルタを作成（実践であればココは不明な値となる）
this.decon_core = new Deconvolution2D <Real>(1, 1, 15, 1, 7, gpuEnable: true)
{
Weight = { Data = MakeOneCore() }
};
this.model = new Deconvolution2D <Real>(1, 1, 15, 1, 7, gpuEnable: true);
this.optimizer = new SGD <Real>(learningRate: 0.01f);
optimizer.SetUp(this.model);
}
public static void Run()
{
//読み込みたいネットワークの構成を FunctionStack に書き連ね、各 Function のパラメータを合わせる
//ここで必ず name を Chainer の変数名に合わせておくこと
FunctionStack <Real> nn = new FunctionStack <Real>(
new Convolution2D <Real>(1, 2, 3, name: "conv1", gpuEnable: true),//必要であればGPUフラグも忘れずに
new ReLU <Real>(),
new MaxPooling2D <Real>(2, 2),
new Convolution2D <Real>(2, 2, 2, name: "conv2", gpuEnable: true),
new ReLU <Real>(),
new MaxPooling2D <Real>(2, 2),
new Linear <Real>(8, 2, name: "fl3"),
new ReLU <Real>(),
new Linear <Real>(2, 2, name: "fl4")
);
/* Chainerでの宣言
* class NN(chainer.Chain):
* def __init__(self):
* super(NN, self).__init__(
* conv1 = L.Convolution2D(1,2,3),
* conv2 = L.Convolution2D(2,2,2),
* fl3 = L.Linear(8,2),
* fl4 = L.Linear(2,2)
* )
*
* def __call__(self, x):
* h_conv1 = F.relu(self.conv1(x))
* h_pool1 = F.max_pooling_2d(h_conv1, 2)
* h_conv2 = F.relu(self.conv2(h_pool1))
* h_pool2 = F.max_pooling_2d(h_conv2, 2)
* h_fc1 = F.relu(self.fl3(h_pool2))
* y = self.fl4(h_fc1)
* return y
*/
//パラメータを読み込み
ChainerModelDataLoader.ModelLoad(MODEL_FILE_PATH, nn);
//あとは通常通り使用する
SGD <Real> sgd = new SGD <Real>(0.1f);
sgd.SetUp(nn);
//入力データ
NdArray <Real> x = new NdArray <Real>(new Real[, , ] {
{
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.9f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.8f, 0.9f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.1f, 0.8f, 0.5f, 0.8f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.3f, 0.3f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.1f, 0.0f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.7f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.4f, 0.8f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.8f, 0.4f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.2f, 0.8f, 0.3f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.8f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.7f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f },
{ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.3f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }
}
});
//教師信号
NdArray <Real> t = new NdArray <Real>(new Real[] { 0.0f, 1.0f });
//訓練を実施
Trainer.Train(nn, x, t, new MeanSquaredError <Real>());
//結果表示用に退避
Convolution2D <Real> l2 = (Convolution2D <Real>)nn.Functions[0];
//Updateを実行するとgradが消費されてしまうため値を先に出力
Console.WriteLine("gw1");
Console.WriteLine(l2.Weight.ToString("Grad"));
Console.WriteLine("gb1");
Console.WriteLine(l2.Bias.ToString("Grad"));
//更新
sgd.Update();
Console.WriteLine("w1");
Console.WriteLine(l2.Weight);
Console.WriteLine("b1");
Console.WriteLine(l2.Bias);
}
public void SGDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, inputCount });
Real[,] dummyGy = (Real[, ])Initializer.GetRealNdArray(new[] { batchCount, outputCount });
Real[,] w = (Real[, ])Initializer.GetRealNdArray(new[] { outputCount, inputCount });
Real[] b = Initializer.GetRealArray(outputCount);
//Chainer
NChainer.Linear <Real> cLinear = new NChainer.Linear <Real>(inputCount, outputCount, false, Real.ToBaseNdArray(w), Real.ToBaseArray(b));
NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>();
cSgd.Setup(cLinear);
Variable <Real> cX = new Variable <Real>(Real.ToBaseNdArray(input));
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = Real.ToBaseNdArray(dummyGy);
cY.Backward();
cSgd.Update();
//KelpNet
KelpNet.Linear linear = new KelpNet.Linear(inputCount, outputCount, false, w, b);
KelpNet.SGD sgd = new SGD();
sgd.SetUp(linear);
NdArray x = new NdArray(Real.ToRealArray(input), new[] { inputCount }, batchCount);
NdArray y = linear.Forward(x)[0];
y.Grad = Real.ToRealArray(dummyGy);
y.Backward();
sgd.Update();
Real[] cW = Real.ToRealArray((Real[, ])cLinear.W.Data);
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
double delta = 0.00001;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
public static void Run()
{
//Weightを分割の前と後で揃える
Real[,] testWeightValues = new Real[, ] {
{ -0.02690255f, 0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f },
{ -0.06963444f, -0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f },
{ -0.04966073f, -0.04697048f, -0.02235234f, -0.09396666f, 0.073189f },
{ 0.06563969f, 0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f },
{ 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f },
{ -0.05200623f, -0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f },
{ -0.0254951f, 0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f },
{ -0.0732494f, 0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f },
{ -0.06818873f, -0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f },
{ -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f },
{ -0.0454098f, 0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f },
{ -0.00866754f, -0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f },
{ -0.01789357f, -0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f },
{ -0.02421552f, 0.05533903f, -0.08627617f, 0.094624f, 0.03319318f },
{ 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f },
{ -0.07128382f, 0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f },
{ 0.09431138f, 0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f },
{ 0.01844446f, -0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f },
{ -0.0261334f, -0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f },
{ 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f }
};
Real[][,] testJaggWeightValues =
{
new Real[, ] {
{ -0.02690255f,0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f },
{ -0.06963444f,-0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f },
{ -0.04966073f,-0.04697048f, -0.02235234f, -0.09396666f, 0.073189f },
{ 0.06563969f,0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f },
{ 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f }
},
new Real[, ] {
{ -0.05200623f,-0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f },
{ -0.0254951f,0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f },
{ -0.0732494f,0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f },
{ -0.06818873f,-0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f },
{ -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f }
},
new Real[, ] {
{ -0.0454098f,0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f },
{ -0.00866754f,-0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f },
{ -0.01789357f,-0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f },
{ -0.02421552f,0.05533903f, -0.08627617f, 0.094624f, 0.03319318f },
{ 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f }
},
new Real[, ] {
{ -0.07128382f,0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f },
{ 0.09431138f,0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f },
{ 0.01844446f,-0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f },
{ -0.0261334f,-0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f },
{ 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f }
}
};
Linear <Real> l0 = new Linear <Real>(5, 20, initialW: testWeightValues, name: "l0");
Linear <Real> l1 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[0], name: "l1");
Linear <Real> l2 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[1], name: "l2");
Linear <Real> l3 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[2], name: "l3");
Linear <Real> l4 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[3], name: "l4");
//FunctionにOptimizerを設定
SGD <Real> sgd = new SGD <Real>();
sgd.SetUp(l0);
//OptimiserにFunctionを登録
SGD <Real> sgdSplit = new SGD <Real>();
sgdSplit.SetUp(l1);
sgdSplit.SetUp(l2);
sgdSplit.SetUp(l3);
sgdSplit.SetUp(l4);
//入力は同値だがGradが加算されてしまうため分ける
Real[] testValue = new Real[] { 0.01618112f, -0.08296648f, -0.05545357f, 0.00389254f, -0.05727582f };
NdArray <Real> testInputValuesA = new NdArray <Real>(testValue);
NdArray <Real> testInputValuesB = new NdArray <Real>(testValue);
Console.WriteLine("l0 for");
NdArray <Real> l0Result = l0.Forward(testInputValuesA)[0];
Console.WriteLine(l0Result);
Console.WriteLine("\nl1 for");
NdArray <Real> l1Result = l1.Forward(testInputValuesB)[0];
Console.WriteLine(l1Result);
Console.WriteLine("\nl2 for");
NdArray <Real> l2Result = l2.Forward(testInputValuesB)[0];
Console.WriteLine(l2Result);
Console.WriteLine("\nl3 for");
NdArray <Real> l3Result = l3.Forward(testInputValuesB)[0];
Console.WriteLine(l3Result);
Console.WriteLine("\nl4 for");
NdArray <Real> l4Result = l4.Forward(testInputValuesB)[0];
Console.WriteLine(l4Result);
Console.WriteLine();
//適当なGrad値をでっち上げる
l0Result.Grad = new Real[]
{
-2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f,
1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f,
-4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f,
-4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f
};
l1Result.Grad = new Real[] { -2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f };
l2Result.Grad = new Real[] { 1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f };
l3Result.Grad = new Real[] { -4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f };
l4Result.Grad = new Real[] { -4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f };
//Backwardを実行
l0.Backward(l0Result);
l1.Backward(l1Result);
l2.Backward(l2Result);
l3.Backward(l3Result);
l4.Backward(l4Result);
Console.WriteLine("\nl0 back");
Console.WriteLine(testInputValuesA.ToString("Grad"));
Console.WriteLine("\nl1-l4 sum back");
Console.WriteLine(testInputValuesB.ToString("Grad"));
sgd.Update();
sgdSplit.Update();
Console.WriteLine("\nl0 Weight");
Console.WriteLine(l0.Weight);
Console.WriteLine("\nl1 Weight");
Console.WriteLine(l1.Weight);
Console.WriteLine("\nl0 Bias");
Console.WriteLine(l0.Bias);
Console.WriteLine("\nl1 Bias");
Console.WriteLine(l1.Bias);
}
public void SGDRandomTest()
{
Python.Initialize();
Chainer.Initialize();
int inputCount = Mother.Dice.Next(2, 50);
int outputCount = Mother.Dice.Next(2, 50);
int batchCount = Mother.Dice.Next(1, 5);
Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount);
Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount);
Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount);
Real[] b = Initializer.GetRandomValues <Real[]>(outputCount);
//Chainer
Linear <Real> cLinear = new Linear <Real>(inputCount, outputCount, false, w, b);
NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>();
cSgd.Setup(cLinear);
Variable <Real> cX = new Variable <Real>(input);
Variable <Real> cY = cLinear.Forward(cX);
cY.Grad = dummyGy;
cY.Backward();
cSgd.Update();
//KelpNet
CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b);
KelpNet.SGD <Real> sgd = new SGD <Real>();
sgd.SetUp(linear);
NdArray <Real> x = new NdArray <Real>(input, asBatch: true);
NdArray <Real> y = linear.Forward(x)[0];
y.Grad = dummyGy.Flatten();
y.Backward();
sgd.Update();
Real[] cW = ((Real[, ])cLinear.W.Data).Flatten();
Real[] cb = (Real[])cLinear.b.Data;
//許容範囲を算出
Real delta = 0.00001f;
//W.grad
Assert.AreEqual(cW.Length, linear.Weight.Data.Length);
for (int i = 0; i < linear.Weight.Data.Length; i++)
{
Assert.AreEqual(cW[i], linear.Weight.Data[i], delta);
}
//b.grad
Assert.AreEqual(cb.Length, linear.Bias.Data.Length);
for (int i = 0; i < linear.Bias.Data.Length; i++)
{
Assert.AreEqual(cb[i], linear.Bias.Data[i], delta);
}
}
public static void Run()
{
Console.WriteLine("Build Vocabulary.");
Vocabulary vocabulary = new Vocabulary();
string trainPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE, TRAIN_FILE_HASH);
string validPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + VALID_FILE, VALID_FILE, VALID_FILE_HASH);
string testPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TEST_FILE, TEST_FILE, TEST_FILE_HASH);
int[] trainData = vocabulary.LoadData(trainPath);
int[] validData = vocabulary.LoadData(validPath);
int[] testData = vocabulary.LoadData(testPath);
int nVocab = vocabulary.Length;
Console.WriteLine("Network Initilizing.");
FunctionStack <Real> model = new FunctionStack <Real>(
new EmbedID <Real>(nVocab, N_UNITS, name: "l1 EmbedID"),
new Dropout <Real>(),
new LSTM <Real>(N_UNITS, N_UNITS, name: "l2 LSTM"),
new Dropout <Real>(),
new LSTM <Real>(N_UNITS, N_UNITS, name: "l3 LSTM"),
new Dropout <Real>(),
new Linear <Real>(N_UNITS, nVocab, name: "l4 Linear")
);
for (int i = 0; i < model.Functions.Length; i++)
{
for (int j = 0; j < model.Functions[i].Parameters.Length; j++)
{
for (int k = 0; k < model.Functions[i].Parameters[j].Data.Length; k++)
{
model.Functions[i].Parameters[j].Data[k] = ((Real)Mother.Dice.NextDouble() * 2.0f - 1.0f) / 10.0f;
}
}
}
//与えられたthresholdで頭打ちではなく、全パラメータのL2Normからレートを取り補正を行う
GradientClipping <Real> gradientClipping = new GradientClipping <Real>(threshold: GRAD_CLIP);
SGD <Real> sgd = new SGD <Real>(learningRate: 0.1f);
gradientClipping.SetUp(model);
sgd.SetUp(model);
Real wholeLen = trainData.Length;
int jump = (int)Math.Floor(wholeLen / BATCH_SIZE);
int epoch = 0;
Console.WriteLine("Train Start.");
for (int i = 0; i < jump * N_EPOCH; i++)
{
NdArray <Real> x = new NdArray <Real>(new[] { 1 }, BATCH_SIZE);
NdArray <int> t = new NdArray <int>(new[] { 1 }, BATCH_SIZE);
for (int j = 0; j < BATCH_SIZE; j++)
{
x.Data[j] = trainData[(int)((jump * j + i) % wholeLen)];
t.Data[j] = trainData[(int)((jump * j + i + 1) % wholeLen)];
}
NdArray <Real> result = model.Forward(x)[0];
Real sumLoss = new SoftmaxCrossEntropy <Real>().Evaluate(result, t);
Console.WriteLine("[{0}/{1}] Loss: {2}", i + 1, jump, sumLoss);
model.Backward(result);
//Run truncated BPTT
if ((i + 1) % BPROP_LEN == 0)
{
gradientClipping.Update();
sgd.Update();
model.ResetState();
}
if ((i + 1) % jump == 0)
{
epoch++;
Console.WriteLine("evaluate");
Console.WriteLine("validation perplexity: {0}", Evaluate(model, validData));
if (epoch >= 6)
{
sgd.LearningRate /= 1.2f;
Console.WriteLine("learning rate =" + sgd.LearningRate);
}
}
}
Console.WriteLine("test start");
Console.WriteLine("test perplexity:" + Evaluate(model, testData));
}
