public override TorchTensor Forward(TorchTensor input) { using (var x = fb.Forward(input)) if (_isTrue) { return(fbT1.Forward(x)); } else { return(fbF2.Forward(fbF1.Forward(x))); } }
public override TorchTensor Forward(TorchTensor input) { using (var l11 = conv1.Forward(input)) using (var l12 = MaxPool2D(l11, kernelSize: new long[] { 2 })) using (var l13 = Relu(l12)) using (var l21 = conv2.Forward(l13)) using (var l22 = FeatureAlphaDropout(l21)) using (var l23 = MaxPool2D(l22, kernelSize: new long[] { 2 })) using (var l24 = Relu(l23)) using (var x = l24.View(new long[] { -1, 320 })) using (var l31 = fc1.Forward(x)) using (var l32 = Relu(l31)) using (var l33 = Dropout(l32)) using (var l41 = fc2.Forward(l33)) return(LogSoftMax(l41, 1)); }
public static void Run() { Stopwatch sw = new Stopwatch(); NdArray inputArrayCpu = new NdArray(Initializer.GetRealArray(INPUT_SIZE)); NdArray inputArrayGpu = new NdArray(Initializer.GetRealArray(INPUT_SIZE)); //Linear Linear linear = new Linear(INPUT_SIZE, OUTPUT_SIZE); Console.WriteLine("◆" + linear.Name); sw.Restart(); NdArray[] gradArrayCpu = linear.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; //DataをGradとして使用 sw.Restart(); linear.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (linear.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = linear.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); linear.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Tanh TanhActivation tanh = new TanhActivation(); Console.WriteLine("\n◆" + tanh.Name); sw.Restart(); gradArrayCpu = tanh.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); tanh.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (tanh.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = tanh.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); tanh.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Sigmoid Sigmoid sigmoid = new Sigmoid(); Console.WriteLine("\n◆" + sigmoid.Name); sw.Restart(); gradArrayCpu = sigmoid.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); sigmoid.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (sigmoid.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = sigmoid.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); sigmoid.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //ReLU ReLU relu = new ReLU(); Console.WriteLine("\n◆" + relu.Name); sw.Restart(); gradArrayCpu = relu.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); relu.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (relu.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = relu.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); relu.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //LeakyReLU LeakyReLU leakyRelu = new LeakyReLU(); Console.WriteLine("\n◆" + leakyRelu.Name); sw.Restart(); gradArrayCpu = leakyRelu.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); leakyRelu.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (leakyRelu.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = leakyRelu.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); leakyRelu.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } NdArray inputImageArrayGpu = new NdArray(Initializer.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5); NdArray inputImageArrayCpu = new NdArray(Initializer.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5); //MaxPooling MaxPooling2D maxPooling2D = new MaxPooling2D(3); Console.WriteLine("\n◆" + maxPooling2D.Name); sw.Restart(); NdArray[] gradImageArrayCpu = maxPooling2D.Forward(inputImageArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); maxPooling2D.Backward(gradImageArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (maxPooling2D.SetGpuEnable(true)) { sw.Restart(); maxPooling2D.Forward(inputImageArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); //メモリ転送のみのため実装がない Console.WriteLine("Backward[Gpu] : None"); } //Conv2D Convolution2D conv2d = new Convolution2D(3, 3, 3); Console.WriteLine("\n◆" + conv2d.Name); sw.Restart(); gradImageArrayCpu = conv2d.Forward(inputImageArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); conv2d.Backward(gradImageArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (conv2d.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradImageArrayGpu = conv2d.Forward(inputImageArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data; sw.Restart(); conv2d.Backward(gradImageArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Deconv2D Deconvolution2D deconv2d = new Deconvolution2D(3, 3, 3); Console.WriteLine("\n◆" + deconv2d.Name); sw.Restart(); gradImageArrayCpu = deconv2d.Forward(inputImageArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); deconv2d.Backward(gradImageArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (deconv2d.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradImageArrayGpu = deconv2d.Forward(inputImageArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data; sw.Restart(); deconv2d.Backward(gradImageArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Dropout Dropout dropout = new Dropout(); Console.WriteLine("\n◆" + dropout.Name); sw.Restart(); gradArrayCpu = dropout.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); dropout.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (dropout.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = dropout.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); dropout.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } }
public static void Run() { //Align Weight before and after splitting Real[,] testWeightValues = { { -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }, { -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }, { -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }, { -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 } }, new Real[, ] { { -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 } }, new Real[, ] { { -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 } }, new Real[, ] { { -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } } }; Linear l0 = new Linear(true, 5, 20, initialW: testWeightValues, name: "l0"); Linear l1 = new Linear(true, 5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear l2 = new Linear(true, 5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear l3 = new Linear(true, 5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear l4 = new Linear(true, 5, 5, initialW: testJaggWeightValues[3], name: "l4"); l0.SetOptimizer(new SGD()); SGD sgd = new SGD(); l1.SetOptimizer(sgd); l2.SetOptimizer(sgd); l3.SetOptimizer(sgd); l4.SetOptimizer(sgd); //Input is equivalent, but Grad is added and it is divided Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 }; NdArray testInputValuesA = new NdArray(testValue); NdArray testInputValuesB = new NdArray(testValue); RILogManager.Default?.SendDebug("l0 for"); NdArray[] l0Result = l0.Forward(true, testInputValuesA); RILogManager.Default?.SendDebug(l0Result.ToString()); RILogManager.Default?.SendDebug("l1 for"); NdArray[] l1Result = l1.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l1Result.ToString()); RILogManager.Default?.SendDebug("l2 for"); NdArray[] l2Result = l2.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l2Result.ToString()); RILogManager.Default?.SendDebug("l3 for"); NdArray[] l3Result = l3.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l3Result.ToString()); RILogManager.Default?.SendDebug("l4 for"); NdArray[] l4Result = l4.Forward(true, testInputValuesB); RILogManager.Default?.SendDebug(l4Result.ToString()); //Create an appropriate Grad value l0Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03, 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04, -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04, -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; l1Result[0].Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 }; l2Result[0].Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 }; l3Result[0].Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 }; l4Result[0].Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; //Backward l0.Backward(true, l0Result); l1.Backward(true, l1Result); l2.Backward(true, l2Result); l3.Backward(true, l3Result); l4.Backward(true, l4Result); RILogManager.Default?.SendDebug("l0 back"); RILogManager.Default?.SendDebug(testInputValuesA.ToString("Grad")); RILogManager.Default?.SendDebug("l1-l4 sum back"); RILogManager.Default?.SendDebug(testInputValuesB.ToString("Grad")); l0.Update(); //Although the format is irregular, since 10 contains SGD sgd.Update(); // Use stochastic gradient descent as the optimizer RILogManager.Default?.SendDebug("l0 Weight"); RILogManager.Default?.SendDebug(l0.Weight.ToString()); RILogManager.Default?.SendDebug("l1 Weight"); RILogManager.Default?.SendDebug(l1.Weight.ToString()); RILogManager.Default?.SendDebug("l0 Bias"); RILogManager.Default?.SendDebug(l0.Bias.ToString()); RILogManager.Default?.SendDebug("l1 Bias"); RILogManager.Default?.SendDebug(l1.Bias.ToString()); }
public static void Run() { //Weightを分割の前と後で揃える Real[,] testWeightValues = { { -0.02690255, 0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444, -0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073, -0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969, 0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 }, { -0.05200623, -0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951, 0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494, 0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873, -0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 }, { -0.0454098, 0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754, -0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357, -0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552, 0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 }, { -0.07128382, 0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138, 0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446, -0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334, -0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255,0.08830735, -0.02041466, -0.0431439, -0.07749002 }, { -0.06963444,-0.03971611, 0.0597842, 0.08824182, -0.06649109 }, { -0.04966073,-0.04697048, -0.02235234, -0.09396666, 0.073189 }, { 0.06563969,0.04446745, -0.07192299, 0.06784364, 0.09575776 }, { 0.05012317, -0.08874852, -0.05977172, -0.05910181, -0.06009106 } }, new Real[, ] { { -0.05200623,-0.09679124, 0.02159978, -0.08058041, -0.01340541 }, { -0.0254951,0.09963084, 0.00936683, -0.08179696, 0.09604459 }, { -0.0732494,0.07253634, 0.05981455, -0.01007657, -0.02992892 }, { -0.06818873,-0.02579817, 0.06767359, -0.03379837, -0.04880046 }, { -0.06429326, -0.08964688, -0.0960066, -0.00286683, -0.05761427 } }, new Real[, ] { { -0.0454098,0.07809167, -0.05030088, -0.02533244, -0.02322736 }, { -0.00866754,-0.03614252, 0.05237325, 0.06478979, -0.03599609 }, { -0.01789357,-0.04479434, -0.05765592, 0.03237658, -0.06403019 }, { -0.02421552,0.05533903, -0.08627617, 0.094624, 0.03319318 }, { 0.02328842, -0.08234859, -0.07979888, 0.01439688, -0.03267198 } }, new Real[, ] { { -0.07128382,0.08531934, 0.07180037, 0.04772871, -0.08938966 }, { 0.09431138,0.02094762, 0.04443646, 0.07653841, 0.02028433 }, { 0.01844446,-0.08441339, 0.01957355, 0.04430714, -0.03080243 }, { -0.0261334,-0.03794889, -0.00638074, 0.07278767, -0.02165155 }, { 0.08390063, -0.03253863, 0.0311571, 0.08088892, -0.07267931 } } }; Linear l0 = new Linear(5, 20, initialW: testWeightValues, name: "l0"); Linear l1 = new Linear(5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear l2 = new Linear(5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear l3 = new Linear(5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear l4 = new Linear(5, 5, initialW: testJaggWeightValues[3], name: "l4"); //FunctionにOptimizerを設定 l0.SetOptimizer(new SGD()); //OptimiserにFunctionを登録 SGD sgd = new SGD(); l1.SetOptimizer(sgd); l2.SetOptimizer(sgd); l3.SetOptimizer(sgd); l4.SetOptimizer(sgd); //入力は同値だがGradが加算されてしまうため分ける Real[] testValue = { 0.01618112, -0.08296648, -0.05545357, 0.00389254, -0.05727582 }; NdArray testInputValuesA = new NdArray(testValue); NdArray testInputValuesB = new NdArray(testValue); Console.WriteLine("l0 for"); NdArray l0Result = l0.Forward(testInputValuesA)[0]; Console.WriteLine(l0Result); Console.WriteLine("\nl1 for"); NdArray l1Result = l1.Forward(testInputValuesB)[0]; Console.WriteLine(l1Result); Console.WriteLine("\nl2 for"); NdArray l2Result = l2.Forward(testInputValuesB)[0]; Console.WriteLine(l2Result); Console.WriteLine("\nl3 for"); NdArray l3Result = l3.Forward(testInputValuesB)[0]; Console.WriteLine(l3Result); Console.WriteLine("\nl4 for"); NdArray l4Result = l4.Forward(testInputValuesB)[0]; Console.WriteLine(l4Result); Console.WriteLine(); //適当なGrad値をでっち上げる l0Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03, 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04, -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04, -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; l1Result.Grad = new Real[] { -2.42022760e-02, 5.02482988e-04, 2.52015481e-04, 8.08797951e-04, -7.19293347e-03 }; l2Result.Grad = new Real[] { 1.40045900e-04, 7.09874439e-05, 2.07651625e-04, 3.80124636e-02, -8.87162634e-04 }; l3Result.Grad = new Real[] { -4.64874669e-04, -1.40792923e-03, -4.12280299e-02, -3.36557830e-04, -1.50323089e-04 }; l4Result.Grad = new Real[] { -4.70047118e-04, 3.61101292e-02, -7.12957408e-04, -3.63163825e-04, -1.12809543e-03 }; //Backwardを実行 l0.Backward(l0Result); l1.Backward(l1Result); l2.Backward(l2Result); l3.Backward(l3Result); l4.Backward(l4Result); Console.WriteLine("\nl0 back"); Console.WriteLine(testInputValuesA.ToString("Grad")); Console.WriteLine("\nl1-l4 sum back"); Console.WriteLine(testInputValuesB.ToString("Grad")); l0.Update(); //書式が変則的だがl0はSGDを内包しているため sgd.Update(); //こちらはOptimizerに関数を登録して使用している Console.WriteLine("\nl0 Weight"); Console.WriteLine(l0.Weight); Console.WriteLine("\nl1 Weight"); Console.WriteLine(l1.Weight); Console.WriteLine("\nl0 Bias"); Console.WriteLine(l0.Bias); Console.WriteLine("\nl1 Bias"); Console.WriteLine(l1.Bias); }
public static void Run() { //Weightを分割の前と後で揃える Real[,] testWeightValues = new Real[, ] { { -0.02690255f, 0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f }, { -0.06963444f, -0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f }, { -0.04966073f, -0.04697048f, -0.02235234f, -0.09396666f, 0.073189f }, { 0.06563969f, 0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f }, { 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f }, { -0.05200623f, -0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f }, { -0.0254951f, 0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f }, { -0.0732494f, 0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f }, { -0.06818873f, -0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f }, { -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f }, { -0.0454098f, 0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f }, { -0.00866754f, -0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f }, { -0.01789357f, -0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f }, { -0.02421552f, 0.05533903f, -0.08627617f, 0.094624f, 0.03319318f }, { 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f }, { -0.07128382f, 0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f }, { 0.09431138f, 0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f }, { 0.01844446f, -0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f }, { -0.0261334f, -0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f }, { 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f } }; Real[][,] testJaggWeightValues = { new Real[, ] { { -0.02690255f,0.08830735f, -0.02041466f, -0.0431439f, -0.07749002f }, { -0.06963444f,-0.03971611f, 0.0597842f, 0.08824182f, -0.06649109f }, { -0.04966073f,-0.04697048f, -0.02235234f, -0.09396666f, 0.073189f }, { 0.06563969f,0.04446745f, -0.07192299f, 0.06784364f, 0.09575776f }, { 0.05012317f, -0.08874852f, -0.05977172f, -0.05910181f, -0.06009106f } }, new Real[, ] { { -0.05200623f,-0.09679124f, 0.02159978f, -0.08058041f, -0.01340541f }, { -0.0254951f,0.09963084f, 0.00936683f, -0.08179696f, 0.09604459f }, { -0.0732494f,0.07253634f, 0.05981455f, -0.01007657f, -0.02992892f }, { -0.06818873f,-0.02579817f, 0.06767359f, -0.03379837f, -0.04880046f }, { -0.06429326f, -0.08964688f, -0.0960066f, -0.00286683f, -0.05761427f } }, new Real[, ] { { -0.0454098f,0.07809167f, -0.05030088f, -0.02533244f, -0.02322736f }, { -0.00866754f,-0.03614252f, 0.05237325f, 0.06478979f, -0.03599609f }, { -0.01789357f,-0.04479434f, -0.05765592f, 0.03237658f, -0.06403019f }, { -0.02421552f,0.05533903f, -0.08627617f, 0.094624f, 0.03319318f }, { 0.02328842f, -0.08234859f, -0.07979888f, 0.01439688f, -0.03267198f } }, new Real[, ] { { -0.07128382f,0.08531934f, 0.07180037f, 0.04772871f, -0.08938966f }, { 0.09431138f,0.02094762f, 0.04443646f, 0.07653841f, 0.02028433f }, { 0.01844446f,-0.08441339f, 0.01957355f, 0.04430714f, -0.03080243f }, { -0.0261334f,-0.03794889f, -0.00638074f, 0.07278767f, -0.02165155f }, { 0.08390063f, -0.03253863f, 0.0311571f, 0.08088892f, -0.07267931f } } }; Linear <Real> l0 = new Linear <Real>(5, 20, initialW: testWeightValues, name: "l0"); Linear <Real> l1 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[0], name: "l1"); Linear <Real> l2 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[1], name: "l2"); Linear <Real> l3 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[2], name: "l3"); Linear <Real> l4 = new Linear <Real>(5, 5, initialW: testJaggWeightValues[3], name: "l4"); //FunctionにOptimizerを設定 SGD <Real> sgd = new SGD <Real>(); sgd.SetUp(l0); //OptimiserにFunctionを登録 SGD <Real> sgdSplit = new SGD <Real>(); sgdSplit.SetUp(l1); sgdSplit.SetUp(l2); sgdSplit.SetUp(l3); sgdSplit.SetUp(l4); //入力は同値だがGradが加算されてしまうため分ける Real[] testValue = new Real[] { 0.01618112f, -0.08296648f, -0.05545357f, 0.00389254f, -0.05727582f }; NdArray <Real> testInputValuesA = new NdArray <Real>(testValue); NdArray <Real> testInputValuesB = new NdArray <Real>(testValue); Console.WriteLine("l0 for"); NdArray <Real> l0Result = l0.Forward(testInputValuesA)[0]; Console.WriteLine(l0Result); Console.WriteLine("\nl1 for"); NdArray <Real> l1Result = l1.Forward(testInputValuesB)[0]; Console.WriteLine(l1Result); Console.WriteLine("\nl2 for"); NdArray <Real> l2Result = l2.Forward(testInputValuesB)[0]; Console.WriteLine(l2Result); Console.WriteLine("\nl3 for"); NdArray <Real> l3Result = l3.Forward(testInputValuesB)[0]; Console.WriteLine(l3Result); Console.WriteLine("\nl4 for"); NdArray <Real> l4Result = l4.Forward(testInputValuesB)[0]; Console.WriteLine(l4Result); Console.WriteLine(); //適当なGrad値をでっち上げる l0Result.Grad = new Real[] { -2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f, 1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f, -4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f, -4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f }; l1Result.Grad = new Real[] { -2.42022760e-02f, 5.02482988e-04f, 2.52015481e-04f, 8.08797951e-04f, -7.19293347e-03f }; l2Result.Grad = new Real[] { 1.40045900e-04f, 7.09874439e-05f, 2.07651625e-04f, 3.80124636e-02f, -8.87162634e-04f }; l3Result.Grad = new Real[] { -4.64874669e-04f, -1.40792923e-03f, -4.12280299e-02f, -3.36557830e-04f, -1.50323089e-04f }; l4Result.Grad = new Real[] { -4.70047118e-04f, 3.61101292e-02f, -7.12957408e-04f, -3.63163825e-04f, -1.12809543e-03f }; //Backwardを実行 l0.Backward(l0Result); l1.Backward(l1Result); l2.Backward(l2Result); l3.Backward(l3Result); l4.Backward(l4Result); Console.WriteLine("\nl0 back"); Console.WriteLine(testInputValuesA.ToString("Grad")); Console.WriteLine("\nl1-l4 sum back"); Console.WriteLine(testInputValuesB.ToString("Grad")); sgd.Update(); sgdSplit.Update(); Console.WriteLine("\nl0 Weight"); Console.WriteLine(l0.Weight); Console.WriteLine("\nl1 Weight"); Console.WriteLine(l1.Weight); Console.WriteLine("\nl0 Bias"); Console.WriteLine(l0.Bias); Console.WriteLine("\nl1 Bias"); Console.WriteLine(l1.Bias); }
public void RnnLSTMRandomTest() { Python.Initialize(); Chainer.Initialize(); Real[,] input = { { 1.0f }, { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f } }; Real[,] teach = { { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f } }; Real[,] input2 = { { 3.0f }, { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f } }; Real[,] teach2 = { { 5.0f }, { 7.0f }, { 9.0f }, { 11.0f }, { 13.0f } }; int outputCount = 1; int inputCount = 1; int hiddenCount = 2; Real[,] upwardInit = Initializer.GetRandomValues <Real[, ]>(hiddenCount, hiddenCount); Real[,] lateralInit = Initializer.GetRandomValues <Real[, ]>(hiddenCount, hiddenCount); Real[,,] biasInit = Initializer.GetRandomValues <Real[, , ]>(1, hiddenCount, 1); Real[,,] forgetBiasInit = Initializer.GetRandomValues <Real[, , ]>(1, hiddenCount, 1); //Chainer Real[,] w1 = Initializer.GetRandomValues <Real[, ]>(hiddenCount, inputCount); Real[] b1 = Initializer.GetRandomValues <Real[]>(hiddenCount); //Chainer Linear <Real> cLinear1 = new Linear <Real>(inputCount, hiddenCount, false, w1, b1); NChainer.LSTM <Real> cLstm = new NChainer.LSTM <Real>(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit); Real[,] w2 = Initializer.GetRandomValues <Real[, ]>(outputCount, hiddenCount); Real[] b2 = Initializer.GetRandomValues <Real[]>(outputCount); Linear <Real> cLinear2 = new Linear <Real>(hiddenCount, outputCount, false, w2, b2); Variable <Real> cX1 = new Variable <Real>(input); Variable <Real> cY11 = cLinear1.Forward(cX1); Variable <Real> cY12 = cLstm.Forward(cY11); Variable <Real> cY13 = cLinear2.Forward(cY12); Variable <Real> cT = new Variable <Real>(teach); Variable <Real> cLoss = new NChainer.MeanSquaredError <Real>().Forward(cY13, cT); cLoss.Backward(); //KelpNet CL.Linear <Real> linear1 = new CL.Linear <Real>(inputCount, hiddenCount, false, w1, b1); LSTM <Real> lstm = new LSTM <Real>(hiddenCount, hiddenCount, lateralInit, upwardInit, biasInit, forgetBiasInit); CL.Linear <Real> linear2 = new CL.Linear <Real>(hiddenCount, outputCount, false, w2, b2); NdArray <Real> x1 = new NdArray <Real>(input, asBatch: true); NdArray <Real> y11 = linear1.Forward(x1)[0]; NdArray <Real> y12 = lstm.Forward(y11)[0]; NdArray <Real> y13 = linear2.Forward(y12)[0]; NdArray <Real> t = new NdArray <Real>(teach, asBatch: true); NdArray <Real> loss = new MeanSquaredError <Real>().Evaluate(y13, t); y13.Backward(); Real[] cY11data = ((Real[, ])cY11.Data).Flatten(); Real[] cY12data = ((Real[, ])cY12.Data).Flatten(); Real[] cY13data = ((Real[, ])cY13.Data).Flatten(); Real[] cXgrad = ((Real[, ])cX1.Grad).Flatten(); Real[] cupwardWGrad = ((Real[, ])cLstm.upward.W.Grad).Flatten(); Real[] cupwardbGrad = (Real[])cLstm.upward.b.Grad; //許容範囲を設定 Real delta = 0.00001f; //y11 Assert.AreEqual(cY11data.Length, y11.Data.Length); for (int i = 0; i < cY11data.Length; i++) { Assert.AreEqual(cY11data[i], y11.Data[i], delta); } //y12 Assert.AreEqual(cY12data.Length, y12.Data.Length); for (int i = 0; i < cY12data.Length; i++) { Assert.AreEqual(cY12data[i], y12.Data[i], delta); } //y13 Assert.AreEqual(cY13data.Length, y13.Data.Length); for (int i = 0; i < cY13data.Length; i++) { Assert.AreEqual(cY13data[i], y13.Data[i], delta); } //許容範囲を設定 delta = 0.0001f; //loss Assert.AreEqual(cLoss.Data[0], loss.Data[0], delta); //x.Grad Assert.AreEqual(cXgrad.Length, x1.Grad.Length); for (int i = 0; i < cXgrad.Length; i++) { Assert.AreEqual(cXgrad[i], x1.Grad[i], delta); } Real[] cWgrad11 = ((Real[, ])cLinear1.W.Grad).Flatten(); Real[] cbgrad11 = (Real[])cLinear1.b.Grad; //W.grad Assert.AreEqual(cWgrad11.Length, linear1.Weight.Grad.Length); for (int i = 0; i < linear1.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad11[i], linear1.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad11.Length, linear1.Bias.Grad.Length); for (int i = 0; i < linear1.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad11[i], linear1.Bias.Grad[i], delta); } Real[] cWgrad12 = ((Real[, ])cLinear2.W.Grad).Flatten(); Real[] cbgrad12 = (Real[])cLinear2.b.Grad; //W.grad Assert.AreEqual(cWgrad12.Length, linear2.Weight.Grad.Length); for (int i = 0; i < linear2.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad12[i], linear2.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad12.Length, linear2.Bias.Grad.Length); for (int i = 0; i < linear2.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad12[i], linear2.Bias.Grad[i], delta); } //W.grad int wLen = lstm.upward.Weight.Grad.Length; Assert.AreEqual(cupwardWGrad.Length, lstm.upward.Weight.Grad.Length); for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad int bLen = lstm.upward.Bias.Length; Assert.AreEqual(cupwardbGrad.Length, lstm.upward.Bias.Grad.Length); for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } //2周目 Variable <Real> cX2 = new Variable <Real>(input2); Variable <Real> cY21 = cLinear1.Forward(cX2); Variable <Real> cY22 = cLstm.Forward(cY21); Variable <Real> cY23 = cLinear2.Forward(cY22); Variable <Real> cT2 = new Variable <Real>(teach2); Variable <Real> cLoss2 = new NChainer.MeanSquaredError <Real>().Forward(cY23, cT2); //KelpNet NdArray <Real> x2 = new NdArray <Real>(input2, asBatch: true); NdArray <Real> y21 = linear1.Forward(x2)[0]; NdArray <Real> y22 = lstm.Forward(y21)[0]; NdArray <Real> y23 = linear2.Forward(y22)[0]; NdArray <Real> t2 = new NdArray <Real>(teach2, asBatch: true); NdArray <Real> loss2 = new MeanSquaredError <Real>().Evaluate(y23, t2); Assert.AreEqual(cLoss2.Data[0], loss2.Data[0], delta); //Backwardを実行 cLoss2.Backward(); y23.Backward(); Real[] cYdata21 = ((Real[, ])cY21.Data).Flatten(); Real[] cYdata22 = ((Real[, ])cY22.Data).Flatten(); Real[] cYdata23 = ((Real[, ])cY23.Data).Flatten(); Real[] cXgrad2 = ((Real[, ])cX2.Grad).Flatten(); Real[] cupwardWGrad2 = ((Real[, ])cLstm.upward.W.Grad).Flatten(); Real[] cupwardbGrad2 = (Real[])cLstm.upward.b.Grad; Real[] clateralWGrad = ((Real[, ])cLstm.lateral.W.Grad).Flatten(); //y21 Assert.AreEqual(cYdata21.Length, y21.Data.Length); for (int i = 0; i < cYdata21.Length; i++) { Assert.AreEqual(cYdata21[i], y21.Data[i], delta); } //y22 Assert.AreEqual(cYdata22.Length, y22.Data.Length); for (int i = 0; i < cYdata22.Length; i++) { Assert.AreEqual(cYdata22[i], y22.Data[i], delta); } //y23 Assert.AreEqual(cYdata23.Length, y23.Data.Length); for (int i = 0; i < cYdata23.Length; i++) { Assert.AreEqual(cYdata23[i], y23.Data[i], delta); } //x.Grad Assert.AreEqual(cXgrad2.Length, x2.Grad.Length); for (int i = 0; i < cXgrad2.Length; i++) { Assert.AreEqual(cXgrad2[i], x2.Grad[i], delta); } //経由が多くかなり誤差が大きい為 delta = 1.0f; Real[] cWgrad22 = ((Real[, ])cLinear2.W.Grad).Flatten(); Real[] cbgrad22 = (Real[])cLinear2.b.Grad; //W.grad Assert.AreEqual(cWgrad22.Length, linear2.Weight.Grad.Length); for (int i = 0; i < linear2.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad22[i], linear2.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad22.Length, linear2.Bias.Grad.Length); for (int i = 0; i < linear2.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad22[i], linear2.Bias.Grad[i], delta); } delta = 2.0f; //W.grad Assert.AreEqual(clateralWGrad.Length, lstm.lateral.Weight.Grad.Length); for (int i = 0; i < clateralWGrad.Length; i++) { Assert.AreEqual(clateralWGrad[i + wLen * 0], lstm.lateral.Weight.Grad[i], delta); } for (int i = 0; i < wLen; i++) { Assert.AreEqual(cupwardWGrad2[i + wLen * 0], lstm.upward.Weight.Grad[i], delta); } //b.grad for (int i = 0; i < bLen; i++) { Assert.AreEqual(cupwardbGrad2[i + wLen * 0], lstm.upward.Bias.Grad[i], delta); } delta = 20.0f; Real[] cWgrad21 = ((Real[, ])cLinear1.W.Grad).Flatten(); Real[] cbgrad21 = (Real[])cLinear1.b.Grad; //W.grad Assert.AreEqual(cWgrad21.Length, linear1.Weight.Grad.Length); for (int i = 0; i < linear1.Weight.Grad.Length; i++) { Assert.AreEqual(cWgrad21[i], linear1.Weight.Grad[i], delta); } //b.grad Assert.AreEqual(cbgrad21.Length, linear1.Bias.Grad.Length); for (int i = 0; i < linear1.Bias.Grad.Length; i++) { Assert.AreEqual(cbgrad21[i], linear1.Bias.Grad[i], delta); } }
public void SGDRandomTest() { Python.Initialize(); Chainer.Initialize(); int inputCount = Mother.Dice.Next(2, 50); int outputCount = Mother.Dice.Next(2, 50); int batchCount = Mother.Dice.Next(1, 5); Real[,] input = Initializer.GetRandomValues <Real[, ]>(batchCount, inputCount); Real[,] dummyGy = Initializer.GetRandomValues <Real[, ]>(batchCount, outputCount); Real[,] w = Initializer.GetRandomValues <Real[, ]>(outputCount, inputCount); Real[] b = Initializer.GetRandomValues <Real[]>(outputCount); //Chainer Linear <Real> cLinear = new Linear <Real>(inputCount, outputCount, false, w, b); NChainer.SGD <Real> cSgd = new NChainer.SGD <Real>(); cSgd.Setup(cLinear); Variable <Real> cX = new Variable <Real>(input); Variable <Real> cY = cLinear.Forward(cX); cY.Grad = dummyGy; cY.Backward(); cSgd.Update(); //KelpNet CL.Linear <Real> linear = new CL.Linear <Real>(inputCount, outputCount, false, w, b); KelpNet.SGD <Real> sgd = new SGD <Real>(); sgd.SetUp(linear); NdArray <Real> x = new NdArray <Real>(input, asBatch: true); NdArray <Real> y = linear.Forward(x)[0]; y.Grad = dummyGy.Flatten(); y.Backward(); sgd.Update(); Real[] cW = ((Real[, ])cLinear.W.Data).Flatten(); Real[] cb = (Real[])cLinear.b.Data; //許容範囲を算出 Real delta = 0.00001f; //W.grad Assert.AreEqual(cW.Length, linear.Weight.Data.Length); for (int i = 0; i < linear.Weight.Data.Length; i++) { Assert.AreEqual(cW[i], linear.Weight.Data[i], delta); } //b.grad Assert.AreEqual(cb.Length, linear.Bias.Data.Length); for (int i = 0; i < linear.Bias.Data.Length; i++) { Assert.AreEqual(cb[i], linear.Bias.Data[i], delta); } }
public static void Run(bool verbose) { Stopwatch sw = new Stopwatch(); NdArray inputArrayCpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE)); NdArray inputArrayGpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE)); Ensure.Argument(inputArrayGpu).NotNull(); Ensure.Argument(inputArrayCpu).NotNull(); //Linear Linear linear = new Linear(verbose, INPUT_SIZE, OUTPUT_SIZE); if (verbose) { RILogManager.Default?.EnterMethod(linear.Name); } sw.Restart(); NdArray[] gradArrayCpu = linear.Forward(verbose, inputArrayCpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } Ensure.Argument(gradArrayCpu).NotNull(); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; // Use Data as Grad sw.Restart(); linear.Backward(verbose, gradArrayCpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } if (linear.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = linear.Forward(verbose, inputArrayGpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); linear.Backward(verbose, gradArrayGpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } } if (verbose) { RILogManager.Default?.ExitMethod(linear.Name); } //Tanh Tanh tanh = new Tanh(); if (verbose) { RILogManager.Default?.EnterMethod(tanh.Name); } sw.Restart(); gradArrayCpu = tanh.Forward(verbose, inputArrayCpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); tanh.Backward(verbose, gradArrayCpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } if (tanh.SetGpuEnable(true)) { HandleGPU(verbose, sw, tanh, inputArrayGpu); } if (verbose) { RILogManager.Default?.ExitMethod(tanh.Name); } //Sigmoid Sigmoid sigmoid = new Sigmoid(); if (verbose) { RILogManager.Default?.EnterMethod(sigmoid.Name); } sw.Restart(); gradArrayCpu = sigmoid.Forward(verbose, inputArrayCpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); sigmoid.Backward(verbose, gradArrayCpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } if (sigmoid.SetGpuEnable(true)) { HandleGPU(verbose, sw, sigmoid, inputArrayGpu); } if (verbose) { RILogManager.Default?.ExitMethod(tanh.Name); } //Softmax Softmax sm = new Softmax(); RILogManager.Default?.EnterMethod(sm.Name); sw.Restart(); gradArrayCpu = sm.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); sm.Backward(verbose, gradArrayCpu); sw.Stop(); if (verbose) { RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } if (verbose) { RILogManager.Default?.ExitMethod(sm.Name); } //Softplus Softplus sp = new Softplus(); if (verbose) { RILogManager.Default?.EnterMethod(sp.Name); } sw.Restart(); gradArrayCpu = sp.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); sp.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); RILogManager.Default?.ExitMethod(sp.Name); //ReLU ReLU relu = new ReLU(); RILogManager.Default?.EnterMethod(relu.Name); sw.Restart(); gradArrayCpu = relu.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); relu.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (relu.SetGpuEnable(true)) { HandleGPU(verbose, sw, relu, inputArrayGpu); } RILogManager.Default?.ExitMethod(relu.Name); //LeakyReLU LeakyReLU leakyRelu = new LeakyReLU(); RILogManager.Default?.EnterMethod(leakyRelu.Name); sw.Restart(); gradArrayCpu = leakyRelu.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); leakyRelu.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (leakyRelu.SetGpuEnable(true)) { HandleGPU(verbose, sw, leakyRelu, inputArrayGpu); } RILogManager.Default?.ExitMethod(leakyRelu.Name); //ReLuTanh ReLuTanh rth = new ReLuTanh(); RILogManager.Default?.EnterMethod(rth.Name); sw.Restart(); gradArrayCpu = rth.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); rth.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (rth.SetGpuEnable(true)) { HandleGPU(verbose, sw, rth, inputArrayGpu); } RILogManager.Default?.ExitMethod(rth.Name); ////Swish //Swish swi = new Swish(); //RILogManager.Default?.SendDebug(swi.Name); //sw.Restart(); //gradArrayCpu = swi.Forward(inputArrayCpu); //sw.Stop(); //RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); //gradArrayCpu[0].Grad = gradArrayCpu[0].Data; //sw.Restart(); //swi.Backward(gradArrayCpu); //sw.Stop(); //RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); NdArray inputImageArrayGpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5); NdArray inputImageArrayCpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5); //MaxPooling MaxPooling maxPooling = new MaxPooling(3); RILogManager.Default?.EnterMethod(maxPooling.Name); sw.Restart(); NdArray[] gradImageArrayCpu = maxPooling.Forward(verbose, inputImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); maxPooling.Backward(verbose, gradImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (maxPooling.SetGpuEnable(true)) { sw.Restart(); maxPooling.Forward(verbose, inputImageArrayGpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); // There is no implementation for memory transfer only RILogManager.Default?.SendDebug("Backward[Gpu] : None"); } RILogManager.Default?.ExitMethod(maxPooling.Name); //AvgPooling AveragePooling avgPooling = new AveragePooling(3); RILogManager.Default?.EnterMethod(avgPooling.Name); sw.Restart(); gradImageArrayCpu = avgPooling.Forward(verbose, inputImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); avgPooling.Backward(verbose, gradImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); RILogManager.Default?.ExitMethod(avgPooling.Name); //Conv2D Convolution2D conv2d = new Convolution2D(verbose, 3, 3, 3); RILogManager.Default?.EnterMethod(conv2d.Name); sw.Restart(); gradImageArrayCpu = conv2d.Forward(verbose, inputImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); conv2d.Backward(verbose, gradImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (conv2d.SetGpuEnable(true)) { HandleGPU(verbose, sw, conv2d, inputArrayGpu); } RILogManager.Default?.ExitMethod(conv2d.Name); //Deconv2D Deconvolution2D deconv2d = new Deconvolution2D(verbose, 3, 3, 3); RILogManager.Default?.EnterMethod(deconv2d.Name); sw.Restart(); gradImageArrayCpu = deconv2d.Forward(verbose, inputImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); deconv2d.Backward(verbose, gradImageArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (deconv2d.SetGpuEnable(true)) { HandleGPU(verbose, sw, deconv2d, inputArrayGpu); } RILogManager.Default?.ExitMethod(deconv2d.Name); //Dropout Dropout dropout = new Dropout(); RILogManager.Default?.EnterMethod(dropout.Name); sw.Restart(); gradArrayCpu = dropout.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); dropout.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (dropout.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = dropout.Forward(verbose, inputArrayGpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); dropout.Backward(verbose, gradArrayGpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } RILogManager.Default?.ExitMethod(dropout.Name); //ArcSinH ArcSinH a = new ArcSinH(); RILogManager.Default?.EnterMethod(a.Name); sw.Restart(); gradArrayCpu = a.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); a.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (a.SetGpuEnable(true)) { HandleGPU(verbose, sw, a, inputArrayGpu); } RILogManager.Default?.ExitMethod(a.Name); //ELU ELU e = new ELU(); RILogManager.Default?.EnterMethod(e.Name); sw.Restart(); gradArrayCpu = e.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); e.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); RILogManager.Default?.ExitMethod(e.Name); //LeakyReluShifted LeakyReLUShifted lrs = new LeakyReLUShifted(); RILogManager.Default?.EnterMethod(lrs.Name); sw.Restart(); gradArrayCpu = lrs.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); lrs.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (lrs.SetGpuEnable(true)) { HandleGPU(verbose, sw, lrs, inputArrayGpu); } RILogManager.Default?.ExitMethod(lrs.Name); //Logistic LogisticFunction lf = new LogisticFunction(); RILogManager.Default?.EnterMethod(lf.Name); sw.Restart(); gradArrayCpu = lf.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); lf.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (lf.SetGpuEnable(true)) { HandleGPU(verbose, sw, lf, inputArrayGpu); } RILogManager.Default?.ExitMethod(lf.Name); //MaxMinusOne MaxMinusOne mmo = new MaxMinusOne(); RILogManager.Default?.EnterMethod(mmo.Name); sw.Restart(); gradArrayCpu = mmo.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); mmo.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (mmo.SetGpuEnable(true)) { HandleGPU(verbose, sw, mmo, inputArrayGpu); } RILogManager.Default?.ExitMethod(mmo.Name); //ScaledELU ScaledELU se = new ScaledELU(); RILogManager.Default?.EnterMethod(se.Name); sw.Restart(); gradArrayCpu = se.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); se.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (se.SetGpuEnable(true)) { HandleGPU(verbose, sw, se, inputArrayGpu); } RILogManager.Default?.ExitMethod(se.Name); //Sine Sine s = new Sine(); RILogManager.Default?.EnterMethod(s.Name); sw.Restart(); gradArrayCpu = s.Forward(verbose, inputArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); s.Backward(verbose, gradArrayCpu); sw.Stop(); RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (s.SetGpuEnable(true)) { HandleGPU(verbose, sw, s, inputArrayGpu); } RILogManager.Default?.ExitMethod(s.Name); }
public void Linear() { bool[] fast_speeds = { false, true }; foreach (bool fast_speed in fast_speeds) { float[] inputData = new float[] { 1, 2, 3, 4, 5, 6 }; int[] inputShape = new int[] { 2, 3 }; //2 samples of 3 float[] weightsData = new float[] { 4, 5, 6, 7, 8, 9 }; //3x2 float[] biasData = new float[] { 1, -1 }; float[] outputData = new float[] { 40, 46, 94, 109 }; int[] outputShape = new int[] { 2, 2 }; float[] biasedOutputData = new float[] { 41, 45, 95, 108 }; var input = ctrl.floatTensorFactory.Create(_data: inputData, _shape: inputShape, _autograd: true); var target = ctrl.floatTensorFactory.Create(_data: outputData, _shape: outputShape); var biasedTarget = ctrl.floatTensorFactory.Create(_data: biasedOutputData, _shape: outputShape); var linear = new Linear(ctrl, 3, 2, weights: weightsData, fast: fast_speed); var biasedLinear = new Linear(ctrl, 3, 2, weights: weightsData, bias: biasData, fast: fast_speed); Assert.True(linear.getParameterCount() == 6); Assert.True(biasedLinear.getParameterCount() == 8); var output = linear.Forward(input); var biasedOutput = biasedLinear.Forward(input); for (int i = 0; i < target.Size; i++) { Assert.AreEqual(output[i], target[i]); Assert.AreEqual(biasedOutput[i], biasedTarget[i]); } //running in batch mode, so testing "two gradients! var grad = ctrl.floatTensorFactory.Create(_data: new float[] { 1, 1, 0, -2 }, _shape: new int[] { 2, 2 }); var grad2 = ctrl.floatTensorFactory.Create(_data: new float[] { 1, 1, 0, -2 }, _shape: new int[] { 2, 2 }); output.Backward(grad); biasedOutput.Backward(grad2); float[] weightGradTarget = new float[] { 1, -7, 2, -8, 3, -9 }; float[] biasGradTarget = new float[] { 1, -1 }; var weightGrad = ctrl.floatTensorFactory.Get(linear.getParameter(0)).Grad; var weightGrad2 = ctrl.floatTensorFactory.Get(biasedLinear.getParameter(0)).Grad; var biasGrad = ctrl.floatTensorFactory.Get(biasedLinear.getParameter(1)).Grad; if (fast_speed) { weightGrad = weightGrad.Transpose(); weightGrad2 = weightGrad2.Transpose(); } for (int i = 0; i < weightGrad.Size; i++) { Assert.AreEqual(weightGradTarget[i], weightGrad[i]); Assert.AreEqual(weightGrad[i], weightGrad2[i]); } for (int i = 0; i < biasGrad.Size; i++) { Assert.AreEqual(biasGradTarget[i], biasGrad[i]); } //now backprop (with random weight initiallization!!) var x1 = ctrl.floatTensorFactory.Create(_data: new float[] { 1, 0 }, _shape: new int[] { 1, 2 }, _autograd: true); var x2 = ctrl.floatTensorFactory.Create(_data: new float[] { 0, 1 }, _shape: new int[] { 1, 2 }, _autograd: true); var y1 = ctrl.floatTensorFactory.Create(_data: new float[] { 5, 6 }, _shape: new int[] { 1, 2 }, _autograd: true); var y2 = ctrl.floatTensorFactory.Create(_data: new float[] { .3f, -8 }, _shape: new int[] { 1, 2 }, _autograd: true); var linear2 = new Linear(ctrl, 2, 2, fast: fast_speed); var prediction1 = linear2.Forward(x1); var prediction2 = linear2.Forward(x2); var err1 = prediction1.Sub(y1); err1.Autograd = false; var err2 = prediction2.Sub(y2); err2.Autograd = false; prediction1.Backward(err1); prediction2.Backward(err2); foreach (int p in linear2.getParameters()) { var param = ctrl.floatTensorFactory.Get(p); if (param.Grad != null) { param.Sub(param.Grad, inline: true); } } var correct_prediction1 = linear2.Forward(x1); var correct_prediction2 = linear2.Forward(x2); for (int i = 0; i < correct_prediction1.Size; i++) { Assert.AreEqual(correct_prediction1[i], y1[i], 1e-6); Assert.AreEqual(correct_prediction2[i], y2[i], 1e-6); } } }
public static void Run() { Stopwatch sw = new Stopwatch(); NdArray inputArrayCpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE)); NdArray inputArrayGpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE)); NdArray[] gradArrayCpu = null; //Linear Linear linear = new Linear(INPUT_SIZE, OUTPUT_SIZE); Console.WriteLine("◆" + linear.Name); if (TestCpu) { sw.Restart(); gradArrayCpu = linear.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; //Use Data as Grad sw.Restart(); linear.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } if (linear.SetGpuEnable(true)) { NdArray[] gradArrayGpu = null; while (true) { sw.Restart(); gradArrayGpu = linear.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); linear.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Tanh Tanh tanh = new Tanh(); Console.WriteLine("\n ◆" + tanh.Name); sw.Restart(); gradArrayCpu = tanh.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); tanh.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (tanh.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = tanh.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); tanh.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Sigmoid Sigmoid sigmoid = new Sigmoid(); Console.WriteLine("\n ◆" + sigmoid.Name); sw.Restart(); gradArrayCpu = sigmoid.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); sigmoid.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (sigmoid.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = sigmoid.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); sigmoid.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //ReLU ReLU relu = new ReLU(); Console.WriteLine("\n ◆" + relu.Name); sw.Restart(); gradArrayCpu = relu.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); relu.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (relu.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = relu.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); relu.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //LeakyReLU LeakyReLU leakyRelu = new LeakyReLU(); Console.WriteLine("\n ◆" + leakyRelu.Name); sw.Restart(); gradArrayCpu = leakyRelu.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); leakyRelu.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (leakyRelu.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = leakyRelu.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); leakyRelu.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } NdArray inputImageArrayGpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5); NdArray inputImageArrayCpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5); //MaxPooling MaxPooling maxPooling = new MaxPooling(3); Console.WriteLine("\n ◆" + maxPooling.Name); sw.Restart(); NdArray[] gradImageArrayCpu = maxPooling.Forward(inputImageArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); maxPooling.Backward(gradImageArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (maxPooling.SetGpuEnable(true)) { sw.Restart(); maxPooling.Forward(inputImageArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); //There is no implementation for memory transfer only Console.WriteLine("Backward[Gpu] : None"); } //Conv2D Convolution2D conv2d = new Convolution2D(3, 3, 3); Console.WriteLine("\n ◆" + conv2d.Name); sw.Restart(); gradImageArrayCpu = conv2d.Forward(inputImageArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); conv2d.Backward(gradImageArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (conv2d.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradImageArrayGpu = conv2d.Forward(inputImageArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data; sw.Restart(); conv2d.Backward(gradImageArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Deconv2D Deconvolution2D deconv2d = new Deconvolution2D(3, 3, 3); Console.WriteLine("\n ◆" + deconv2d.Name); sw.Restart(); gradImageArrayCpu = deconv2d.Forward(inputImageArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data; sw.Restart(); deconv2d.Backward(gradImageArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (deconv2d.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradImageArrayGpu = deconv2d.Forward(inputImageArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data; sw.Restart(); deconv2d.Backward(gradImageArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } //Dropout Dropout dropout = new Dropout(); Console.WriteLine("\n ◆" + dropout.Name); sw.Restart(); gradArrayCpu = dropout.Forward(inputArrayCpu); sw.Stop(); Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayCpu[0].Grad = gradArrayCpu[0].Data; sw.Restart(); dropout.Backward(gradArrayCpu); sw.Stop(); Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); if (dropout.SetGpuEnable(true)) { sw.Restart(); NdArray[] gradArrayGpu = dropout.Forward(inputArrayGpu); sw.Stop(); Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); gradArrayGpu[0].Grad = gradArrayGpu[0].Data; sw.Restart(); dropout.Backward(gradArrayGpu); sw.Stop(); Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs"); } }