public unsafe void FullyConnectedForward() { FullyConnectedLayer fc = new FullyConnectedLayer(TensorInfo.Linear(231), 125, ActivationType.Sigmoid, WeightsInitializationMode.GlorotUniform, BiasInitializationMode.Gaussian); Tensor x = CreateRandomTensor(400, fc.InputInfo.Size); fixed(float *pw = fc.Weights, pb = fc.Biases) { Tensor.Reshape(pw, fc.InputInfo.Size, fc.OutputInfo.Size, out Tensor w); Tensor.Reshape(pb, 1, fc.OutputInfo.Size, out Tensor b); Tensor.New(x.Entities, fc.OutputInfo.Size, out Tensor y1); CpuDnn.FullyConnectedForward(x, w, b, y1); Gpu gpu = Gpu.Default; using (DeviceMemory <float> x_gpu = gpu.AllocateDevice(x), w_gpu = gpu.AllocateDevice(w), b_gpu = gpu.AllocateDevice(b), y_gpu = gpu.AllocateDevice <float>(y1.Size)) { Dnn.Get(gpu).FullyConnectedForward(x.Entities, x.Length, y1.Length, x_gpu.Ptr, w_gpu.Ptr, b_gpu.Ptr, y_gpu.Ptr); y_gpu.CopyToHost(y1.Entities, y1.Length, out Tensor y2); Assert.IsTrue(y1.ContentEquals(y2)); Tensor.Free(x, y1, y2); } } }
public unsafe void FullyConnectedBackwardData() { FullyConnectedLayer fc = new FullyConnectedLayer(TensorInfo.Linear(231), 125, ActivationType.Sigmoid, WeightsInitializationMode.GlorotUniform, BiasInitializationMode.Gaussian); Tensor dy = CreateRandomTensor(400, fc.OutputInfo.Size); fixed(float *pw = fc.Weights, pb = fc.Biases) { Tensor.Reshape(pw, fc.InputInfo.Size, fc.OutputInfo.Size, out Tensor w); Tensor.Reshape(pb, 1, fc.OutputInfo.Size, out Tensor b); Tensor.New(dy.Entities, fc.InputInfo.Size, out Tensor dx1); CpuDnn.FullyConnectedBackwardData(w, dy, dx1); Gpu gpu = Gpu.Default; using (DeviceMemory <float> dy_gpu = gpu.AllocateDevice(dy), w_gpu = gpu.AllocateDevice(w), dx_gpu = gpu.AllocateDevice <float>(dx1.Size)) { Dnn.Get(gpu).FullyConnectedBackwardData(dy.Entities, fc.InputInfo.Size, fc.OutputInfo.Size, dy_gpu.Ptr, w_gpu.Ptr, dx_gpu.Ptr); dx_gpu.CopyToHost(dx1.Entities, dx1.Length, out Tensor dx2); Assert.IsTrue(dx1.ContentEquals(dx2)); Tensor.Free(dy, dx1, dx2); } } }
public void ActivationForward() { Tensor x = CreateRandomTensor(400, 1200); Tensor.Like(x, out Tensor y1); CpuDnn.ActivationForward(x, ActivationFunctions.Sigmoid, y1); Gpu gpu = Gpu.Default; using (DeviceMemory <float> x_gpu = gpu.AllocateDevice(x), y_gpu = gpu.AllocateDevice <float>(x.Size)) { Dnn.Get(gpu).ActivationForward(x.Entities, x.Length, x_gpu.Ptr, y_gpu.Ptr, ActivationFunctions.Sigmoid); y_gpu.CopyToHost(y1.Entities, y1.Length, out Tensor y2); Assert.IsTrue(y1.ContentEquals(y2)); Tensor.Free(x, y1, y2); } }
public void ActivationBackward() { Tensor x = CreateRandomTensor(400, 1200), dy = CreateRandomTensor(400, 1200); Tensor.Like(x, out Tensor dx1); CpuDnn.ActivationBackward(x, dy, ActivationFunctions.SigmoidPrime, dx1); Gpu gpu = Gpu.Default; using (DeviceMemory <float> x_gpu = gpu.AllocateDevice(x), dy_gpu = gpu.AllocateDevice(dy)) { Dnn.Get(gpu).ActivationBackward(x.Entities, x.Length, x_gpu.Ptr, dy_gpu.Ptr, ActivationFunctions.SigmoidPrime, dy_gpu.Ptr); dy_gpu.CopyToHost(dy.Entities, dy.Length, out Tensor dx2); Assert.IsTrue(dx1.ContentEquals(dx2)); Tensor.Free(x, dy, dx1, dx2); } }
public void PerActivationBatchNormalizationForward() { // Setup Tensor x = CreateRandomTensor(400, 250); Tensor.NewZeroed(1, 250, out Tensor mu); Tensor.LikeZeroed(mu, out Tensor sigma2); Tensor.New(1, 250, out Tensor gamma); Tensor.NewZeroed(1, 250, out Tensor beta); for (int i = 0; i < 250; i++) { gamma[i] = ThreadSafeRandom.NextFloat(); } // Cpu Tensor.Like(x, out Tensor y1); CpuDnn.BatchNormalizationForward(NormalizationMode.PerActivation, TensorInfo.Linear(250), x, 1, mu, sigma2, gamma, beta, y1); // Gpu Gpu gpu = Gpu.Default; using (DeviceMemory <float> x_gpu = gpu.AllocateDevice(x), y_gpu = gpu.AllocateDevice <float>(x.Size), gamma_gpu = gpu.AllocateDevice(gamma), beta_gpu = gpu.AllocateDevice(beta), run_mean = gpu.AllocateDevice <float>(mu.Size), run_var = gpu.AllocateDevice <float>(mu.Size)) { TensorDescriptor desc = new TensorDescriptor(); desc.Set4D(DataType.FLOAT, TensorFormat.CUDNN_TENSOR_NCHW, x.Entities, x.Length, 1, 1); TensorDescriptor gammaBetadesc = new TensorDescriptor(); gammaBetadesc.Set4D(DataType.FLOAT, TensorFormat.CUDNN_TENSOR_NCHW, 1, x.Length, 1, 1); Dnn.Get(gpu).BatchNormalizationForwardTraining( BatchNormMode.PER_ACTIVATION, 1, 0, desc, x_gpu.Ptr, desc, y_gpu.Ptr, gammaBetadesc, gamma_gpu.Ptr, beta_gpu.Ptr, 1, run_mean.Ptr, run_var.Ptr, CpuDnn.CUDNN_BN_MIN_EPSILON,
public void FullyConnectedBackwardFilter() { FullyConnectedLayer fc = new FullyConnectedLayer(TensorInfo.Linear(231), 125, ActivationType.Sigmoid, WeightsInitializationMode.GlorotUniform, BiasInitializationMode.Gaussian); Tensor x = CreateRandomTensor(400, fc.InputInfo.Size), dy = CreateRandomTensor(x.Entities, fc.OutputInfo.Size); Tensor.New(fc.InputInfo.Size, fc.OutputInfo.Size, out Tensor dJdw1); CpuDnn.FullyConnectedBackwardFilter(x, dy, dJdw1); dJdw1.Reshape(1, dJdw1.Size, out dJdw1); Gpu gpu = Gpu.Default; using (DeviceMemory <float> x_gpu = gpu.AllocateDevice(x), dy_gpu = gpu.AllocateDevice(dy), djdb_gpu = gpu.AllocateDevice <float>(fc.Weights.Length)) { Dnn.Get(gpu).FullyConnectedBackwardFilter(x.Entities, fc.InputInfo.Size, fc.OutputInfo.Size, x_gpu.Ptr, dy_gpu.Ptr, djdb_gpu.Ptr); djdb_gpu.CopyToHost(1, fc.Weights.Length, out Tensor dJdw2); Assert.IsTrue(dJdw1.ContentEquals(dJdw2)); Tensor.Free(x, dy, dJdw1, dJdw2); } }