/// <summary>
        ///     This function performs forward dropout operation over x returning results in y. If dropout was
        ///     used as a parameter to cudnnSetDropoutDescriptor, the approximately dropout fraction of x values
        ///     will be replaces by 0, and the rest will be scaled by 1/(1-dropout) This function should not be
        ///     running concurrently with another cudnnDropoutForward function using the same states.
        /// </summary>
        /// <param name="dropoutDesc">Handle to a previously created dropout descriptor object.</param>
        /// <param name="xDesc">Handle to the previously initialized input tensor descriptor.</param>
        /// <param name="x">Data pointer to GPU memory associated with the tensor descriptor srcDesc.</param>
        /// <param name="yDesc">Handle to the previously initialized output tensor descriptor.</param>
        /// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor destDesc.</param>
        /// <param name="reserveSpace">
        ///     Data pointer to GPU memory used by this function. It is expected that contents of
        ///     reserveSpace doe not change between cudnnDropoutForward and cudnnDropoutBackward calls.
        /// </param>
        public void DropoutForward(DropoutDescriptor dropoutDesc,
                                   TensorDescriptor xDesc,
                                   CudaDeviceVariable <float> x,
                                   TensorDescriptor yDesc,
                                   CudaDeviceVariable <float> y,
                                   CudaDeviceVariable <byte> reserveSpace)
        {
            var res = CudaDNNNativeMethods.cudnnDropoutForward(this.Handle, dropoutDesc.Desc, xDesc.Desc, x.DevicePointer, yDesc.Desc, y.DevicePointer, reserveSpace.DevicePointer,
                                                               reserveSpace.SizeInBytes);

            if (res != cudnnStatus.Success)
            {
                throw new CudaDNNException(res);
            }
        }
Esempio n. 2
0
        public override void Dropout(double dropProbability, Volume <double> result)
        {
            var resultStorage = result.Storage as VolumeStorage;

            if (resultStorage == null)
            {
                throw new ArgumentException($"{nameof(result)} storage should be VolumeStorage", nameof(result));
            }

            // Copy to device if not already done
            this._volumeStorage.CopyToDevice();
            resultStorage.CopyToDevice();

            using var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext);
            using var srcDesc     = new TensorDescriptor();
            using var resultDesc  = new TensorDescriptor();

            srcDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
                                          this.Shape.Dimensions[3], this.Shape.Dimensions[2],
                                          this.Shape.Dimensions[1], this.Shape.Dimensions[0]);

            resultDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
                                             result.Shape.Dimensions[3], result.Shape.Dimensions[2],
                                             result.Shape.Dimensions[1], result.Shape.Dimensions[0]);

            var stateSize = this._context.CudnnContext.GetDropoutStateSize();

            if (resultStorage.DropoutStateStorage == null || resultStorage.DropoutStateStorage.Size != stateSize)
            {
                resultStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
            }

            dropoutDesc.SetDropoutDescriptor((float)dropProbability, resultStorage.DropoutStateStorage, stateSize, 0);

            var reserveSpace = this._context.CudnnContext.GetDropoutReserveSpaceSize(srcDesc);

            reserveSpace = reserveSpace == 0 ? new SizeT(1) : reserveSpace;

            if (resultStorage.DropoutStorage == null || resultStorage.DropoutStorage.Size != reserveSpace)
            {
                resultStorage.DropoutStorage = new CudaDeviceVariable <byte>(reserveSpace);
            }

            this._context.CudnnContext.DropoutForward(dropoutDesc,
                                                      srcDesc, this._volumeStorage.DeviceBuffer,
                                                      resultDesc, resultStorage.DeviceBuffer,
                                                      resultStorage.DropoutStorage);
        }
Esempio n. 3
0
        /// <summary>
        ///     This function performs forward dropout operation over x returning results in y. If dropout was
        ///     used as a parameter to cudnnSetDropoutDescriptor, the approximately dropout fraction of x values
        ///     will be replaces by 0, and the rest will be scaled by 1/(1-dropout) This function should not be
        ///     running concurrently with another cudnnDropoutForward function using the same states.
        /// </summary>
        /// <param name="dropoutDesc">Handle to a previously created dropout descriptor object.</param>
        /// <param name="xDesc">Handle to the previously initialized input tensor descriptor.</param>
        /// <param name="x">Data pointer to GPU memory associated with the tensor descriptor srcDesc.</param>
        /// <param name="yDesc">Handle to the previously initialized output tensor descriptor.</param>
        /// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor destDesc.</param>
        /// <param name="reserveSpace">
        ///     Data pointer to GPU memory used by this function. It is expected that contents of
        ///     reserveSpace doe not change between cudnnDropoutForward and cudnnDropoutBackward calls.
        /// </param>
        public void DropoutForward(DropoutDescriptor dropoutDesc,
                                   TensorDescriptor xDesc,
                                   CudaDeviceVariable <double> x,
                                   TensorDescriptor yDesc,
                                   CudaDeviceVariable <double> y,
                                   CudaDeviceVariable <byte> reserveSpace)
        {
            var res = CudaDNNNativeMethods.cudnnDropoutForward(this.Handle, dropoutDesc.Desc, xDesc.Desc, x.DevicePointer, yDesc.Desc, y.DevicePointer, reserveSpace.DevicePointer,
                                                               reserveSpace.SizeInBytes);

            //Debug.WriteLine("{0:G}, {1}: {2}", DateTime.Now, "cudnnDropoutForward", res);
            if (res != cudnnStatus.Success)
            {
                throw new CudaDNNException(res);
            }
        }
Esempio n. 4
0
        private void DoDropout(Volume <float> result, float dropProbability)
        {
            var resultStorage = result.Storage as VolumeStorage;

            if (resultStorage == null)
            {
                throw new ArgumentException($"{nameof(result)} storage should be VolumeStorage", nameof(result));
            }

            // Copy to device if not already done
            this._volumeStorage.CopyToDevice();
            resultStorage.CopyToDevice();

            using (var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext))
                using (var srcDesc = new TensorDescriptor())
                    using (var resultDesc = new TensorDescriptor())
                    {
                        srcDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
                                                      this.Shape.GetDimension(3), this.Shape.GetDimension(2),
                                                      this.Shape.GetDimension(1), this.Shape.GetDimension(0));

                        resultDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
                                                         result.Shape.GetDimension(3), result.Shape.GetDimension(2),
                                                         result.Shape.GetDimension(1), result.Shape.GetDimension(0));

                        var stateSize = this._context.CudnnContext.GetDropoutStateSize();
                        if (this._volumeStorage.DropoutStateStorage == null || this._volumeStorage.DropoutStateStorage.Size != stateSize)
                        {
                            this._volumeStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
                        }

                        dropoutDesc.SetDropoutDescriptor(dropProbability, this._volumeStorage.DropoutStateStorage, stateSize, 0);

                        var reserveSpace = this._context.CudnnContext.GetDropoutReserveSpaceSize(srcDesc);
                        reserveSpace = reserveSpace == 0 ? new SizeT(1) : reserveSpace;

                        if (this._volumeStorage.DropoutStorage == null || this._volumeStorage.DropoutStorage.Size != reserveSpace)
                        {
                            this._volumeStorage.DropoutStorage = new CudaDeviceVariable <byte>(reserveSpace);
                        }

                        this._context.CudnnContext.DropoutForward(dropoutDesc, srcDesc, this._volumeStorage.DeviceBuffer, resultDesc, resultStorage.DeviceBuffer, this._volumeStorage.DropoutStorage);
                    }
        }
Esempio n. 5
0
        public override void DropoutGradient(Volume <double> input, Volume <double> outputGradient, double dropProbability, Volume <double> inputGradient)
        {
            var outputStorage         = this.Storage as VolumeStorage;
            var inputStorage          = input.Storage as VolumeStorage;
            var outputGradientStorage = outputGradient.Storage as VolumeStorage;
            var inputGradientStorage  = inputGradient.Storage as VolumeStorage;

            // Copy to device if not already done
            inputStorage.CopyToDevice();
            outputGradientStorage.CopyToDevice();
            inputGradientStorage.CopyToDevice();

            using var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext);
            using var dOutputDesc = new TensorDescriptor();
            using var dDataDesc   = new TensorDescriptor();

            var stateSize = this._context.CudnnContext.GetDropoutStateSize();

            if (outputStorage.DropoutStateStorage == null || outputStorage.DropoutStateStorage.Size != stateSize)
            {
                outputStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
            }

            dropoutDesc.SetDropoutDescriptor((float)dropProbability, outputStorage.DropoutStateStorage, stateSize, 0);

            dDataDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
                                            this.Shape.Dimensions[3],
                                            this.Shape.Dimensions[2],
                                            this.Shape.Dimensions[1],
                                            this.Shape.Dimensions[0]);

            dOutputDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
                                              outputGradient.Shape.Dimensions[3],
                                              outputGradient.Shape.Dimensions[2],
                                              outputGradient.Shape.Dimensions[1],
                                              outputGradient.Shape.Dimensions[0]);

            this._context.CudnnContext.DropoutBackward(dropoutDesc,
                                                       dOutputDesc, outputGradientStorage.DeviceBuffer,
                                                       dDataDesc, inputGradientStorage.DeviceBuffer,
                                                       outputStorage.DropoutStorage);
        }
Esempio n. 6
0
        public override void DoDropoutGradient(Volume <float> input, Volume <float> outputGradient, Volume <float> inputGradient, float dropProbability)
        {
            var inputStorage          = input.Storage as VolumeStorage;
            var outputGradientStorage = outputGradient.Storage as VolumeStorage;
            var inputGradientStorage  = inputGradient.Storage as VolumeStorage;

            // Copy to device if not already done
            inputStorage.CopyToDevice();
            outputGradientStorage.CopyToDevice();
            inputGradientStorage.CopyToDevice();

            using (var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext))
                using (var dOutputDesc = new TensorDescriptor())
                    using (var dDataDesc = new TensorDescriptor())
                    {
                        var stateSize = this._context.CudnnContext.GetDropoutStateSize();
                        if (inputStorage.DropoutStateStorage == null || inputStorage.DropoutStateStorage.Size != stateSize)
                        {
                            inputStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
                        }

                        dropoutDesc.SetDropoutDescriptor(dropProbability, inputStorage.DropoutStateStorage, stateSize, 0);

                        dDataDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
                                                        this.Shape.GetDimension(3),
                                                        this.Shape.GetDimension(2),
                                                        this.Shape.GetDimension(1),
                                                        this.Shape.GetDimension(0));

                        dOutputDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
                                                          outputGradient.Shape.GetDimension(3),
                                                          outputGradient.Shape.GetDimension(2),
                                                          outputGradient.Shape.GetDimension(1),
                                                          outputGradient.Shape.GetDimension(0));

                        this._context.CudnnContext.DropoutBackward(dropoutDesc,
                                                                   dOutputDesc, outputGradientStorage.DeviceBuffer,
                                                                   dDataDesc, inputGradientStorage.DeviceBuffer,
                                                                   inputStorage.DropoutStorage);
                    }
        }