/// <summary>
/// This function performs forward dropout operation over x returning results in y. If dropout was
/// used as a parameter to cudnnSetDropoutDescriptor, the approximately dropout fraction of x values
/// will be replaces by 0, and the rest will be scaled by 1/(1-dropout) This function should not be
/// running concurrently with another cudnnDropoutForward function using the same states.
/// </summary>
/// <param name="dropoutDesc">Handle to a previously created dropout descriptor object.</param>
/// <param name="xDesc">Handle to the previously initialized input tensor descriptor.</param>
/// <param name="x">Data pointer to GPU memory associated with the tensor descriptor srcDesc.</param>
/// <param name="yDesc">Handle to the previously initialized output tensor descriptor.</param>
/// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor destDesc.</param>
/// <param name="reserveSpace">
/// Data pointer to GPU memory used by this function. It is expected that contents of
/// reserveSpace doe not change between cudnnDropoutForward and cudnnDropoutBackward calls.
/// </param>
public void DropoutForward(DropoutDescriptor dropoutDesc,
TensorDescriptor xDesc,
CudaDeviceVariable <float> x,
TensorDescriptor yDesc,
CudaDeviceVariable <float> y,
CudaDeviceVariable <byte> reserveSpace)
{
var res = CudaDNNNativeMethods.cudnnDropoutForward(this.Handle, dropoutDesc.Desc, xDesc.Desc, x.DevicePointer, yDesc.Desc, y.DevicePointer, reserveSpace.DevicePointer,
reserveSpace.SizeInBytes);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public override void Dropout(double dropProbability, Volume <double> result)
{
var resultStorage = result.Storage as VolumeStorage;
if (resultStorage == null)
{
throw new ArgumentException($"{nameof(result)} storage should be VolumeStorage", nameof(result));
}
// Copy to device if not already done
this._volumeStorage.CopyToDevice();
resultStorage.CopyToDevice();
using var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext);
using var srcDesc = new TensorDescriptor();
using var resultDesc = new TensorDescriptor();
srcDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
this.Shape.Dimensions[3], this.Shape.Dimensions[2],
this.Shape.Dimensions[1], this.Shape.Dimensions[0]);
resultDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
result.Shape.Dimensions[3], result.Shape.Dimensions[2],
result.Shape.Dimensions[1], result.Shape.Dimensions[0]);
var stateSize = this._context.CudnnContext.GetDropoutStateSize();
if (resultStorage.DropoutStateStorage == null || resultStorage.DropoutStateStorage.Size != stateSize)
{
resultStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
}
dropoutDesc.SetDropoutDescriptor((float)dropProbability, resultStorage.DropoutStateStorage, stateSize, 0);
var reserveSpace = this._context.CudnnContext.GetDropoutReserveSpaceSize(srcDesc);
reserveSpace = reserveSpace == 0 ? new SizeT(1) : reserveSpace;
if (resultStorage.DropoutStorage == null || resultStorage.DropoutStorage.Size != reserveSpace)
{
resultStorage.DropoutStorage = new CudaDeviceVariable <byte>(reserveSpace);
}
this._context.CudnnContext.DropoutForward(dropoutDesc,
srcDesc, this._volumeStorage.DeviceBuffer,
resultDesc, resultStorage.DeviceBuffer,
resultStorage.DropoutStorage);
}
/// <summary>
/// This function performs forward dropout operation over x returning results in y. If dropout was
/// used as a parameter to cudnnSetDropoutDescriptor, the approximately dropout fraction of x values
/// will be replaces by 0, and the rest will be scaled by 1/(1-dropout) This function should not be
/// running concurrently with another cudnnDropoutForward function using the same states.
/// </summary>
/// <param name="dropoutDesc">Handle to a previously created dropout descriptor object.</param>
/// <param name="xDesc">Handle to the previously initialized input tensor descriptor.</param>
/// <param name="x">Data pointer to GPU memory associated with the tensor descriptor srcDesc.</param>
/// <param name="yDesc">Handle to the previously initialized output tensor descriptor.</param>
/// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor destDesc.</param>
/// <param name="reserveSpace">
/// Data pointer to GPU memory used by this function. It is expected that contents of
/// reserveSpace doe not change between cudnnDropoutForward and cudnnDropoutBackward calls.
/// </param>
public void DropoutForward(DropoutDescriptor dropoutDesc,
TensorDescriptor xDesc,
CudaDeviceVariable <double> x,
TensorDescriptor yDesc,
CudaDeviceVariable <double> y,
CudaDeviceVariable <byte> reserveSpace)
{
var res = CudaDNNNativeMethods.cudnnDropoutForward(this.Handle, dropoutDesc.Desc, xDesc.Desc, x.DevicePointer, yDesc.Desc, y.DevicePointer, reserveSpace.DevicePointer,
reserveSpace.SizeInBytes);
//Debug.WriteLine("{0:G}, {1}: {2}", DateTime.Now, "cudnnDropoutForward", res);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
private void DoDropout(Volume <float> result, float dropProbability)
{
var resultStorage = result.Storage as VolumeStorage;
if (resultStorage == null)
{
throw new ArgumentException($"{nameof(result)} storage should be VolumeStorage", nameof(result));
}
// Copy to device if not already done
this._volumeStorage.CopyToDevice();
resultStorage.CopyToDevice();
using (var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext))
using (var srcDesc = new TensorDescriptor())
using (var resultDesc = new TensorDescriptor())
{
srcDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
this.Shape.GetDimension(3), this.Shape.GetDimension(2),
this.Shape.GetDimension(1), this.Shape.GetDimension(0));
resultDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
result.Shape.GetDimension(3), result.Shape.GetDimension(2),
result.Shape.GetDimension(1), result.Shape.GetDimension(0));
var stateSize = this._context.CudnnContext.GetDropoutStateSize();
if (this._volumeStorage.DropoutStateStorage == null || this._volumeStorage.DropoutStateStorage.Size != stateSize)
{
this._volumeStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
}
dropoutDesc.SetDropoutDescriptor(dropProbability, this._volumeStorage.DropoutStateStorage, stateSize, 0);
var reserveSpace = this._context.CudnnContext.GetDropoutReserveSpaceSize(srcDesc);
reserveSpace = reserveSpace == 0 ? new SizeT(1) : reserveSpace;
if (this._volumeStorage.DropoutStorage == null || this._volumeStorage.DropoutStorage.Size != reserveSpace)
{
this._volumeStorage.DropoutStorage = new CudaDeviceVariable <byte>(reserveSpace);
}
this._context.CudnnContext.DropoutForward(dropoutDesc, srcDesc, this._volumeStorage.DeviceBuffer, resultDesc, resultStorage.DeviceBuffer, this._volumeStorage.DropoutStorage);
}
}
public override void DropoutGradient(Volume <double> input, Volume <double> outputGradient, double dropProbability, Volume <double> inputGradient)
{
var outputStorage = this.Storage as VolumeStorage;
var inputStorage = input.Storage as VolumeStorage;
var outputGradientStorage = outputGradient.Storage as VolumeStorage;
var inputGradientStorage = inputGradient.Storage as VolumeStorage;
// Copy to device if not already done
inputStorage.CopyToDevice();
outputGradientStorage.CopyToDevice();
inputGradientStorage.CopyToDevice();
using var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext);
using var dOutputDesc = new TensorDescriptor();
using var dDataDesc = new TensorDescriptor();
var stateSize = this._context.CudnnContext.GetDropoutStateSize();
if (outputStorage.DropoutStateStorage == null || outputStorage.DropoutStateStorage.Size != stateSize)
{
outputStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
}
dropoutDesc.SetDropoutDescriptor((float)dropProbability, outputStorage.DropoutStateStorage, stateSize, 0);
dDataDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
this.Shape.Dimensions[3],
this.Shape.Dimensions[2],
this.Shape.Dimensions[1],
this.Shape.Dimensions[0]);
dOutputDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double,
outputGradient.Shape.Dimensions[3],
outputGradient.Shape.Dimensions[2],
outputGradient.Shape.Dimensions[1],
outputGradient.Shape.Dimensions[0]);
this._context.CudnnContext.DropoutBackward(dropoutDesc,
dOutputDesc, outputGradientStorage.DeviceBuffer,
dDataDesc, inputGradientStorage.DeviceBuffer,
outputStorage.DropoutStorage);
}
public override void DoDropoutGradient(Volume <float> input, Volume <float> outputGradient, Volume <float> inputGradient, float dropProbability)
{
var inputStorage = input.Storage as VolumeStorage;
var outputGradientStorage = outputGradient.Storage as VolumeStorage;
var inputGradientStorage = inputGradient.Storage as VolumeStorage;
// Copy to device if not already done
inputStorage.CopyToDevice();
outputGradientStorage.CopyToDevice();
inputGradientStorage.CopyToDevice();
using (var dropoutDesc = new DropoutDescriptor(this._context.CudnnContext))
using (var dOutputDesc = new TensorDescriptor())
using (var dDataDesc = new TensorDescriptor())
{
var stateSize = this._context.CudnnContext.GetDropoutStateSize();
if (inputStorage.DropoutStateStorage == null || inputStorage.DropoutStateStorage.Size != stateSize)
{
inputStorage.DropoutStateStorage = new CudaDeviceVariable <byte>(stateSize);
}
dropoutDesc.SetDropoutDescriptor(dropProbability, inputStorage.DropoutStateStorage, stateSize, 0);
dDataDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
this.Shape.GetDimension(3),
this.Shape.GetDimension(2),
this.Shape.GetDimension(1),
this.Shape.GetDimension(0));
dOutputDesc.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float,
outputGradient.Shape.GetDimension(3),
outputGradient.Shape.GetDimension(2),
outputGradient.Shape.GetDimension(1),
outputGradient.Shape.GetDimension(0));
this._context.CudnnContext.DropoutBackward(dropoutDesc,
dOutputDesc, outputGradientStorage.DeviceBuffer,
dDataDesc, inputGradientStorage.DeviceBuffer,
inputStorage.DropoutStorage);
}
}