/// <summary>
/// This function is used to query the amount of space required to store the states of the random number generators
/// used by cudnnDropoutForward function.
/// </summary>
public SizeT GetDropoutStateSize()
{
var sizeInBytes = new SizeT();
var res = CudaDNNNativeMethods.cudnnDropoutGetStatesSize(this.Handle, ref sizeInBytes);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(sizeInBytes);
}
/// <summary>
/// This function is used to query the amount of reserve needed to run dropout with the input dimensions given by
/// xDesc.
/// The same reserve space is expected to be passed to cudnnDropoutForward and cudnnDropoutBackward, and its contents
/// is
/// expected to remain unchanged between cudnnDropoutForward and cudnnDropoutBackward calls.
/// </summary>
/// <param name="xDesc">Handle to a previously initialized tensor descriptor, describing input to a dropout operation.</param>
public SizeT GetDropoutReserveSpaceSize(TensorDescriptor xDesc)
{
var sizeInBytes = new SizeT();
var res = CudaDNNNativeMethods.cudnnDropoutGetReserveSpaceSize(xDesc.Desc, ref sizeInBytes);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(sizeInBytes);
}
/// <summary>
/// This function is used to query the amount of space required to store the states of the random number generators
/// used by cudnnDropoutForward function.
/// </summary>
public SizeT GetDropoutStateSize()
{
var sizeInBytes = new SizeT();
var res = CudaDNNNativeMethods.cudnnDropoutGetStatesSize(this.Handle, ref sizeInBytes);
//Debug.WriteLine("{0:G}, {1}: {2}", DateTime.Now, "cudnnDropoutGetStatesSize", res);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(sizeInBytes);
}
/// <summary>
/// This function is used to query the amount of reserve needed to run dropout with the input dimensions given by
/// xDesc.
/// The same reserve space is expected to be passed to cudnnDropoutForward and cudnnDropoutBackward, and its contents
/// is
/// expected to remain unchanged between cudnnDropoutForward and cudnnDropoutBackward calls.
/// </summary>
/// <param name="xDesc">Handle to a previously initialized tensor descriptor, describing input to a dropout operation.</param>
public SizeT GetDropoutReserveSpaceSize(TensorDescriptor xDesc)
{
var sizeInBytes = new SizeT();
var res = CudaDNNNativeMethods.cudnnDropoutGetReserveSpaceSize(xDesc.Desc, ref sizeInBytes);
// Debug.WriteLine("{0:G}, {1}: {2}", DateTime.Now, "cudnnDropoutGetReserveSpaceSize", res);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(sizeInBytes);
}
/// <summary>
/// This function performs forward dropout operation over x returning results in y. If dropout was
/// used as a parameter to cudnnSetDropoutDescriptor, the approximately dropout fraction of x values
/// will be replaces by 0, and the rest will be scaled by 1/(1-dropout) This function should not be
/// running concurrently with another cudnnDropoutForward function using the same states.
/// </summary>
/// <param name="dropoutDesc">Handle to a previously created dropout descriptor object.</param>
/// <param name="xDesc">Handle to the previously initialized input tensor descriptor.</param>
/// <param name="x">Data pointer to GPU memory associated with the tensor descriptor srcDesc.</param>
/// <param name="yDesc">Handle to the previously initialized output tensor descriptor.</param>
/// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor destDesc.</param>
/// <param name="reserveSpace">
/// Data pointer to GPU memory used by this function. It is expected that contents of
/// reserveSpace doe not change between cudnnDropoutForward and cudnnDropoutBackward calls.
/// </param>
public void DropoutForward(DropoutDescriptor dropoutDesc,
TensorDescriptor xDesc,
CudaDeviceVariable <float> x,
TensorDescriptor yDesc,
CudaDeviceVariable <float> y,
CudaDeviceVariable <byte> reserveSpace)
{
var res = CudaDNNNativeMethods.cudnnDropoutForward(this.Handle, dropoutDesc.Desc, xDesc.Desc, x.DevicePointer, yDesc.Desc, y.DevicePointer, reserveSpace.DevicePointer,
reserveSpace.SizeInBytes);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This function performs forward dropout operation over x returning results in y. If dropout was
/// used as a parameter to cudnnSetDropoutDescriptor, the approximately dropout fraction of x values
/// will be replaces by 0, and the rest will be scaled by 1/(1-dropout) This function should not be
/// running concurrently with another cudnnDropoutForward function using the same states.
/// </summary>
/// <param name="dropoutDesc">Handle to a previously created dropout descriptor object.</param>
/// <param name="xDesc">Handle to the previously initialized input tensor descriptor.</param>
/// <param name="x">Data pointer to GPU memory associated with the tensor descriptor srcDesc.</param>
/// <param name="yDesc">Handle to the previously initialized output tensor descriptor.</param>
/// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor destDesc.</param>
/// <param name="reserveSpace">
/// Data pointer to GPU memory used by this function. It is expected that contents of
/// reserveSpace doe not change between cudnnDropoutForward and cudnnDropoutBackward calls.
/// </param>
public void DropoutForward(DropoutDescriptor dropoutDesc,
TensorDescriptor xDesc,
CudaDeviceVariable <double> x,
TensorDescriptor yDesc,
CudaDeviceVariable <double> y,
CudaDeviceVariable <byte> reserveSpace)
{
var res = CudaDNNNativeMethods.cudnnDropoutForward(this.Handle, dropoutDesc.Desc, xDesc.Desc, x.DevicePointer, yDesc.Desc, y.DevicePointer, reserveSpace.DevicePointer,
reserveSpace.SizeInBytes);
//Debug.WriteLine("{0:G}, {1}: {2}", DateTime.Now, "cudnnDropoutForward", res);
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public override void DoMultiply(Volume <float> right, Volume <float> result)
{
var resultStorage = result.Storage as VolumeStorage;
if (resultStorage == null)
{
throw new ArgumentException($"{nameof(result)} storage should be VolumeStorage", nameof(result));
}
var rightStorage = right.Storage as VolumeStorage;
if (rightStorage == null)
{
throw new ArgumentException($"{nameof(right)} storage should be VolumeStorage", nameof(right));
}
// Copy to device if not already done
this._volumeStorage.CopyToDevice();
rightStorage.CopyToDevice();
resultStorage.CopyToDevice();
var aStorage = this._volumeStorage;
var bStorage = rightStorage;
if (bStorage.Shape.TotalLength > aStorage.Shape.TotalLength)
{
aStorage = rightStorage;
bStorage = this._volumeStorage;
}
var bShape = bStorage.Shape;
var n = aStorage.Shape.GetDimension(3);
var c = aStorage.Shape.GetDimension(2);
var h = aStorage.Shape.GetDimension(1);
var w = aStorage.Shape.GetDimension(0);
// Add tensors
using (var descA = new TensorDescriptor())
using (var descB = new TensorDescriptor())
using (var descC = new TensorDescriptor())
{
descA.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float, n, c, h, w);
descB.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float, bShape.GetDimension(3), bShape.GetDimension(2), bShape.GetDimension(1),
bShape.GetDimension(0));
descC.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Float, n, c, h, w);
using (var opt = new OpTensorDescriptor(this._context.CudnnContext))
{
opt.SetOpTensorDescriptor(
cudnnOpTensorOp.OpTensorMul,
cudnnDataType.Float,
cudnnNanPropagation.PropagateNan);
var one = 1.0f;
var zero = 0.0f;
var status = CudaDNNNativeMethods.cudnnOpTensor(
this._context.CudnnContext.Handle,
opt.Desc,
ref one, descA.Desc, aStorage.DeviceBuffer.DevicePointer,
ref one, descB.Desc, bStorage.DeviceBuffer.DevicePointer,
ref zero, descC.Desc, resultStorage.DeviceBuffer.DevicePointer);
if (status != cudnnStatus.Success)
{
throw new Exception(CudaDNNNativeMethods.cudnnGetErrorString(status));
}
resultStorage.Location = DataLocation.Device;
}
}
}
private void Op(Volume <double> right, cudnnOpTensorOp op, Volume <double> result)
{
var resultStorage = result.Storage as VolumeStorage;
if (resultStorage == null)
{
throw new ArgumentException($"{nameof(result)} storage should be VolumeStorage", nameof(result));
}
VolumeStorage rightStorage = null;
if (right != null)
{
rightStorage = right.Storage as VolumeStorage;
if (rightStorage == null)
{
throw new ArgumentException($"{nameof(right)} storage should be VolumeStorage", nameof(right));
}
}
// Copy to device if not already done
this._volumeStorage.CopyToDevice();
rightStorage?.CopyToDevice();
resultStorage.CopyToDevice();
var aStorage = this._volumeStorage;
Shape bShape = null;
VolumeStorage bStorage = null;
if (rightStorage != null)
{
bStorage = rightStorage;
if (bStorage.Shape.TotalLength > aStorage.Shape.TotalLength)
{
aStorage = rightStorage;
bStorage = this._volumeStorage;
}
bShape = bStorage.Shape;
}
var n = aStorage.Shape.Dimensions[3];
var c = aStorage.Shape.Dimensions[2];
var h = aStorage.Shape.Dimensions[1];
var w = aStorage.Shape.Dimensions[0];
// Add tensors
using var descA = new TensorDescriptor();
using var descB = new TensorDescriptor();
using var descC = new TensorDescriptor();
descA.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double, n, c, h, w);
if (bShape != null)
{
descB.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double, bShape.Dimensions[3], bShape.Dimensions[2], bShape.Dimensions[1],
bShape.Dimensions[0]);
}
descC.SetTensor4dDescriptor(cudnnTensorFormat.NCHW, cudnnDataType.Double, n, c, h, w);
using var opt = new OpTensorDescriptor(this._context.CudnnContext);
opt.SetOpTensorDescriptor(
op,
cudnnDataType.Double,
cudnnNanPropagation.PropagateNan);
var one = 1.0;
var zero = 0.0;
var status = CudaDNNNativeMethods.cudnnOpTensor(
this._context.CudnnContext.Handle,
opt.Desc,
ref one, descA.Desc, aStorage.DeviceBuffer.DevicePointer,
ref one, bStorage != null ? descB.Desc : descA.Desc, bStorage?.DeviceBuffer.DevicePointer ?? aStorage.DeviceBuffer.DevicePointer,
ref zero, descC.Desc, resultStorage.DeviceBuffer.DevicePointer);
if (status != cudnnStatus.Success)
{
throw new Exception(CudaDNNNativeMethods.cudnnGetErrorString(status));
}
resultStorage.Location = DataLocation.Device;
}
