/// <summary>
/// </summary>
public CudaDNNContext()
{
_handle = new cudnnHandle();
res = CudaDNNNativeMethods.cudnnCreate(ref _handle);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreate", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// </summary>
public TensorDescriptor()
{
_desc = new cudnnTensorDescriptor();
res = CudaDNNNativeMethods.cudnnCreateTensorDescriptor(ref _desc);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreateTensorDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
///
/// </summary>
/// <param name="context"></param>
public SpatialTransformerDescriptor(CudaDNNContext context)
{
_handle = context.Handle;
_desc = new cudnnSpatialTransformerDescriptor();
res = CudaDNNNativeMethods.cudnnCreateSpatialTransformerDescriptor(ref _desc);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreateSpatialTransformerDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void CopyAlgorithmDescriptor(AlgorithmDescriptor dest)
{
res = CudaDNNNativeMethods.cudnnCopyAlgorithmDescriptor(_desc, dest.Desc);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCopyAlgorithmDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This function initializes a previously created CTC Loss descriptor object.
/// </summary>
/// <param name="dataType">Math precision.</param>
public void SetCTCLossDescriptor(cudnnDataType dataType)
{
res = CudaDNNNativeMethods.cudnnSetCTCLossDescriptor(_desc, dataType);
Debug.Write("");//Line(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetCTCLossDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This function sets the stream to be used by the cudnn library to execute its routines.
/// </summary>
/// <param name="stream">the stream to be used by the library.</param>
public void SetStream(CudaStream stream)
{
res = CudaDNNNativeMethods.cudnnSetStream(_handle, stream.Stream);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetStream", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public void SetFusedOpsVariantParamPackAttribute(cudnnFusedOpsVariantParamLabel paramLabel, IntPtr param)
{
res = CudaDNNNativeMethods.cudnnSetFusedOpsVariantParamPackAttribute(_pack, paramLabel, param);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetFusedOpsVariantParamPackAttribute", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
///
/// </summary>
public void GetCTCLossDescriptorEx(ref cudnnDataType compType, ref cudnnLossNormalizationMode normMode, ref cudnnNanPropagation gradMode)
{
res = CudaDNNNativeMethods.cudnnGetCTCLossDescriptorEx(_desc, ref compType, ref normMode, ref gradMode);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetCTCLossDescriptorEx", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
private static string GetErrorMessageFromCUResult(cudnnStatus error)
{
string message = string.Empty;
switch (error)
{
case cudnnStatus.Success:
message = "The operation completed successfully.";
break;
case cudnnStatus.NotInitialized:
message = "The cuDNN library was not initialized properly.\nThis error is usually returned when a call to cudnnCreate() fails or when cudnnCreate() has not been called prior to calling another cuDNN routine. In the former case, it is usually due to an error in the CUDA Runtime API called by cudnnCreate() or by an error in the hardware setup.";
break;
case cudnnStatus.AllocFailed:
message = "Resource allocation failed inside the cuDNN library. This is usually caused by an internal cudaMalloc() failure.\nTo correct: prior to the function call, deallocate previously allocated memory as much as possible.";
break;
case cudnnStatus.BadParam:
message = "An incorrect value or parameter was passed to the function.\nTo correct: ensure that all the parameters being passed have valid values.";
break;
case cudnnStatus.InternalError:
message = "An internal cuDNN operation failed.";
break;
case cudnnStatus.InvalidValue:
message = "";
break;
case cudnnStatus.ArchMismatch:
message = "The function requires a feature absent from the current GPU device. Note that cuDNN only supports devices with compute capabilities greater than or equal to 3.0.\nTo correct: compile and run the application on a device with appropriate compute capability.";
break;
case cudnnStatus.MappingError:
message = "An access to GPU memory space failed, which is usually caused by a failure to bind a texture.\nTo correct: prior to the function call, unbind any previously bound textures.\nOtherwise, this may indicate an internal error/bug in the library.";
break;
case cudnnStatus.ExecutionFailed:
message = "The GPU program failed to execute. This is usually caused by a failure to launch some cuDNN kernel on the GPU, which can occur for multiple reasons.\nTo correct: check that the hardware, an appropriate version of the driver, and the cuDNN library are correctly installed.\nOtherwise, this may indicate a internal error/bug in the library.";
break;
case cudnnStatus.NotSupported:
message = "The functionality requested is not presently supported by cuDNN.";
break;
case cudnnStatus.LicenseError:
message = "The functionality requested requires some license and an error was detected when trying to check the current licensing. This error can happen if the license is not present or is expired or if the environment variable NVIDIA_LICENSE_FILE is not set properly.";
break;
default:
break;
}
return(error.ToString() + ": " + message);
}
/// <summary>
/// </summary>
public CudaDNNContext()
{
_handle = new cudnnHandle();
res = CudaDNNNativeMethods.cudnnCreate(ref _handle);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreate", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// </summary>
public FusedOpsVariantParamPack(cudnnFusedOps ops)
{
_pack = new cudnnFusedOpsVariantParamPack();
res = CudaDNNNativeMethods.cudnnCreateFusedOpsVariantParamPack(ref _pack, ops);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreateFusedOpsVariantParamPack", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// </summary>
public PersistentRNNPlan(RNNDescriptor rnnDesc, int minibatch, cudnnDataType dataType)
{
_desc = new cudnnPersistentRNNPlan();
res = CudaDNNNativeMethods.cudnnCreatePersistentRNNPlan(rnnDesc.Desc, minibatch, dataType, ref _desc);
Debug.Write("");//Line(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreatePersistentRNNPlan", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// </summary>
public TensorDescriptor()
{
_desc = new cudnnTensorDescriptor();
res = CudaDNNNativeMethods.cudnnCreateTensorDescriptor(ref _desc);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreateTensorDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// </summary>
public AlgorithmPerformances(int count)
{
_perfs = new cudnnAlgorithmPerformance[count];
res = CudaDNNNativeMethods.cudnnCreateAlgorithmPerformance(_perfs, count);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreateAlgorithmPerformance", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// </summary>
public LRNDescriptor(CudaDNNContext context)
{
_handle = context.Handle;
_desc = new cudnnLRNDescriptor();
res = CudaDNNNativeMethods.cudnnCreateLRNDescriptor(ref _desc);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnCreateLRNDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This function queries the parameters of the previouly initialized activation descriptor object.
/// </summary>
/// <param name="mode">Enumerant to specify the activation mode.</param>
/// <param name="reluNanOpt">Nan propagation option for the relu.</param>
/// <param name="reluCeiling">The ceiling for the clipped relu.</param>
public void GetActivationDescriptor(ref cudnnActivationMode mode,
ref cudnnNanPropagation reluNanOpt,
ref double reluCeiling)
{
res = CudaDNNNativeMethods.cudnnGetActivationDescriptor(_desc, ref mode, ref reluNanOpt, ref reluCeiling);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetActivationDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This function provides the output dimensions of a tensor after Nd pooling has been applied
/// </summary>
/// <param name="inputTensorDesc">Handle to the previously initialized input tensor descriptor.</param>
/// <param name="nbDims">Number of dimensions in which pooling is to be applied.</param>
/// <param name="outputTensorDimA">Array of nbDims output dimensions</param>
public void GetPoolingNdForwardOutputDim(TensorDescriptor inputTensorDesc,
int nbDims,
int[] outputTensorDimA)
{
res = CudaDNNNativeMethods.cudnnGetPoolingNdForwardOutputDim(_desc, inputTensorDesc.Desc, nbDims, outputTensorDimA);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetPoolingNdForwardOutputDim", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This function computes the gradient of a grid generation operation.
/// </summary>
/// <param name="dgrid">Data pointer to GPU memory contains the input differential data.</param>
/// <param name="dtheta">Data pointer to GPU memory contains the output differential data.</param>
/// <returns></returns>
public void SpatialTfGridGeneratorBackward(
CudaDeviceVariable <float> dgrid,
CudaDeviceVariable <float> dtheta)
{
res = CudaDNNNativeMethods.cudnnSpatialTfGridGeneratorBackward(_handle, _desc, dtheta.DevicePointer, dgrid.DevicePointer);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSpatialTfGridGeneratorBackward", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
///<summary>
/// This function initializes then previously created activation descriptor object.
/// </summary>
/// <param name="mode">Enumerant to specify the activation mode.</param>
/// <param name="reluNanOpt">Nan propagation option for the relu.</param>
/// <param name="coef">floating point number to specify the clipping threashold when the activation
/// mode is set to CUDNN_ACTIVATION_CLIPPED_RELU or to specify the alpha
/// coefficient when the activation mode is set to CUDNN_ACTIVATION_ELU.</param>
public void SetActivationDescriptor(cudnnActivationMode mode,
cudnnNanPropagation reluNanOpt,
double coef)
{
res = CudaDNNNativeMethods.cudnnSetActivationDescriptor(_desc, mode, reluNanOpt, coef);
Debug.Write("");//Line(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetActivationDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public SizeT InitTransformDest(TensorDescriptor srcDesc, TensorDescriptor destDesc)
{
SizeT destSizeInBytes = new SizeT();
res = CudaDNNNativeMethods.cudnnInitTransformDest(_desc, srcDesc.Desc, destDesc.Desc, ref destSizeInBytes);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnInitTransformDest", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(destSizeInBytes);
}
public void SetFilterNdDescriptor(cudnnDataType dataType, // image data type
int nbDims,
int[] filterDimA
)
{
res = CudaDNNNativeMethods.cudnnSetFilterNdDescriptor(_desc, dataType, nbDims, filterDimA);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetFilterNdDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/* Set all data points of a tensor to a given value : srcDest = alpha * srcDest */
public void ScaleTensor(TensorDescriptor srcDestDesc,
CudaDeviceVariable <double> srcDestData,
double alpha
)
{
res = CudaDNNNativeMethods.cudnnScaleTensor(_handle, srcDestDesc.Desc, srcDestData.DevicePointer, ref alpha);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnScaleTensor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
///
/// </summary>
/// <param name="opTensorOp"></param>
/// <param name="opTensorCompType"></param>
/// <param name="opTensorNanOpt"></param>
public void GetOpTensorDescriptor(
ref cudnnOpTensorOp opTensorOp,
ref cudnnDataType opTensorCompType,
ref cudnnNanPropagation opTensorNanOpt)
{
res = CudaDNNNativeMethods.cudnnGetOpTensorDescriptor(_desc, ref opTensorOp, ref opTensorCompType, ref opTensorNanOpt);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetOpTensorDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public void GetAlgorithmPerformance(int index, ref AlgorithmDescriptor algoDesc, ref cudnnStatus status, ref float time, ref SizeT memory)
{
cudnnAlgorithmDescriptor descTemp = new cudnnAlgorithmDescriptor();
res = CudaDNNNativeMethods.cudnnGetAlgorithmPerformance(_perfs[index], ref descTemp, ref status, ref time, ref memory);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetAlgorithmPerformance", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
algoDesc = new AlgorithmDescriptor(descTemp);
}
public cudnnAlgorithm GetAlgorithmDescriptor()
{
cudnnAlgorithm algo = new cudnnAlgorithm();
res = CudaDNNNativeMethods.cudnnGetAlgorithmDescriptor(_desc, ref algo);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetAlgorithmDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(algo);
}
/// <summary>
/// Retrieves the values stored in a previously initialized tensor transform descriptor.
/// </summary>
public void GetTensorTransformDescriptor(uint nbDims,
cudnnTensorFormat destFormat, int[] padBeforeA,
int[] padAfterA, uint[] foldA,
cudnnFoldingDirection direction)
{
res = CudaDNNNativeMethods.cudnnGetTensorTransformDescriptor(_desc, nbDims, ref destFormat, padBeforeA, padAfterA, foldA, ref direction);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetTensorTransformDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public SizeT GetAlgorithmSpaceSize(CudaDNNContext ctx)
{
SizeT size = new SizeT();
res = CudaDNNNativeMethods.cudnnGetAlgorithmSpaceSize(ctx.Handle, _desc, ref size);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetAlgorithmSpaceSize", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(size);
}
/// <summary>
///
/// </summary>
public cudnnDataType GetCTCLossDescriptor()
{
cudnnDataType dataType = new cudnnDataType();
res = CudaDNNNativeMethods.cudnnGetCTCLossDescriptor(_desc, ref dataType);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetCTCLossDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(dataType);
}
/// <summary>
/// This function returns the size of the tensor in memory in respect to the given descriptor.
/// This function can be used to know the amount of GPU memory to be allocated to hold that tensor.
/// </summary>
/// <returns>Size in bytes needed to hold the tensor in GPU memory.</returns>
public SizeT GetTensorSizeInBytes()
{
SizeT retVal = 0;
res = CudaDNNNativeMethods.cudnnGetTensorSizeInBytes(_desc, ref retVal);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetTensorSizeInBytes", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
return(retVal);
}
/// <summary>
/// For IDisposable
/// </summary>
/// <param name="fDisposing"></param>
protected virtual void Dispose(bool fDisposing)
{
if (fDisposing && !disposed)
{
//Ignore if failing
res = CudaDNNNativeMethods.cudnnDestroyPoolingDescriptor(_desc);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnDestroyPoolingDescriptor", res));
disposed = true;
}
if (!fDisposing && !disposed)
Debug.WriteLine(String.Format("ManagedCUDA not-disposed warning: {0}", this.GetType()));
}
// Retrieve the settings currently stored in an LRN layer descriptor
// Any of the provided pointers can be NULL (no corresponding value will be returned)
public void GetLRNDescriptor(ref uint lrnN,
ref double lrnAlpha,
ref double lrnBeta,
ref double lrnK
)
{
res = CudaDNNNativeMethods.cudnnGetLRNDescriptor(_desc, ref lrnN, ref lrnAlpha, ref lrnBeta, ref lrnK);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetLRNDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public void SetSeqDataDescriptor(cudnnDataType dataType, int nbDims,
int[] dimA, cudnnSeqDataAxis[] axes,
int[] seqLengthArray, float paddingFill)
{
res = CudaDNNNativeMethods.cudnnSetSeqDataDescriptor(_desc, dataType, nbDims,
dimA, axes,
seqLengthArray.Length, seqLengthArray, ref paddingFill);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetSeqDataDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
public void GetSeqDataDescriptor(ref cudnnDataType dataType, ref int nbDims,
int[] dimA, cudnnSeqDataAxis[] axes,
ref SizeT seqLengthArraySize, int[] seqLengthArray, IntPtr paddingFill)
{
res = CudaDNNNativeMethods.cudnnGetSeqDataDescriptor(_desc, ref dataType, ref nbDims,
dimA.Length, dimA, axes,
ref seqLengthArraySize, dimA.Length, seqLengthArray, paddingFill);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetSeqDataDescriptor", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This function provides the output dimensions of a tensor after 2d pooling has been applied
/// </summary>
/// <param name="inputTensorDesc">Handle to the previously initialized input tensor descriptor.</param>
/// <param name="n">Number of images in the output</param>
/// <param name="c">Number of channels in the output</param>
/// <param name="h">Height of images in the output</param>
/// <param name="w">Width of images in the output</param>
public void GetPooling2dForwardOutputDim(TensorDescriptor inputTensorDesc,
ref int n,
ref int c,
ref int h,
ref int w)
{
res = CudaDNNNativeMethods.cudnnGetPooling2dForwardOutputDim(_desc, inputTensorDesc.Desc, ref n, ref c, ref h, ref w);
Debug.Write(""); //Line(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetPooling2dForwardOutputDim", res));
if (res != cudnnStatus.Success)
{
throw new CudaDNNException(res);
}
}
/// <summary>
/// This routine executes the recurrent neural network described by rnnDesc with
/// output gradients dy, dhy, dhc, weights w and input gradients dx, dhx, dcx.
/// workspace is required for intermediate storage. The data in reserveSpace must have
/// previously been generated by cudnnRNNForwardTraining. The same reserveSpace data must
/// be used for future calls to cudnnRNNBackwardWeights if they execute on the same input data.
/// </summary>
/// <param name="yDesc">An array of tensor descriptors describing the output from each
/// recurrent iteration. The first dimension of the tensor depends on the direction
/// argument passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc:
/// * If direction is CUDNN_UNIDIRECTIONAL the first dimension should match the hiddenSize
/// argument passed to cudnnSetRNNDescriptor.
/// * If direction is CUDNN_BIDIRECTIONAL the first dimension should match double the
/// hiddenSize argument passed to cudnnSetRNNDescriptor.
/// The second dimension of the tensor n must match the second dimension of the tensor n in dyDesc.
/// The tensor must be fully packed.</param>
/// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor yDesc.</param>
/// <param name="dyDesc">An array of tensor descriptors describing the gradient at the output from each
/// recurrent iteration. The first dimension of the tensor depends on the direction argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc:
/// * If direction is CUDNN_UNIDIRECTIONAL the first dimension should match the hiddenSize
/// argument passed to cudnnSetRNNDescriptor.
/// * If direction is CUDNN_BIDIRECTIONAL the first dimension should match double the hiddenSize
/// argument passed to cudnnSetRNNDescriptor.
/// The second dimension of the tensor n must match the second dimension of the tensor n in dxDesc. The
/// tensor must be fully packed.</param>
/// <param name="dy">Data pointer to GPU memory associated with the tensor descriptors in the array dyDesc.</param>
/// <param name="dhyDesc">Handle to a previously initialized tensor descriptor describing the gradients at the
/// final hidden state of the RNN. The first dimension of the tensor must match the hiddenSize argument passed
/// to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the second
/// dimension of the first tensor described in dyDesc. The third dimension must match the numLayers argument
/// passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully packed.</param>
/// <param name="dhy">Data pointer to GPU memory associated with the tensor descriptor dhyDesc. If a NULL pointer
/// is passed, the gradients at the final hidden state of the network will be initialized to zero.</param>
/// <param name="dcyDesc">Handle to a previously initialized tensor descriptor describing the gradients at
/// the final cell state of the RNN. The first dimension of the tensor must match the hiddenSize argument
/// passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the
/// second dimension of the first tensor described in dyDesc. The third dimension must match the numLayers argument
/// passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully packed.</param>
/// <param name="dcy">Data pointer to GPU memory associated with the tensor descriptor dcyDesc. If a NULL pointer
/// is passed, the gradients at the final cell state of the network will be initialized to zero.</param>
/// <param name="wDesc">Handle to a previously initialized filter descriptor describing the weights for the RNN.</param>
/// <param name="w">Data pointer to GPU memory associated with the filter descriptor wDesc.</param>
/// <param name="hxDesc">Handle to a previously initialized tensor descriptor describing the initial hidden
/// state of the RNN. The first dimension of the tensor must match the hiddenSize argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the second
/// dimension of the first tensor described in xDesc. The third dimension must match the numLayers
/// argument passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be
/// fully packed.</param>
/// <param name="hx">Data pointer to GPU memory associated with the tensor descriptor hxDesc. If a NULL pointer is
/// passed, the initial hidden state of the network will be initialized to zero.</param>
/// <param name="cxDesc">Handle to a previously initialized tensor descriptor describing the
/// initial cell state for LSTM networks. The first dimension of the tensor must match the
/// hiddenSize argument passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The
/// second dimension must match the second dimension of the first tensor described in xDesc. The
/// third dimension must match the numLayers argument passed to the cudnnSetRNNDescriptor call
/// used to initialize rnnDesc. The tensor must be fully packed.</param>
/// <param name="cx">Data pointer to GPU memory associated with the tensor descriptor cxDesc.
/// If a NULL pointer is passed, the initial cell state of the network will be initialized to zero.</param>
/// <param name="dxDesc">An array of tensor descriptors describing the gradient at the input of each recurrent iteration.
/// Each tensor descriptor must have the same first dimension. The second dimension of the tensors may decrease from
/// element n to element n+1 but may not increase. The tensor must be fully packed.</param>
/// <param name="dx">Data pointer to GPU memory associated with the tensor descriptors in the array dxDesc. </param>
/// <param name="dhxDesc">Handle to a previously initialized tensor descriptor describing the gradient at the initial hidden
/// state of the RNN. The first dimension of the tensor must match the hiddenSize argument passed to the cudnnSetRNNDescriptor
/// call used to initialize rnnDesc. The second dimension must match the second dimension of the first tensor described in xDesc.
/// The third dimension must match the numLayers argument passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc.
/// The tensor must be fully packed.</param>
/// <param name="dhx">Data pointer to GPU memory associated with the tensor descriptor dhxDesc. If a NULL pointer is passed, the
/// gradient at the hidden input of the network will not be set.</param>
/// <param name="dcxDesc">Handle to a previously initialized tensor descriptor describing the gradient
/// at the initial cell state of the RNN. The first dimension of the tensor must match the hiddenSize argument passed
/// to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the second dimension
/// of the first tensor described in xDesc. The third dimension must match the numLayers argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully packed.</param>
/// <param name="dcx">Data pointer to GPU memory associated with the tensor descriptor dcxDesc. If
/// a NULL pointer is passed, the gradient at the cell input of the network will not be set.</param>
/// <param name="workspace">Data pointer to GPU memory to be used as a workspace for this call.</param>
/// <param name="workSpaceSizeInBytes">Specifies the size in bytes of the provided workspace.</param>
/// <param name="reserveSpace">Data pointer to GPU memory to be used as a reserve space for this call.</param>
/// <param name="reserveSpaceSizeInBytes">Specifies the size in bytes of the provided reserveSpace.</param>
public void RNNBackwardData(
TensorDescriptor[] yDesc,
CudaDeviceVariable<float> y,
TensorDescriptor[] dyDesc,
CudaDeviceVariable<float> dy,
TensorDescriptor dhyDesc,
CudaDeviceVariable<float> dhy,
TensorDescriptor dcyDesc,
CudaDeviceVariable<float> dcy,
FilterDescriptor wDesc,
CudaDeviceVariable<float> w,
TensorDescriptor hxDesc,
CudaDeviceVariable<float> hx,
TensorDescriptor cxDesc,
CudaDeviceVariable<float> cx,
TensorDescriptor[] dxDesc,
CudaDeviceVariable<float> dx,
TensorDescriptor dhxDesc,
CudaDeviceVariable<float> dhx,
TensorDescriptor dcxDesc,
CudaDeviceVariable<float> dcx,
CudaDeviceVariable<byte> workspace,
SizeT workSpaceSizeInBytes,
CudaDeviceVariable<byte> reserveSpace,
SizeT reserveSpaceSizeInBytes)
{
var a1 = yDesc.Select(q => q.Desc).ToArray();
var a2 = dyDesc.Select(q => q.Desc).ToArray();
var a3 = dxDesc.Select(q => q.Desc).ToArray();
res = CudaDNNNativeMethods.cudnnRNNBackwardData(
_handle, _desc, a1, y.DevicePointer, a2, dy.DevicePointer, dhyDesc.Desc, dhy.DevicePointer, dcyDesc.Desc, dcy.DevicePointer, wDesc.Desc, w.DevicePointer,
hxDesc.Desc, hx.DevicePointer, cxDesc.Desc, cx.DevicePointer, a3, dx.DevicePointer, dhxDesc.Desc, dhx.DevicePointer, dcxDesc.Desc, dcx.DevicePointer,
workspace.DevicePointer, workSpaceSizeInBytes, reserveSpace.DevicePointer, reserveSpaceSizeInBytes);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnRNNBackwardData", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void GetPooling2dDescriptor(ref cudnnPoolingMode mode,
ref int windowHeight,
ref int windowWidth,
ref int verticalPadding,
ref int horizontalPadding,
ref int verticalStride,
ref int horizontalStride
)
{
res = CudaDNNNativeMethods.cudnnGetPooling2dDescriptor(_desc, ref mode, ref windowHeight, ref windowWidth, ref verticalPadding, ref horizontalPadding, ref verticalStride, ref horizontalStride);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetPooling2dDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
// LRN uses a window [center-lookBehind, center+lookAhead], where
// lookBehind = floor( (lrnN-1)/2 ), lookAhead = lrnN-lookBehind-1.
// So for n=10, the window is [k-4...k...k+5] with a total of 10 samples.
// Values of double parameters will be cast down to tensor data type.
public void SetLRNDescriptor(uint lrnN,
double lrnAlpha,
double lrnBeta,
double lrnK
)
{
res = CudaDNNNativeMethods.cudnnSetLRNDescriptor(_desc, lrnN, lrnAlpha, lrnBeta, lrnK);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetLRNDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void GetConvolutionNdDescriptor(int arrayLengthRequested,
ref int arrayLength,
int[] padA,
int[] strideA,
int[] upscaleA,
ref cudnnConvolutionMode mode
)
{
res = CudaDNNNativeMethods.cudnnGetConvolutionNdDescriptor(_desc, arrayLengthRequested, ref arrayLength, padA, strideA, upscaleA, ref mode);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetConvolutionNdDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/* Softmax functions: All of the form "output = alpha * Op(inputs) + beta * output" */
/* Function to perform forward softmax */
public void SoftmaxForward(cudnnSoftmaxAlgorithm algorithm,
cudnnSoftmaxMode mode,
float alpha,
TensorDescriptor srcDesc,
CudaDeviceVariable<float> srcData,
float beta,
TensorDescriptor destDesc,
CudaDeviceVariable<float> destData
)
{
res = CudaDNNNativeMethods.cudnnSoftmaxForward(_handle, algorithm, mode, ref alpha, srcDesc.Desc, srcData.DevicePointer, ref beta, destDesc.Desc, destData.DevicePointer);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSoftmaxForward", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/* Tensor layout conversion helper (dest = alpha * src + beta * dest) */
public void TransformTensor(float alpha,
TensorDescriptor srcDesc,
CudaDeviceVariable<float> srcData,
float beta,
TensorDescriptor destDesc,
CudaDeviceVariable<float> destData
)
{
res = CudaDNNNativeMethods.cudnnTransformTensor(_handle, ref alpha, srcDesc.Desc, srcData.DevicePointer, ref beta, destDesc.Desc, destData.DevicePointer);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnTransformTensor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function initializes a previously created generic Tensor descriptor object into a
/// 4D tensor, similarly to cudnnSetTensor4dDescriptor but with the strides explicitly
/// passed as parameters. This can be used to lay out the 4D tensor in any order or simply to
/// define gaps between dimensions.
/// </summary>
/// <param name="dataType">Data type.</param>
/// <param name="n">Number of images.</param>
/// <param name="c">Number of feature maps per image.</param>
/// <param name="h">Height of each feature map.</param>
/// <param name="w">Width of each feature map.</param>
/// <param name="nStride">Stride between two consecutive images.</param>
/// <param name="cStride">Stride between two consecutive feature maps.</param>
/// <param name="hStride">Stride between two consecutive rows.</param>
/// <param name="wStride">Stride between two consecutive columns.</param>
public void SetTensor4dDescriptorEx(cudnnDataType dataType, // image data type
int n, // number of inputs (batch size)
int c, // number of input feature maps
int h, // height of input section
int w, // width of input section
int nStride,
int cStride,
int hStride,
int wStride
)
{
res = CudaDNNNativeMethods.cudnnSetTensor4dDescriptorEx(_desc, dataType, n, c, h, w, nStride, cStride, hStride, wStride);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetTensor4dDescriptorEx", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function is used to obtain a pointer and descriptor for the matrix parameters in layer within
/// the RNN described by rnnDesc with inputs dimensions defined by xDesc.
/// </summary>
/// <param name="layer">The layer to query.</param>
/// <param name="xDesc">An array of tensor descriptors describing the input to each recurrent iteration.</param>
/// <param name="wDesc">Handle to a previously initialized filter descriptor describing the weights for the RNN.</param>
/// <param name="w">Data pointer to GPU memory associated with the filter descriptor wDesc.</param>
/// <param name="linLayerID">
/// The linear layer to obtain information about:
/// * If mode in rnnDesc was set to CUDNN_RNN_RELU or CUDNN_RNN_TANH a value of 0 references the matrix multiplication
/// applied to the input from the previous layer, a value of 1 references the matrix multiplication applied to the recurrent input.
/// * If mode in rnnDesc was set to CUDNN_LSTM values of 0-3 reference matrix multiplications applied to the input from the
/// previous layer, value of 4-7 reference matrix multiplications applied to the recurrent input.
/// ‣ Values 0 and 4 reference the input gate.
/// ‣ Values 1 and 5 reference the forget gate.
/// ‣ Values 2 and 6 reference the new memory gate.
/// ‣ Values 3 and 7 reference the output gate.
/// * If mode in rnnDesc was set to CUDNN_GRU values of 0-2 reference matrix multiplications applied to the input
/// from the previous layer, value of 3-5 reference matrix multiplications applied to the recurrent input.
/// ‣ Values 0 and 3 reference the reset gate.
/// ‣ Values 1 and 4 reference the update gate.
/// ‣ Values 2 and 5 reference the new memory gate.
/// </param>
/// <param name="linLayerMatDesc">Handle to a previously created filter descriptor.</param>
/// <param name="linLayerMat">Data pointer to GPU memory associated with the filter descriptor linLayerMatDesc.</param>
public void GetRNNLinLayerMatrixParams(
int layer,
TensorDescriptor[] xDesc,
FilterDescriptor wDesc,
CudaDeviceVariable<double> w,
int linLayerID,
FilterDescriptor linLayerMatDesc,
CudaDeviceVariable<SizeT> linLayerMat // void **
)
{
var a1 = xDesc.Select(x => x.Desc).ToArray();
res = CudaDNNNativeMethods.cudnnGetRNNLinLayerMatrixParams(_handle, _desc, layer, a1, wDesc.Desc, w.DevicePointer, linLayerID, linLayerMatDesc.Desc, linLayerMat.DevicePointer);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetRNNLinLayerMatrixParams", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void cudnnDivisiveNormalizationForward(
cudnnDivNormMode mode,
double alpha,
cudnnTensorDescriptor srcDesc, // same desc for means, temp, temp2
CUdeviceptr srcData,
CUdeviceptr srcMeansData, // if NULL, means are assumed to be zero
CUdeviceptr tempData,
CUdeviceptr tempData2,
double beta,
cudnnTensorDescriptor destDesc,
CUdeviceptr destData)
{
res = CudaDNNNativeMethods.cudnnDivisiveNormalizationForward(_handle, _desc, mode, ref alpha, srcDesc, srcData, srcMeansData, tempData, tempData2, ref beta, destDesc, destData);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnDivisiveNormalizationForward", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function initializes a previously created RNN descriptor object.
/// </summary>
/// <param name="hiddenSize">Size of the internal hidden state for each layer.</param>
/// <param name="seqLength">Number of iterations to unroll over.</param>
/// <param name="numLayers">Number of layers.</param>
/// <param name="dropoutDesc">Handle to a previously created and initialized dropout descriptor.</param>
/// <param name="inputMode">Specifies the behavior at the input to the first layer.</param>
/// <param name="direction">Specifies the recurrence pattern. (eg. bidirectional)</param>
/// <param name="mode">The type of RNN to compute.</param>
/// <param name="dataType">Math precision.</param>
public void SetRNNDescriptor(
int hiddenSize,
int seqLength,
int numLayers,
DropoutDescriptor dropoutDesc, // Between layers, not between recurrent steps.
cudnnRNNInputMode inputMode,
cudnnDirectionMode direction,
cudnnRNNMode mode,
cudnnDataType dataType)
{
res = CudaDNNNativeMethods.cudnnSetRNNDescriptor(_desc, hiddenSize, seqLength, numLayers, dropoutDesc.Desc, inputMode, direction, mode, dataType);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetRNNDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function initializes a previously created filter descriptor object. Filters layout must
/// be contiguous in memory.
/// </summary>
/// <param name="dataType">Data type.</param>
/// <param name="format">Enumerant holding the layout format.</param>
/// <param name="nbDims">Dimension of the filter.</param>
/// <param name="filterDimA">Array of dimension nbDims containing the size of the filter for each dimension.</param>
public void SetFilterNdDescriptor(cudnnDataType dataType, // image data type
cudnnTensorFormat format,
int nbDims,
int[] filterDimA
)
{
res = CudaDNNNativeMethods.cudnnSetFilterNdDescriptor(_desc, dataType, format, nbDims, filterDimA);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetFilterNdDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function initializes a previously created filter descriptor object into a 4D filter.
/// Filters layout must be contiguous in memory.
/// </summary>
/// <param name="dataType">Data type.</param>
/// <param name="format">Enumerant holding the layout format.</param>
/// <param name="k">Number of output feature maps.</param>
/// <param name="c">Number of input feature maps.</param>
/// <param name="h">Height of each filter.</param>
/// <param name="w">Width of each filter.</param>
public void SetFilter4dDescriptor(cudnnDataType dataType, // image data type
cudnnTensorFormat format,
int k, // number of output feature maps
int c, // number of input feature maps
int h, // height of each input filter
int w // width of each input fitler
)
{
res = CudaDNNNativeMethods.cudnnSetFilter4dDescriptor(_desc, dataType, format, k, c, h, w);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetFilter4dDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void GetConvolution2dDescriptor(ref int pad_h, // zero-padding height
ref int pad_w, // zero-padding width
ref int u, // vertical filter stride
ref int v, // horizontal filter stride
ref int upscalex, // upscale the input in x-direction
ref int upscaley, // upscale the input in y-direction
ref cudnnConvolutionMode mode
)
{
res = CudaDNNNativeMethods.cudnnGetConvolution2dDescriptor(_desc, ref pad_h, ref pad_w, ref u, ref v, ref upscalex, ref upscaley, ref mode);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetConvolution2dDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void SetConvolutionNdDescriptor(int arrayLength, /* nbDims-2 size */
int[] padA,
int[] filterStrideA,
int[] upscaleA,
cudnnConvolutionMode mode
)
{
res = CudaDNNNativeMethods.cudnnSetConvolutionNdDescriptor(_desc, arrayLength, padA, filterStrideA, upscaleA, mode);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetConvolutionNdDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/* Function to perform backward activation */
public void ActivationBackward(cudnnActivationMode mode,
double alpha,
TensorDescriptor srcDesc,
CudaDeviceVariable<double> srcData,
TensorDescriptor srcDiffDesc,
CudaDeviceVariable<double> srcDiffData,
TensorDescriptor destDesc,
CudaDeviceVariable<double> destData,
double beta,
TensorDescriptor destDiffDesc,
CudaDeviceVariable<double> destDiffData
)
{
res = CudaDNNNativeMethods.cudnnActivationBackward(_handle, mode, ref alpha, srcDesc.Desc, srcData.DevicePointer, srcDiffDesc.Desc, srcDiffData.DevicePointer, destDesc.Desc, destData.DevicePointer, ref beta, destDiffDesc.Desc, destDiffData.DevicePointer);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnActivationForward", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This routine accumulates weight gradients dw from the recurrent neural network described
/// by rnnDesc with inputs x, hx, and outputs y. The mode of operation in this case is additive,
/// the weight gradients calculated will be added to those already existing in dw. workspace
/// is required for intermediate storage. The data in reserveSpace must have previously been
/// generated by cudnnRNNBackwardData.
/// </summary>
/// <param name="xDesc">An array of tensor descriptors describing the input to each recurrent iteration.
/// Each tensor descriptor must have the same first dimension. The second dimension of the tensors may
/// decrease from element n to element n+1 but may not increase. The tensor must be fully packed.</param>
/// <param name="x">Data pointer to GPU memory associated with the tensor descriptors in the array xDesc.</param>
/// <param name="hxDesc">Handle to a previously initialized tensor descriptor describing the initial hidden
/// state of the RNN. The first dimension of the tensor must match the hiddenSize argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the second dimension
/// of the first tensor described in xDesc. The third dimension must match the numLayers argument passed to
/// the cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully packed. </param>
/// <param name="hx">Data pointer to GPU memory associated with the tensor descriptor hxDesc. If
/// a NULL pointer is passed, the initial hidden state of the network will be initialized to zero.</param>
/// <param name="yDesc">An array of tensor descriptors describing the output from each
/// recurrent iteration. The first dimension of the tensor depends on the direction
/// argument passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc:
/// * If direction is CUDNN_UNIDIRECTIONAL the first dimension should match the hiddenSize
/// argument passed to cudnnSetRNNDescriptor.
/// * If direction is CUDNN_BIDIRECTIONAL the first dimension should match double the hiddenSize
/// argument passed to cudnnSetRNNDescriptor.
/// The second dimension of the tensor n must match the second dimension of the tensor n in dyDesc.
/// The tensor must be fully packed.</param>
/// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor yDesc.</param>
/// <param name="workspace">Data pointer to GPU memory to be used as a workspace for this call.</param>
/// <param name="workSpaceSizeInBytes">Specifies the size in bytes of the provided workspace.</param>
/// <param name="dwDesc">Handle to a previously initialized filter descriptor describing the gradients of the weights for the RNN.</param>
/// <param name="dw">Data pointer to GPU memory associated with the filter descriptor dwDesc.</param>
/// <param name="reserveSpace">Data pointer to GPU memory to be used as a reserve space for this call.</param>
/// <param name="reserveSpaceSizeInBytes">Specifies the size in bytes of the provided reserveSpace.</param>
public void RNNBackwardWeights(
TensorDescriptor[] xDesc,
CudaDeviceVariable<float> x,
TensorDescriptor hxDesc,
CudaDeviceVariable<float> hx,
TensorDescriptor[] yDesc,
CudaDeviceVariable<float> y,
CudaDeviceVariable<byte> workspace,
SizeT workSpaceSizeInBytes,
FilterDescriptor dwDesc,
CudaDeviceVariable<float> dw,
CudaDeviceVariable<byte> reserveSpace,
SizeT reserveSpaceSizeInBytes)
{
var a1 = xDesc.Select(q => q.Desc).ToArray();
var a2 = yDesc.Select(q => q.Desc).ToArray();
res = CudaDNNNativeMethods.cudnnRNNBackwardWeights(
_handle, _desc, a1, x.DevicePointer, hxDesc.Desc, hx.DevicePointer, a2, y.DevicePointer, workspace.DevicePointer, workSpaceSizeInBytes, dwDesc.Desc, dw.DevicePointer, reserveSpace.DevicePointer, reserveSpaceSizeInBytes);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnRNNBackwardWeights", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function queries the parameters of the previouly initialized Tensor4D descriptor object.
/// </summary>
/// <param name="dataType">Data type.</param>
/// <param name="n">Number of images.</param>
/// <param name="c">Number of feature maps per image.</param>
/// <param name="h">Height of each feature map.</param>
/// <param name="w">Width of each feature map.</param>
/// <param name="nStride">Stride between two consecutive images.</param>
/// <param name="cStride">Stride between two consecutive feature maps.</param>
/// <param name="hStride">Stride between two consecutive rows.</param>
/// <param name="wStride">Stride between two consecutive columns.</param>
public void GetTensor4dDescriptor(ref cudnnDataType dataType, // image data type
ref int n, // number of inputs (batch size)
ref int c, // number of input feature maps
ref int h, // height of input section
ref int w, // width of input section
ref int nStride,
ref int cStride,
ref int hStride,
ref int wStride
)
{
res = CudaDNNNativeMethods.cudnnGetTensor4dDescriptor(_desc, ref dataType, ref n, ref c, ref h, ref w, ref nStride, ref cStride, ref hStride, ref wStride);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetTensor4dDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function is used to query the amount of parameter space required to execute the RNN described by
/// rnnDesc with inputs dimensions defined by xDesc.
/// </summary>
/// <param name="xDesc">An array of tensor descriptors describing the input to each recurrent iteration</param>
/// <param name="sizeInBytes">Minimum amount of GPU memory needed as parameter space to be able to execute an RNN with the specified descriptor and input tensors.</param>
public void cudnnGetRNNParamsSize(
TensorDescriptor[] xDesc,
ref SizeT sizeInBytes
)
{
var a1 = xDesc.Select(x => x.Desc).ToArray();
res = CudaDNNNativeMethods.cudnnGetRNNParamsSize(_handle, _desc, a1, ref sizeInBytes);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetRNNParamsSize", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This function initializes a previously created generic Tensor descriptor object.
/// </summary>
/// <param name="dataType">Data type.</param>
/// <param name="nbDims">Dimension of the tensor.</param>
/// <param name="dimA">Array of dimension nbDims that contain the size of the tensor for every dimension.</param>
/// <param name="strideA">Array of dimension nbDims that contain the stride of the tensor for every dimension.</param>
public void SetTensorNdDescriptor(cudnnDataType dataType,
int nbDims,
int[] dimA,
int[] strideA
)
{
res = CudaDNNNativeMethods.cudnnSetTensorNdDescriptor(_desc, dataType, nbDims, dimA, strideA);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetTensorNdDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/// <summary>
/// This routine executes the recurrent neural network described by rnnDesc with inputs x, hx, cx, weights w
/// and outputs y, hy, cy. workspace is required for intermediate storage. reserveSpace stores data required
/// for training. The same reserveSpace data must be used for future calls to cudnnRNNBackwardData and
/// cudnnRNNBackwardWeights if these execute on the same input data.
/// </summary>
/// <param name="xDesc">An array of tensor descriptors describing the input to each recurrent iteration. Each
/// tensor descriptor must have the same first dimension. The second dimension of the tensors may decrease
/// from element n to element n+1 but may not increase. The tensor must be fully packed.</param>
/// <param name="x">Data pointer to GPU memory associated with the tensor descriptors in the array xDesc.</param>
/// <param name="hxDesc">Handle to a previously initialized tensor descriptor describing the initial hidden state
/// of the RNN. The first dimension of the tensor must match the hiddenSize argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the second
/// dimension of the first tensor described in xDesc. The third dimension must match the numLayers argument
/// passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully packed.</param>
/// <param name="hx">Data pointer to GPU memory associated with the tensor descriptor hxDesc. If a NULL pointer
/// is passed, the initial hidden state of the network will be initialized to zero.</param>
/// <param name="cxDesc">Handle to a previously initialized tensor descriptor describing the initial
/// cell state for LSTM networks. The first dimension of the tensor must match the hiddenSize argument
/// passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match
/// the second dimension of the first tensor described in xDesc. The third dimension must match the numLayers
/// argument passed to the cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully
/// packed.</param>
/// <param name="cx">Data pointer to GPU memory associated with the tensor descriptor cxDesc. If a NULL pointer is
/// passed, the initial cell state of the network will be initialized to zero.</param>
/// <param name="wDesc">Handle to a previously initialized filter descriptor describing the weights for the RNN.</param>
/// <param name="w">Data pointer to GPU memory associated with the filter descriptor wDesc.</param>
/// <param name="yDesc">An array of tensor descriptors describing the output from each recurrent iteration. The first
/// dimension of the tensor depends on the direction argument passed to the cudnnSetRNNDescriptor
/// call used to initialize rnnDesc:
/// * If direction is CUDNN_UNIDIRECTIONAL the first dimension should match the hiddenSize
/// argument passed to cudnnSetRNNDescriptor.
/// * If direction is CUDNN_BIDIRECTIONAL the first dimension should match double the hiddenSize
/// argument passed to cudnnSetRNNDescriptor.
/// The second dimension of the tensor n must match the second dimension of the tensor
/// n in xDesc. The tensor must be fully packed.</param>
/// <param name="y">Data pointer to GPU memory associated with the output tensor descriptor yDesc.</param>
/// <param name="hyDesc">Handle to a previously initialized tensor descriptor describing the final
/// hidden state of the RNN. The first dimension of the tensor must match the hiddenSize argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the second dimension
/// of the first tensor described in xDesc. The third dimension must match the numLayers argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully packed.</param>
/// <param name="hy">Data pointer to GPU memory associated with the tensor descriptor hyDesc. If a
/// NULL pointer is passed, the final hidden state of the network will not be saved.</param>
/// <param name="cyDesc">Handle to a previously initialized tensor descriptor describing the final cell state
/// for LSTM networks. The first dimension of the tensor must match the hiddenSize argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The second dimension must match the second dimension
/// of the first tensor described in xDesc. The third dimension must match the numLayers argument passed to the
/// cudnnSetRNNDescriptor call used to initialize rnnDesc. The tensor must be fully packed.</param>
/// <param name="cy">Data pointer to GPU memory associated with the tensor descriptor cyDesc. If a NULL pointer is
/// passed, the final cell state of the network will be not be saved.</param>
/// <param name="workspace">Data pointer to GPU memory to be used as a workspace for this call.</param>
/// <param name="workSpaceSizeInBytes">Specifies the size in bytes of the provided workspace.</param>
/// <param name="reserveSpace">Data pointer to GPU memory to be used as a reserve space for this call.</param>
/// <param name="reserveSpaceSizeInBytes">Specifies the size in bytes of the provided reserveSpace.</param>
public void RNNForwardTraining(
TensorDescriptor[] xDesc,
CudaDeviceVariable<double> x,
TensorDescriptor hxDesc,
CudaDeviceVariable<double> hx,
TensorDescriptor cxDesc,
CudaDeviceVariable<double> cx,
FilterDescriptor wDesc,
CudaDeviceVariable<double> w,
TensorDescriptor[] yDesc,
CudaDeviceVariable<double> y,
TensorDescriptor hyDesc,
CudaDeviceVariable<double> hy,
TensorDescriptor cyDesc,
CudaDeviceVariable<double> cy,
CudaDeviceVariable<byte> workspace,
SizeT workSpaceSizeInBytes,
CudaDeviceVariable<byte> reserveSpace,
SizeT reserveSpaceSizeInBytes)
{
var a1 = xDesc.Select(q => q.Desc).ToArray();
var a2 = yDesc.Select(q => q.Desc).ToArray();
res = CudaDNNNativeMethods.cudnnRNNForwardTraining(
_handle, _desc, a1, x.DevicePointer, hxDesc.Desc, hx.DevicePointer, cxDesc.Desc, cx.DevicePointer, wDesc.Desc, w.DevicePointer,
a2, y.DevicePointer, hyDesc.Desc, hy.DevicePointer, cyDesc.Desc, cy.DevicePointer, workspace.DevicePointer, workSpaceSizeInBytes, reserveSpace.DevicePointer, reserveSpaceSizeInBytes);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnRNNForwardTraining", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/* Set all data points of a tensor to a given value : srcDest = value */
public void SetTensor(TensorDescriptor srcDestDesc,
CudaDeviceVariable<double> srcDestData,
CudaDeviceVariable<double> value
)
{
res = CudaDNNNativeMethods.cudnnSetTensor(_handle, srcDestDesc.Desc, srcDestData.DevicePointer, value.DevicePointer);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetTensor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void cudnnLRNCrossChannelForward(
cudnnLRNMode lrnMode,
double alpha,
cudnnTensorDescriptor srcDesc,
CUdeviceptr srcData,
double beta,
cudnnTensorDescriptor destDesc,
CUdeviceptr destData)
{
res = CudaDNNNativeMethods.cudnnLRNCrossChannelForward(_handle, _desc, lrnMode, ref alpha, srcDesc, srcData, ref beta, destDesc, destData);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnLRNCrossChannelForward", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void GetPoolingNdForwardOutputDim(TensorDescriptor inputTensorDesc,
int nbDims,
int[] outputTensorDimA)
{
res = CudaDNNNativeMethods.cudnnGetPoolingNdForwardOutputDim(_desc, inputTensorDesc.Desc, nbDims, outputTensorDimA);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetPoolingNdForwardOutputDim", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void SetPoolingNdDescriptor(cudnnPoolingMode mode,
int nbDims,
int[] windowDimA,
int[] paddingA,
int[] strideA
)
{
res = CudaDNNNativeMethods.cudnnSetPoolingNdDescriptor(_desc, mode, nbDims, windowDimA, paddingA, strideA);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnSetPoolingNdDescriptor", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
/* Helper function to return the dimensions of the output tensor given a convolution descriptor */
public void GetConvolution2dForwardOutputDim(TensorDescriptor inputTensorDesc,
FilterDescriptor filterDesc,
ref int n,
ref int c,
ref int h,
ref int w
)
{
res = CudaDNNNativeMethods.cudnnGetConvolution2dForwardOutputDim(_desc, inputTensorDesc.Desc, filterDesc.Desc, ref n, ref c, ref h, ref w);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnGetConvolution2dForwardOutputDim", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
public void cudnnDivisiveNormalizationBackward(
cudnnDivNormMode mode,
float alpha,
cudnnTensorDescriptor srcDesc, // same desc for diff, means, temp, temp2
CUdeviceptr srcData,
CUdeviceptr srcMeansData, // if NULL, means are assumed to be zero
CUdeviceptr srcDiffData,
CUdeviceptr tempData,
CUdeviceptr tempData2,
float betaData,
cudnnTensorDescriptor destDataDesc, // same desc for dest, means, meansDiff
CUdeviceptr destDataDiff, // output data differential
CUdeviceptr destMeansDiff // output means differential, can be NULL
)
{
res = CudaDNNNativeMethods.cudnnDivisiveNormalizationBackward(_handle, _desc, mode, ref alpha, srcDesc, srcData, srcMeansData, srcDiffData, tempData, tempData2, ref betaData, destDataDesc, destDataDiff, destMeansDiff);
Debug.WriteLine(String.Format("{0:G}, {1}: {2}", DateTime.Now, "cudnnDivisiveNormalizationBackward", res));
if (res != cudnnStatus.Success) throw new CudaDNNException(res);
}
