public void Backward(CudnnPoolingDescriptor pooling, CudnnTensorDescriptor srcTensor, double[] srcData, CudnnTensorDescriptor srcDiffTensor, double[] srcDiffData,
CudnnTensorDescriptor destTensor, double[] destData, CudnnTensorDescriptor destDiffTensor, double[] destDiffData)
{
Contract.Requires(pooling != null);
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(destTensor != null);
Contract.Requires(destData != null);
Contract.Requires(srcDiffTensor != null);
Contract.Requires(srcDiffData != null);
Contract.Requires(destDiffTensor != null);
Contract.Requires(destDiffData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Double, srcTensor, srcDiffTensor, destTensor, destDiffTensor);
using (var srcDataGpu = new CudaDeviceVariable <double>(srcData.Length))
using (var srcDiffDataGpu = new CudaDeviceVariable <double>(srcDiffData.Length))
using (var destDataGpu = new CudaDeviceVariable <double>(destData.Length))
using (var destDiffDataGpu = new CudaDeviceVariable <double>(destDiffData.Length))
{
srcDataGpu.CopyToDevice(srcData);
srcDiffDataGpu.CopyToDevice(srcDiffData);
destDataGpu.CopyToDevice(destData);
Invoke(() => CudnnNativeMethods.cudnnPoolingBackward(handle, pooling.Handle,
srcTensor.Handle, srcDataGpu.DevicePointer, srcDiffTensor.Handle, srcDiffDataGpu.DevicePointer,
destTensor.Handle, destDataGpu.DevicePointer, destDiffTensor.Handle, destDiffDataGpu.DevicePointer));
destDiffDataGpu.CopyToHost(destDiffData);
}
}
private void DisposeNative()
{
Contract.Ensures(this.Handle.Pointer == IntPtr.Zero);
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnDestroyConvolutionDescriptor(this.Handle));
this.Handle.Pointer = IntPtr.Zero;
}
public void Backward(CudnnSoftmaxAlgorithm algorithm, CudnnSoftmaxMode mode,
CudnnTensorDescriptor srcTensor, float[] srcData, CudnnTensorDescriptor srcDiffTensor, float[] srcDiffData,
CudnnTensorDescriptor destDiffTensor, float[] destDiffData)
{
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(srcDiffTensor != null);
Contract.Requires(srcDiffData != null);
Contract.Requires(destDiffTensor != null);
Contract.Requires(destDiffData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Float, srcTensor, srcDiffTensor, destDiffTensor);
using (var srcDataGpu = new CudaDeviceVariable <float>(srcData.Length))
using (var srcDiffDataGpu = new CudaDeviceVariable <float>(srcDiffData.Length))
using (var destDiffDataGpu = new CudaDeviceVariable <float>(destDiffData.Length))
{
srcDataGpu.CopyToDevice(srcData);
srcDiffDataGpu.CopyToDevice(srcDiffData);
Invoke(() => CudnnNativeMethods.cudnnSoftmaxBackward(handle, algorithm, mode,
srcTensor.Handle, srcDataGpu.DevicePointer, srcDiffTensor.Handle, srcDiffDataGpu.DevicePointer,
destDiffTensor.Handle, destDiffDataGpu.DevicePointer));
destDiffDataGpu.CopyToHost(destDiffData);
}
}
public void BackwardData(CudnnFilterDescriptor filter, double[] filterData, CudnnTensorDescriptor diffTensor, double[] diffData, CudnnConvolutionDescriptor convolution, CudnnTensorDescriptor gradient, double[] gradientData, CudnnAccumulateResult accumulate)
{
Contract.Requires(filter != null);
Contract.Requires(filterData != null);
Contract.Requires(diffTensor != null);
Contract.Requires(diffData != null);
Contract.Requires(convolution != null);
Contract.Requires(gradient != null);
Contract.Requires(gradientData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Double, filter, diffTensor, gradient);
using (var filterDataGpu = new CudaDeviceVariable <double>(filterData.Length))
using (var diffDataGpu = new CudaDeviceVariable <double>(diffData.Length))
using (var gradientDataGpu = new CudaDeviceVariable <double>(gradientData.Length))
{
filterDataGpu.CopyToDevice(filterData);
diffDataGpu.CopyToDevice(diffData);
Invoke(() => CudnnNativeMethods.cudnnConvolutionBackwardData(handle, filter.Handle, filterDataGpu.DevicePointer, diffTensor.Handle, diffDataGpu.DevicePointer, convolution.Handle, gradient.Handle, gradientDataGpu.DevicePointer, accumulate));
gradientDataGpu.CopyToHost(gradientData);
}
}
public void Cudnn_Descriptors_ConstructTensorWithMajorRow()
{
var param = new CudnnTensorDescriptorParameters(CudnnType.Float, CudnnTensorFormat.MajorRow, 10, 5, 2, 4);
using (var tensor = CudnnContext.CreateTensor(param))
using (var tensor2 = CudnnContext.CreateTensor())
{
Assert.True(tensor.IsInitialized);
Assert.False(tensor2.IsInitialized);
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnSetTensor4dDescriptor(tensor2.Handle, CudnnTensorFormat.MajorRow, CudnnType.Float, 10, 5, 2, 4));
CudnnType dataType = default(CudnnType);
int n1 = 0, c1 = 0, h1 = 0, w1 = 0;
int nStride1 = 0, cStride1 = 0, hStride1 = 0, wStride1 = 0;
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnGetTensor4dDescriptor(tensor.Handle, out dataType, out n1, out c1, out h1, out w1, out nStride1, out cStride1, out hStride1, out wStride1));
int n2 = 0, c2 = 0, h2 = 0, w2 = 0;
int nStride2 = 0, cStride2 = 0, hStride2 = 0, wStride2 = 0;
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnGetTensor4dDescriptor(tensor2.Handle, out dataType, out n2, out c2, out h2, out w2, out nStride2, out cStride2, out hStride2, out wStride2));
Assert.Equal(n2, n1);
Assert.Equal(c2, c1);
Assert.Equal(h2, h1);
Assert.Equal(w2, w1);
Assert.Equal(nStride2, nStride1);
Assert.Equal(cStride2, cStride1);
Assert.Equal(hStride2, hStride1);
Assert.Equal(wStride2, wStride1);
}
}
public void Cudnn_NativeApi_HandleLifecycle()
{
CudnnHandle handle = default(CudnnHandle);
Success(() => CudnnNativeMethods.cudnnCreate(out handle));
Assert.NotNull(handle);
Success(() => CudnnNativeMethods.cudnnDestroy(handle));
}
public void Cudnn_Descriptors_ConstructTensorWithInterleaved_Current()
{
using (var tensor = CudnnContext.CreateTensor())
{
Assert.False(tensor.IsInitialized);
Assert.Throws <NotSupportedException>(
() => CudnnContext.Invoke(() => CudnnNativeMethods.cudnnSetTensor4dDescriptor(tensor.Handle, CudnnTensorFormat.Interleaved, CudnnType.Double, 10, 5, 2, 4)));
}
}
public static CudnnConvolutionDescriptor CreateConvolution()
{
CudnnConvolutionDescriptorHandle handle = default(CudnnConvolutionDescriptorHandle);
Invoke(() => CudnnNativeMethods.cudnnCreateConvolutionDescriptor(out handle));
Contract.Assume(handle.Pointer != IntPtr.Zero);
return(new CudnnConvolutionDescriptor(handle));
}
public void BackwardBias(CudnnTensorDescriptor srcTensor, CudaDeviceVariable <double> srcData, CudnnTensorDescriptor destTensor, CudaDeviceVariable <double> destData, CudnnAccumulateResult accumulate)
{
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(destTensor != null);
Contract.Requires(destData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Double, srcTensor, destTensor);
Invoke(() => CudnnNativeMethods.cudnnConvolutionBackwardBias(handle, srcTensor.Handle, srcData.DevicePointer, destTensor.Handle, destData.DevicePointer, accumulate));
}
public CudnnConvolutionTensorDim GetOutputTensor(CudnnConvolutionPath path)
{
Contract.Ensures(Contract.Result <CudnnConvolutionTensorDim>() != null);
ThrowIfNotInitialized();
int n = 0, c = 0, h = 0, w = 0;
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnGetOutputTensor4dDim(this.Handle, path, out n, out c, out h, out w));
return(new CudnnConvolutionTensorDim(path, n, c, h, w));
}
public void Forward(CudnnActivationMode mode, CudnnTensorDescriptor srcTensor, CudaDeviceVariable <double> srcData, CudnnTensorDescriptor destTensor, CudaDeviceVariable <double> destData)
{
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(destTensor != null);
Contract.Requires(destData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Double, srcTensor, destTensor);
Invoke(() => CudnnNativeMethods.cudnnActivationForward(handle, mode, srcTensor.Handle, srcData.DevicePointer, destTensor.Handle, destData.DevicePointer));
}
public void Cudnn_NativeApi_TensorLifecycle()
{
CudnnHandle handle = default(CudnnHandle);
Success(() => CudnnNativeMethods.cudnnCreate(out handle));
CudnnTensorDescriptorHandle tensorDescriptor = default(CudnnTensorDescriptorHandle);
Success(() => CudnnNativeMethods.cudnnCreateTensor4dDescriptor(out tensorDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroyTensor4dDescriptor(tensorDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroy(handle));
}
public void Cudnn_NativeApi_PoolingLifecycle()
{
CudnnHandle handle = default(CudnnHandle);
Success(() => CudnnNativeMethods.cudnnCreate(out handle));
CudnnPoolingDescriptorHandle poolingDescriptor = default(CudnnPoolingDescriptorHandle);
Success(() => CudnnNativeMethods.cudnnCreatePoolingDescriptor(out poolingDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroyPoolingDescriptor(poolingDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroy(handle));
}
public void Cudnn_NativeApi_ConvolutionLifecycle()
{
CudnnHandle handle = default(CudnnHandle);
Success(() => CudnnNativeMethods.cudnnCreate(out handle));
CudnnConvolutionDescriptorHandle convolutionDescriptor = default(CudnnConvolutionDescriptorHandle);
Success(() => CudnnNativeMethods.cudnnCreateConvolutionDescriptor(out convolutionDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroyConvolutionDescriptor(convolutionDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroy(handle));
}
public void Cudnn_NativeApi_FilterLifecycle()
{
CudnnHandle handle = default(CudnnHandle);
Success(() => CudnnNativeMethods.cudnnCreate(out handle));
CudnnFilterDescriptorHandle filterDescriptor = default(CudnnFilterDescriptorHandle);
Success(() => CudnnNativeMethods.cudnnCreateFilterDescriptor(out filterDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroyFilterDescriptor(filterDescriptor));
Success(() => CudnnNativeMethods.cudnnDestroy(handle));
}
public void Forward(CudnnPoolingDescriptor pooling, CudnnTensorDescriptor srcTensor, CudaDeviceVariable <float> srcData, CudnnTensorDescriptor destTensor, CudaDeviceVariable <float> destData)
{
Contract.Requires(pooling != null);
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(destTensor != null);
Contract.Requires(destData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Float, srcTensor, destTensor);
Invoke(() => CudnnNativeMethods.cudnnPoolingForward(handle, pooling.Handle, srcTensor.Handle, srcData.DevicePointer, destTensor.Handle, destData.DevicePointer));
}
public static CudnnContext Create(CudaStream stream = null)
{
CudnnHandle handle = default(CudnnHandle);
Invoke(() => CudnnNativeMethods.cudnnCreate(out handle));
Contract.Assume(handle.Pointer != IntPtr.Zero);
if (stream != null)
{
Invoke(() => CudnnNativeMethods.cudnnSetStream(handle, stream.Stream));
}
return(new CudnnContext(handle, stream));
}
private void DisposeNative()
{
try
{
if (this.handle.Pointer != IntPtr.Zero)
{
Invoke(() => CudnnNativeMethods.cudnnDestroy(handle));
}
}
finally
{
this.handle.Pointer = IntPtr.Zero;
}
}
public void Forward(CudnnSoftmaxAlgorithm algorithm, CudnnSoftmaxMode mode,
CudnnTensorDescriptor srcTensor, CudaDeviceVariable <float> srcData, CudnnTensorDescriptor destTensor, CudaDeviceVariable <float> destData)
{
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(destTensor != null);
Contract.Requires(destData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Float, srcTensor, destTensor);
Invoke(() => CudnnNativeMethods.cudnnSoftmaxForward(handle, algorithm, mode,
srcTensor.Handle, srcData.DevicePointer, destTensor.Handle, destData.DevicePointer));
}
public void SetParameters(CudnnTensorDescriptorParameters param)
{
if (param == null)
{
throw new ArgumentNullException("param");
}
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnSetTensor4dDescriptorEx(
this.Handle, param.Type,
param.Num, param.Channels, param.Height, param.Width,
param.NumStride, param.ChannelsStride, param.HeightStride, param.WidthStride));
this.descriptorParams = param;
}
public void SetParameters(CudnnFilterDescriptorParameters param)
{
if (param == null)
{
throw new ArgumentNullException("param");
}
Contract.EndContractBlock();
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnSetFilterDescriptor(
this.Handle, param.Type,
param.Output, param.Input, param.Height, param.Width));
this.descriptorParams = param;
}
public void BackwardFilter(CudnnTensorDescriptor srcTensor, CudaDeviceVariable <double> srcData, CudnnTensorDescriptor diffTensor, CudaDeviceVariable <double> diffData, CudnnConvolutionDescriptor convolution, CudnnFilterDescriptor gradient, CudaDeviceVariable <double> gradientData, CudnnAccumulateResult accumulate)
{
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(diffTensor != null);
Contract.Requires(diffData != null);
Contract.Requires(convolution != null);
Contract.Requires(gradient != null);
Contract.Requires(gradientData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Double, srcTensor, diffTensor, gradient);
Invoke(() => CudnnNativeMethods.cudnnConvolutionBackwardFilter(handle, srcTensor.Handle, srcData.DevicePointer, diffTensor.Handle, diffData.DevicePointer, convolution.Handle, gradient.Handle, gradientData.DevicePointer, accumulate));
}
public void Forward(CudnnTensorDescriptor srcTensor, CudaDeviceVariable <float> srcData, CudnnFilterDescriptor filter, CudaDeviceVariable <float> filterData, CudnnConvolutionDescriptor convolution, CudnnTensorDescriptor destTensor, CudaDeviceVariable <float> destData, CudnnAccumulateResult accumulate)
{
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(filter != null);
Contract.Requires(filterData != null);
Contract.Requires(convolution != null);
Contract.Requires(destTensor != null);
Contract.Requires(destData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Float, srcTensor, destTensor, filter);
Invoke(() => CudnnNativeMethods.cudnnConvolutionForward(handle, srcTensor.Handle, srcData.DevicePointer, filter.Handle, filterData.DevicePointer, convolution.Handle, destTensor.Handle, destData.DevicePointer, accumulate));
}
public void SetParameters(CudnnConvolutionDescriptorParameters param)
{
if (param == null)
{
throw new ArgumentNullException("param");
}
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnSetConvolutionDescriptor(
this.Handle, param.Tensor.Handle, param.Filter.Handle,
param.HeightPadding, param.WidthPadding,
param.HeightStride, param.WidthStride,
param.HeightUpscale, param.WidthUpscale,
param.Mode));
initialized = true;
}
public void ContextWithStream()
{
using (var cuda = new CudaContext())
using (var stream = new CudaStream(CUStreamFlags.Default))
{
using (var context = CudnnContext.Create(stream))
{
Assert.True(context.IsInitialized);
var streamId = default(CUstream);
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnGetStream(context.Handle, out streamId));
Assert.Equal(stream.Stream, streamId);
}
}
}
public void SetParameters(CudnnPoolingDescriptorParameters param)
{
if (param == null)
{
throw new ArgumentNullException("param");
}
Contract.EndContractBlock();
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnSetPoolingDescriptor(
this.Handle, param.Mode,
param.Height, param.Width,
param.HeightStride, param.WidthStride));
this.descriptorParams = param;
}
public void Cudnn_Descriptors_ConstructFilterWithSetup(CudnnFilterDescriptorParameters param)
{
using (var filter = CudnnContext.CreateFilter(param))
{
Assert.True(filter.IsInitialized);
CudnnType dataType = default(CudnnType);
int k = 0, c = 0, h = 0, w = 0;
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnGetFilterDescriptor(filter.Handle, out dataType, out k, out c, out h, out w));
Assert.Equal(filter.Parameters.Type, dataType);
Assert.Equal(filter.Parameters.Output, k);
Assert.Equal(filter.Parameters.Input, c);
Assert.Equal(filter.Parameters.Height, h);
Assert.Equal(filter.Parameters.Width, w);
}
}
public void BackwardBias(CudnnTensorDescriptor srcTensor, double[] srcData, CudnnTensorDescriptor destTensor, double[] destData, CudnnAccumulateResult accumulate)
{
Contract.Requires(srcTensor != null);
Contract.Requires(srcData != null);
Contract.Requires(destTensor != null);
Contract.Requires(destData != null);
ThrowIfNotInitialized();
CheckIfCompatible(CudnnType.Double, srcTensor, destTensor);
using (var srcDataGpu = new CudaDeviceVariable <double>(srcData.Length))
using (var destDataGpu = new CudaDeviceVariable <double>(destData.Length))
{
srcDataGpu.CopyToDevice(srcData);
Invoke(() => CudnnNativeMethods.cudnnConvolutionBackwardBias(handle, srcTensor.Handle, srcDataGpu.DevicePointer, destTensor.Handle, destDataGpu.DevicePointer, accumulate));
destDataGpu.CopyToHost(destData);
}
}
public void Cudnn_Descriptors_ConstructPoolingWithSetup(CudnnPoolingDescriptorParameters param)
{
using (var pooling = CudnnContext.CreatePooling(param))
{
Assert.True(pooling.IsInitialized);
CudnnPoolingMode mode = default(CudnnPoolingMode);
int windowHeight = 0, windowWidth = 0;
int verticalStride = 0, horizontalStride = 0;
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnGetPoolingDescriptor(pooling.Handle, out mode, out windowHeight, out windowWidth, out verticalStride, out horizontalStride));
Assert.Equal(pooling.Parameters.Mode, mode);
Assert.Equal(pooling.Parameters.Height, windowHeight);
Assert.Equal(pooling.Parameters.Width, windowWidth);
Assert.Equal(pooling.Parameters.HeightStride, verticalStride);
Assert.Equal(pooling.Parameters.WidthStride, horizontalStride);
}
}
public void Cudnn_Descriptors_ConstructTensorWithSetup(CudnnTensorDescriptorParameters param)
{
using (var tensor = CudnnContext.CreateTensor(param))
{
Assert.True(tensor.IsInitialized);
CudnnType dataType = default(CudnnType);
int n = 0, c = 0, h = 0, w = 0;
int nStride = 0, cStride = 0, hStride = 0, wStride = 0;
CudnnContext.Invoke(() => CudnnNativeMethods.cudnnGetTensor4dDescriptor(tensor.Handle, out dataType, out n, out c, out h, out w, out nStride, out cStride, out hStride, out wStride));
Assert.Equal(tensor.Parameters.Num, n);
Assert.Equal(tensor.Parameters.Channels, c);
Assert.Equal(tensor.Parameters.Height, h);
Assert.Equal(tensor.Parameters.Width, w);
Assert.Equal(tensor.Parameters.NumStride, nStride);
Assert.Equal(tensor.Parameters.ChannelsStride, cStride);
Assert.Equal(tensor.Parameters.HeightStride, hStride);
Assert.Equal(tensor.Parameters.WidthStride, wStride);
}
}
