/// <summary>
/// Factory method to construct an OrtValue of Tensor type on top of pre-allocated memory.
/// This can be a piece of native memory allocated by OrtAllocator (possibly on a device)
/// or a piece of pinned managed memory.
///
/// The resulting OrtValue does not own the underlying memory buffer and will not attempt to
/// deallocate it.
/// </summary>
/// <param name="memInfo">Memory Info. For managed memory it is a default cpu.
/// For Native memory must be obtained from the allocator or OrtMemoryAllocation instance</param>
/// <param name="elementType">DataType for the Tensor</param>
/// <param name="shape">Tensor shape</param>
/// <param name="dataBuffer">Pointer to a raw memory buffer</param>
/// <param name="bufferLength">Buffer length in bytes</param>
/// <returns>A disposable instance of OrtValue</returns>
public static OrtValue CreateTensorValueWithData(OrtMemoryInfo memInfo, TensorElementType elementType,
long[] shape,
IntPtr dataBuffer,
long bufferLength)
{
Type type;
int width;
TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width);
if (width < 1)
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unsupported data type (such as string)");
}
var shapeSize = ArrayUtilities.GetSizeForShape(shape);
if ((shapeSize * width) > bufferLength)
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Can not bind the shape to smaller buffer");
}
IntPtr ortValueHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue(
memInfo.Pointer,
dataBuffer,
(UIntPtr)bufferLength,
shape,
(UIntPtr)shape.Length,
elementType,
out ortValueHandle
));
return(new OrtValue(ortValueHandle));
}
/// <summary>
/// This is a factory method that creates a disposable instance of FixedBufferOnnxValue
/// on top of a buffer. Internally, it will pin managed buffer and will create
/// an OrtValue containing a tensor that will not own the memory.
/// Such instance of FixedBufferOnnxValue can be used both as input and output in InferenceSession.Run()
/// overload. As compared to CreateFromTensor(), this allows you to pass in buffers with custom data types
/// that are blittable as defined in https://docs.microsoft.com/en-us/dotnet/framework/interop/blittable-and-non-blittable-types
/// I.e. those that have the same binary representation as the original type. This includes all existing types
/// but may also allow using custom types for Float16 and BFloat16 providing they have the same layout and size.
/// The resulting instance must be disposed of to release pinned memory and deallocate native OrtValue
/// See example below.
/// </summary>
/// <typeparam name="T">Blittable data type, compatible with supported types</typeparam>
/// <param name="memoryInfo">memoryInfo. For managed buffers simply use OrtMemoryInfo.DefaultInstance</param>
/// <param name="memory"></param>
/// <param name="elementType">TensorElementType</param>
/// <param name="shape">shape of the tensor to be created</param>
/// <param name="bytesSize">size of the allocation in bytes</param>
/// <returns>a disposable instance of FixedBufferOnnxValue</returns>
/// <example>
/// Here is an example of using a 3rd party library class for processing float16/bfloat16.
/// Currently, to pass tensor data and create a tensor one must copy data to Float16/BFloat16 structures
/// so DenseTensor can recognize it.
///
/// If you are using a library that has a class Half and it is blittable, that is its managed in memory representation
/// matches native one and its size is 16-bits, you can use the following conceptual example
/// to feed/fetch data for inference using Half array. This allows you to avoid copying data from your Half[] to Float16[]
///
/// \code{.cs}
/// unsafe { Debug.Assert(sizeof(ushort) == sizeof(Half)); }
/// Half[] input = new Half[] { 5646, 12345 };
/// var input_shape = new long[] {input.Length};
/// Half[] output = new Half[40]; // Whatever the expected len/shape is must match
/// var output_shape = new long[] {output.Length};
///
/// var memInfo = OrtMemoryInfo.DefaultInstance; // CPU
///
/// using(var fixedBufferInput = FixedBufferOnnxvalue.CreateFromMemory<Half>(memInfo,
/// input, TensorElementType.Float16, input_shape, input.Length * sizeof(ushort))
/// using(var fixedBufferOutput = FixedBufferOnnxvalue.CreateFromMemory<Half>(memInfo,
/// output, TensorElementType.Float16, output_shape, output.Length * sizeof(ushort))
/// {
/// FixedBufferOnnxvalue[] inputValues = new FixedBufferOnnxvalue[]{fixedBufferInput};
/// FixedBufferOnnxvalue[] outputValues = new FixedBufferOnnxvalue[]{fixedBufferOutput};
/// session.Run(inputNames, inputValues, outputNames, outputValues);
/// // Output is now in output[]
/// }
/// \endcode
/// </example>
public static FixedBufferOnnxValue CreateFromMemory <T>(OrtMemoryInfo memoryInfo, Memory <T> memory,
TensorElementType elementType, long[] shape, long bytesSize)
{
if (elementType == TensorElementType.String)
{
throw new ArgumentException("String data type is not supported");
}
var memHandle = memory.Pin();
try
{
IntPtr memPtr;
unsafe
{
memPtr = (IntPtr)memHandle.Pointer;
}
var ortValue = OrtValue.CreateTensorValueWithData(memoryInfo,
elementType,
shape,
memPtr, bytesSize);
return(new FixedBufferOnnxValue(memHandle, ortValue, OnnxValueType.ONNX_TYPE_TENSOR, elementType));
}
catch (Exception)
{
memHandle.Dispose();
throw;
}
}
/// <summary>
/// This function will bind model output with the given name to a device
/// specified by the memInfo.
/// </summary>
/// <param name="name">output name</param>
/// <param name="memInfo">instance of memory info</param>
public void BindOutputToDevice(string name, OrtMemoryInfo memInfo)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutputToDevice(handle, pinnedName.Pointer, memInfo.Pointer));
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="memInfo">use to accurately describe a piece of memory that this is wrapping</param>
/// <param name="shape">shape of this buffer</param>
/// <param name="elementType">element type</param>
/// <param name="pointer">the actual pointer to memory</param>
/// <param name="sizeInBytes">size of the allocation in bytes</param>
public OrtExternalAllocation(OrtMemoryInfo memInfo, long[] shape, Tensors.TensorElementType elementType, IntPtr pointer, long sizeInBytes)
{
Type type;
int width;
if (!TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width))
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument,
"Unable to query type information for data type: " + elementType.ToString());
}
if (elementType == TensorElementType.String)
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument,
"Strings are not supported by this API");
}
var shapeSize = ArrayUtilities.GetSizeForShape(shape);
var requiredBufferSize = shapeSize * width;
if (requiredBufferSize > sizeInBytes)
{
var message = String.Format("Shape of {0} elements requires a buffer of at least {1} bytes. Provided: {2} bytes",
shapeSize, requiredBufferSize, sizeInBytes);
throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message);
}
Info = memInfo;
Shape = shape;
ElementType = elementType;
Pointer = pointer;
Size = sizeInBytes;
}
public bool Equals(OrtMemoryInfo other)
{
if (this == other)
{
return(true);
}
int result = -1;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCompareMemoryInfo(handle, other.Pointer, out result));
return(result == 0);
}
/// <summary>
/// Factory method to construct an OrtValue of Tensor type on top of pre-allocated memory.
/// This can be a piece of native memory allocated by OrtAllocator (possibly on a device)
/// or a piece of pinned managed memory.
///
/// The resulting OrtValue does not own the underlying memory buffer and will not attempt to
/// deallocate it.
/// </summary>
/// <param name="memInfo">Memory Info. For managed memory it is a default cpu.
/// For Native memory must be obtained from the allocator or OrtMemoryAllocation instance</param>
/// <param name="elementType">DataType for the Tensor</param>
/// <param name="shape">Tensor shape</param>
/// <param name="dataBuffer">Pointer to a raw memory buffer</param>
/// <param name="bufferLength">Buffer length in bytes</param>
/// <returns>A disposable instance of OrtValue</returns>
public static OrtValue CreateTensorValueWithData(OrtMemoryInfo memInfo, TensorElementType elementType,
long[] shape,
IntPtr dataBuffer,
long bufferLength)
{
Type type;
int width;
if (!TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width))
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument,
"Unable to query type information for data type: " + elementType.ToString());
}
if (elementType == TensorElementType.String)
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument,
"Cannot map managed strings buffer to native OrtValue");
}
var shapeSize = ArrayUtilities.GetSizeForShape(shape);
var requiredBufferSize = shapeSize * width;
if (requiredBufferSize > bufferLength)
{
var message = String.Format("Shape of: {0} elements requires a buffer of at least {1} bytes. Provided: {2} bytes",
shapeSize, requiredBufferSize, bufferLength);
throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message);
}
IntPtr ortValueHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue(
memInfo.Pointer,
dataBuffer,
(UIntPtr)bufferLength,
shape,
(UIntPtr)shape.Length,
elementType,
out ortValueHandle
));
return(new OrtValue(ortValueHandle));
}
/// <summary>
/// Creates an instance of OrtAllocator according to the specifications in OrtMemorInfo.
/// The requested allocator should be available within the given session instance. This means
/// both, the native library was build with specific allocators (for instance CUDA) and the corresponding
/// provider was added to SessionsOptions before instantiating the session object.
/// </summary>
/// <param name="session"></param>
/// <param name="memInfo"></param>
public OrtAllocator(InferenceSession session, OrtMemoryInfo memInfo)
: base(IntPtr.Zero, true)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateAllocator(session.Handle, memInfo.Pointer, out handle));
_owned = true;
}
/// <summary>
/// Create and register an allocator to the OrtEnv instance
/// so as to enable sharing across all sessions using the OrtEnv instance
/// <param name="memInfo">OrtMemoryInfo instance to be used for allocator creation</param>
/// <param name="arenaCfg">OrtArenaCfg instance that will be used to define the behavior of the arena based allocator</param>
/// </summary>
public void CreateAndRegisterAllocator(OrtMemoryInfo memInfo, OrtArenaCfg arenaCfg)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateAndRegisterAllocator(Handle, memInfo.Pointer, arenaCfg.Pointer));
}
