/// <summary> /// Factory method to construct an OrtValue of Tensor type on top of pre-allocated memory. /// This can be a piece of native memory allocated by OrtAllocator (possibly on a device) /// or a piece of pinned managed memory. /// /// The resulting OrtValue does not own the underlying memory buffer and will not attempt to /// deallocate it. /// </summary> /// <param name="memInfo">Memory Info. For managed memory it is a default cpu. /// For Native memory must be obtained from the allocator or OrtMemoryAllocation instance</param> /// <param name="elementType">DataType for the Tensor</param> /// <param name="shape">Tensor shape</param> /// <param name="dataBuffer">Pointer to a raw memory buffer</param> /// <param name="bufferLength">Buffer length in bytes</param> /// <returns>A disposable instance of OrtValue</returns> public static OrtValue CreateTensorValueWithData(OrtMemoryInfo memInfo, TensorElementType elementType, long[] shape, IntPtr dataBuffer, long bufferLength) { Type type; int width; TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width); if (width < 1) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unsupported data type (such as string)"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); if ((shapeSize * width) > bufferLength) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Can not bind the shape to smaller buffer"); } IntPtr ortValueHandle = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( memInfo.Pointer, dataBuffer, (UIntPtr)bufferLength, shape, (UIntPtr)shape.Length, elementType, out ortValueHandle )); return(new OrtValue(ortValueHandle)); }
/// <summary> /// This is a factory method that creates a disposable instance of FixedBufferOnnxValue /// on top of a buffer. Internally, it will pin managed buffer and will create /// an OrtValue containing a tensor that will not own the memory. /// Such instance of FixedBufferOnnxValue can be used both as input and output in InferenceSession.Run() /// overload. As compared to CreateFromTensor(), this allows you to pass in buffers with custom data types /// that are blittable as defined in https://docs.microsoft.com/en-us/dotnet/framework/interop/blittable-and-non-blittable-types /// I.e. those that have the same binary representation as the original type. This includes all existing types /// but may also allow using custom types for Float16 and BFloat16 providing they have the same layout and size. /// The resulting instance must be disposed of to release pinned memory and deallocate native OrtValue /// See example below. /// </summary> /// <typeparam name="T">Blittable data type, compatible with supported types</typeparam> /// <param name="memoryInfo">memoryInfo. For managed buffers simply use OrtMemoryInfo.DefaultInstance</param> /// <param name="memory"></param> /// <param name="elementType">TensorElementType</param> /// <param name="shape">shape of the tensor to be created</param> /// <param name="bytesSize">size of the allocation in bytes</param> /// <returns>a disposable instance of FixedBufferOnnxValue</returns> /// <example> /// Here is an example of using a 3rd party library class for processing float16/bfloat16. /// Currently, to pass tensor data and create a tensor one must copy data to Float16/BFloat16 structures /// so DenseTensor can recognize it. /// /// If you are using a library that has a class Half and it is blittable, that is its managed in memory representation /// matches native one and its size is 16-bits, you can use the following conceptual example /// to feed/fetch data for inference using Half array. This allows you to avoid copying data from your Half[] to Float16[] /// /// \code{.cs} /// unsafe { Debug.Assert(sizeof(ushort) == sizeof(Half)); } /// Half[] input = new Half[] { 5646, 12345 }; /// var input_shape = new long[] {input.Length}; /// Half[] output = new Half[40]; // Whatever the expected len/shape is must match /// var output_shape = new long[] {output.Length}; /// /// var memInfo = OrtMemoryInfo.DefaultInstance; // CPU /// /// using(var fixedBufferInput = FixedBufferOnnxvalue.CreateFromMemory<Half>(memInfo, /// input, TensorElementType.Float16, input_shape, input.Length * sizeof(ushort)) /// using(var fixedBufferOutput = FixedBufferOnnxvalue.CreateFromMemory<Half>(memInfo, /// output, TensorElementType.Float16, output_shape, output.Length * sizeof(ushort)) /// { /// FixedBufferOnnxvalue[] inputValues = new FixedBufferOnnxvalue[]{fixedBufferInput}; /// FixedBufferOnnxvalue[] outputValues = new FixedBufferOnnxvalue[]{fixedBufferOutput}; /// session.Run(inputNames, inputValues, outputNames, outputValues); /// // Output is now in output[] /// } /// \endcode /// </example> public static FixedBufferOnnxValue CreateFromMemory <T>(OrtMemoryInfo memoryInfo, Memory <T> memory, TensorElementType elementType, long[] shape, long bytesSize) { if (elementType == TensorElementType.String) { throw new ArgumentException("String data type is not supported"); } var memHandle = memory.Pin(); try { IntPtr memPtr; unsafe { memPtr = (IntPtr)memHandle.Pointer; } var ortValue = OrtValue.CreateTensorValueWithData(memoryInfo, elementType, shape, memPtr, bytesSize); return(new FixedBufferOnnxValue(memHandle, ortValue, OnnxValueType.ONNX_TYPE_TENSOR, elementType)); } catch (Exception) { memHandle.Dispose(); throw; } }
/// <summary> /// This function will bind model output with the given name to a device /// specified by the memInfo. /// </summary> /// <param name="name">output name</param> /// <param name="memInfo">instance of memory info</param> public void BindOutputToDevice(string name, OrtMemoryInfo memInfo) { var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned); using (var pinnedName = new PinnedGCHandle(utf8NamePinned)) NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutputToDevice(handle, pinnedName.Pointer, memInfo.Pointer)); }
/// <summary> /// Constructor /// </summary> /// <param name="memInfo">use to accurately describe a piece of memory that this is wrapping</param> /// <param name="shape">shape of this buffer</param> /// <param name="elementType">element type</param> /// <param name="pointer">the actual pointer to memory</param> /// <param name="sizeInBytes">size of the allocation in bytes</param> public OrtExternalAllocation(OrtMemoryInfo memInfo, long[] shape, Tensors.TensorElementType elementType, IntPtr pointer, long sizeInBytes) { Type type; int width; if (!TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width)) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unable to query type information for data type: " + elementType.ToString()); } if (elementType == TensorElementType.String) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Strings are not supported by this API"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); var requiredBufferSize = shapeSize * width; if (requiredBufferSize > sizeInBytes) { var message = String.Format("Shape of {0} elements requires a buffer of at least {1} bytes. Provided: {2} bytes", shapeSize, requiredBufferSize, sizeInBytes); throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message); } Info = memInfo; Shape = shape; ElementType = elementType; Pointer = pointer; Size = sizeInBytes; }
public bool Equals(OrtMemoryInfo other) { if (this == other) { return(true); } int result = -1; NativeApiStatus.VerifySuccess(NativeMethods.OrtCompareMemoryInfo(handle, other.Pointer, out result)); return(result == 0); }
/// <summary> /// Factory method to construct an OrtValue of Tensor type on top of pre-allocated memory. /// This can be a piece of native memory allocated by OrtAllocator (possibly on a device) /// or a piece of pinned managed memory. /// /// The resulting OrtValue does not own the underlying memory buffer and will not attempt to /// deallocate it. /// </summary> /// <param name="memInfo">Memory Info. For managed memory it is a default cpu. /// For Native memory must be obtained from the allocator or OrtMemoryAllocation instance</param> /// <param name="elementType">DataType for the Tensor</param> /// <param name="shape">Tensor shape</param> /// <param name="dataBuffer">Pointer to a raw memory buffer</param> /// <param name="bufferLength">Buffer length in bytes</param> /// <returns>A disposable instance of OrtValue</returns> public static OrtValue CreateTensorValueWithData(OrtMemoryInfo memInfo, TensorElementType elementType, long[] shape, IntPtr dataBuffer, long bufferLength) { Type type; int width; if (!TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width)) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unable to query type information for data type: " + elementType.ToString()); } if (elementType == TensorElementType.String) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Cannot map managed strings buffer to native OrtValue"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); var requiredBufferSize = shapeSize * width; if (requiredBufferSize > bufferLength) { var message = String.Format("Shape of: {0} elements requires a buffer of at least {1} bytes. Provided: {2} bytes", shapeSize, requiredBufferSize, bufferLength); throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message); } IntPtr ortValueHandle = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( memInfo.Pointer, dataBuffer, (UIntPtr)bufferLength, shape, (UIntPtr)shape.Length, elementType, out ortValueHandle )); return(new OrtValue(ortValueHandle)); }
/// <summary> /// Creates an instance of OrtAllocator according to the specifications in OrtMemorInfo. /// The requested allocator should be available within the given session instance. This means /// both, the native library was build with specific allocators (for instance CUDA) and the corresponding /// provider was added to SessionsOptions before instantiating the session object. /// </summary> /// <param name="session"></param> /// <param name="memInfo"></param> public OrtAllocator(InferenceSession session, OrtMemoryInfo memInfo) : base(IntPtr.Zero, true) { NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateAllocator(session.Handle, memInfo.Pointer, out handle)); _owned = true; }
/// <summary> /// Create and register an allocator to the OrtEnv instance /// so as to enable sharing across all sessions using the OrtEnv instance /// <param name="memInfo">OrtMemoryInfo instance to be used for allocator creation</param> /// <param name="arenaCfg">OrtArenaCfg instance that will be used to define the behavior of the arena based allocator</param> /// </summary> public void CreateAndRegisterAllocator(OrtMemoryInfo memInfo, OrtArenaCfg arenaCfg) { NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateAndRegisterAllocator(Handle, memInfo.Pointer, arenaCfg.Pointer)); }