/// <summary> /// Factory method to construct an OrtValue of Tensor type on top of pre-allocated memory. /// This can be a piece of native memory allocated by OrtAllocator (possibly on a device) /// or a piece of pinned managed memory. /// /// The resulting OrtValue does not own the underlying memory buffer and will not attempt to /// deallocate it. /// </summary> /// <param name="memInfo">Memory Info. For managed memory it is a default cpu. /// For Native memory must be obtained from the allocator or OrtMemoryAllocation instance</param> /// <param name="elementType">DataType for the Tensor</param> /// <param name="shape">Tensor shape</param> /// <param name="dataBuffer">Pointer to a raw memory buffer</param> /// <param name="bufferLength">Buffer length in bytes</param> /// <returns>A disposable instance of OrtValue</returns> public static OrtValue CreateTensorValueWithData(OrtMemoryInfo memInfo, TensorElementType elementType, long[] shape, IntPtr dataBuffer, long bufferLength) { Type type; int width; TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width); if (width < 1) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unsupported data type (such as string)"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); if ((shapeSize * width) > bufferLength) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Can not bind the shape to smaller buffer"); } IntPtr ortValueHandle = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( memInfo.Pointer, dataBuffer, (UIntPtr)bufferLength, shape, (UIntPtr)shape.Length, elementType, out ortValueHandle )); return(new OrtValue(ortValueHandle)); }
internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) { var valueType = NativeMethods.OrtOnnxTypeFromTypeInfo(typeInfo); if (valueType != OnnxValueType.ONNX_TYPE_TENSOR && valueType != OnnxValueType.ONNX_TYPE_SPARSETENSOR) { return(new NodeMetadata(valueType, new int[] { }, typeof(NamedOnnxValue))); } IntPtr tensorInfo = NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo); // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo* if (tensorInfo == IntPtr.Zero) { return(null); } TensorElementType type = NativeMethods.OrtGetTensorElementType(tensorInfo); Type dotnetType = null; int width = 0; TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width); ulong numDimensions = NativeMethods.OrtGetNumOfDimensions(tensorInfo); long[] dimensions = new long[(int)numDimensions]; NativeMethods.OrtGetDimensions(tensorInfo, dimensions, numDimensions); int[] intDimensions = new int[(int)numDimensions]; for (ulong i = 0; i < numDimensions; i++) { intDimensions[i] = (int)dimensions[i]; } return(new NodeMetadata(valueType, intDimensions, dotnetType)); }
/// <summary> /// Constructor /// </summary> /// <param name="memInfo">use to accurately describe a piece of memory that this is wrapping</param> /// <param name="shape">shape of this buffer</param> /// <param name="elementType">element type</param> /// <param name="pointer">the actual pointer to memory</param> /// <param name="sizeInBytes">size of the allocation in bytes</param> public OrtExternalAllocation(OrtMemoryInfo memInfo, long[] shape, Tensors.TensorElementType elementType, IntPtr pointer, long sizeInBytes) { Type type; int width; if (!TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width)) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unable to query type information for data type: " + elementType.ToString()); } if (elementType == TensorElementType.String) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Strings are not supported by this API"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); var requiredBufferSize = shapeSize * width; if (requiredBufferSize > sizeInBytes) { var message = String.Format("Shape of {0} elements requires a buffer of at least {1} bytes. Provided: {2} bytes", shapeSize, requiredBufferSize, sizeInBytes); throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message); } Info = memInfo; Shape = shape; ElementType = elementType; Pointer = pointer; Size = sizeInBytes; }
internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) { OnnxValueType valueType; { IntPtr valType; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetOnnxTypeFromTypeInfo(typeInfo, out valType)); valueType = (OnnxValueType)valType; } if (valueType != OnnxValueType.ONNX_TYPE_TENSOR && valueType != OnnxValueType.ONNX_TYPE_SPARSETENSOR) { return(new NodeMetadata(valueType, new int[] { }, new string[] { }, typeof(NamedOnnxValue))); } // This should not be released IntPtr tensorInfo; NativeApiStatus.VerifySuccess(NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo, out tensorInfo)); //(IntPtr)(int)(uint) // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo* if (tensorInfo == IntPtr.Zero) { return(null); } TensorElementType type; { IntPtr el_type; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(tensorInfo, out el_type)); type = (TensorElementType)el_type; } Type dotnetType = null; int width = 0; TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width); UIntPtr numDimensions; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(tensorInfo, out numDimensions)); long[] dimensions = new long[(int)numDimensions]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(tensorInfo, dimensions, numDimensions)); int[] intDimensions = new int[(int)numDimensions]; for (var i = 0; i < (long)numDimensions; i++) { intDimensions[i] = (int)dimensions[i]; } IntPtr[] dimensionNamePtrs = new IntPtr[(int)numDimensions]; NativeApiStatus.VerifySuccess( NativeMethods.OrtGetSymbolicDimensions(tensorInfo, dimensionNamePtrs, numDimensions)); string[] symbolicDimensions = new string[(int)numDimensions]; for (var i = 0; i < (int)numDimensions; i++) { symbolicDimensions[i] = NativeOnnxValueHelper.StringFromNativeUtf8(dimensionNamePtrs[i]); } return(new NodeMetadata(valueType, intDimensions, symbolicDimensions, dotnetType)); }
unsafe internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) { OnnxValueType valueType; unsafe { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetOnnxTypeFromTypeInfo(typeInfo, new IntPtr(&valueType))); } if (valueType != OnnxValueType.ONNX_TYPE_TENSOR && valueType != OnnxValueType.ONNX_TYPE_SPARSETENSOR) { return(new NodeMetadata(valueType, new int[] { }, new string[] { }, typeof(NamedOnnxValue))); } IntPtr tensorInfo; NativeApiStatus.VerifySuccess(NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo, out tensorInfo)); //(IntPtr)(int)(uint) // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo* if (tensorInfo == IntPtr.Zero) { return(null); } TensorElementType type; unsafe { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(tensorInfo, new IntPtr(&type))); } Type dotnetType = null; int width = 0; TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width); UIntPtr numDimensions; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(tensorInfo, out numDimensions)); var dimensions = stackalloc long[(int)numDimensions]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(tensorInfo, dimensions, numDimensions)); int[] intDimensions = new int[(int)numDimensions]; for (var i = 0; i < (long)numDimensions; i++) { intDimensions[i] = (int)dimensions[i]; } IntPtr[] dimensionNamePtrs = new IntPtr[(int)numDimensions]; NativeApiStatus.VerifySuccess( NativeMethods.OrtGetSymbolicDimensions(tensorInfo, dimensionNamePtrs, numDimensions)); string[] symbolicDimensions = new string[(int)numDimensions]; for (var i = 0; i < (int)numDimensions; i++) { symbolicDimensions[i] = Marshal.PtrToStringAnsi(dimensionNamePtrs[i]); //assumes charset = ANSI } return(new NodeMetadata(valueType, intDimensions, symbolicDimensions, dotnetType)); }
public NativeOnnxTensorMemory(IntPtr onnxValueHandle) { IntPtr typeAndShape = IntPtr.Zero; try { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeAndType(onnxValueHandle, out typeAndShape)); TensorElementType elemType = NativeMethods.OrtGetTensorElementType(typeAndShape); Type type = null; int width = 0; TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width); if (typeof(T) != type) { throw new NotSupportedException(nameof(NativeOnnxTensorMemory <T>) + " does not support T = " + nameof(T)); } _elementWidth = width; _onnxValueHandle = onnxValueHandle; // derive the databuffer pointer, element_count, element_width, and shape NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferHandle)); // throws OnnxRuntimeException if native call failed ulong dimension = NativeMethods.OrtGetNumOfDimensions(typeAndShape); long count = NativeMethods.OrtGetTensorShapeElementCount(typeAndShape); // count can be negative. if (count < 0) { throw new NotSupportedException("Symbolic dimensions in the tensor is not supported"); } long[] shape = new long[dimension]; NativeMethods.OrtGetDimensions(typeAndShape, shape, dimension); //Note: shape must be alive during the call _elementCount = (int)count; _dimensions = new int[dimension]; for (ulong i = 0; i < dimension; i++) { _dimensions[i] = (int)shape[i]; } } catch (Exception e) { //TODO: cleanup any partially created state //Do not call ReleaseTensor here. If the constructor has thrown exception, then this NativeOnnxTensorWrapper is not created, so caller should take appropriate action to dispose throw e; } finally { if (typeAndShape != IntPtr.Zero) { NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape); } } }
/// <summary> /// Factory method to construct an OrtValue of Tensor type on top of pre-allocated memory. /// This can be a piece of native memory allocated by OrtAllocator (possibly on a device) /// or a piece of pinned managed memory. /// /// The resulting OrtValue does not own the underlying memory buffer and will not attempt to /// deallocate it. /// </summary> /// <param name="memInfo">Memory Info. For managed memory it is a default cpu. /// For Native memory must be obtained from the allocator or OrtMemoryAllocation instance</param> /// <param name="elementType">DataType for the Tensor</param> /// <param name="shape">Tensor shape</param> /// <param name="dataBuffer">Pointer to a raw memory buffer</param> /// <param name="bufferLength">Buffer length in bytes</param> /// <returns>A disposable instance of OrtValue</returns> public static OrtValue CreateTensorValueWithData(OrtMemoryInfo memInfo, TensorElementType elementType, long[] shape, IntPtr dataBuffer, long bufferLength) { Type type; int width; if (!TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width)) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unable to query type information for data type: " + elementType.ToString()); } if (elementType == TensorElementType.String) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Cannot map managed strings buffer to native OrtValue"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); var requiredBufferSize = shapeSize * width; if (requiredBufferSize > bufferLength) { var message = String.Format("Shape of: {0} elements requires a buffer of at least {1} bytes. Provided: {2} bytes", shapeSize, requiredBufferSize, bufferLength); throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message); } IntPtr ortValueHandle = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( memInfo.Pointer, dataBuffer, (UIntPtr)bufferLength, shape, (UIntPtr)shape.Length, elementType, out ortValueHandle )); return(new OrtValue(ortValueHandle)); }
private NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) { IntPtr tensorInfo = NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo); // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo* TensorElementType type = NativeMethods.OrtGetTensorElementType(tensorInfo); Type dotnetType = null; int width = 0; TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width); ulong numDimensions = NativeMethods.OrtGetNumOfDimensions(tensorInfo); long[] dimensions = new long[(int)numDimensions]; NativeMethods.OrtGetDimensions(tensorInfo, dimensions, numDimensions); int[] intDimensions = new int[(int)numDimensions]; for (ulong i = 0; i < numDimensions; i++) { intDimensions[i] = (int)dimensions[i]; } return(new NodeMetadata(intDimensions, dotnetType)); }
public NativeOnnxTensorMemory(IntPtr onnxValueHandle, bool isStringTensor = false) { IntPtr typeAndShape = IntPtr.Zero; try { Type type = null; int width = 0; _onnxValueHandle = onnxValueHandle; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(onnxValueHandle, out typeAndShape)); TensorElementType elemType = NativeMethods.OrtGetTensorElementType(typeAndShape); TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width); if (typeof(T) != type) { throw new NotSupportedException(nameof(NativeOnnxTensorMemory <T>) + " does not support T = " + nameof(T)); } _elementWidth = width; var dimension = NativeMethods.OrtGetDimensionsCount(typeAndShape).ToUInt64(); long count = NativeMethods.OrtGetTensorShapeElementCount(typeAndShape); // count can be negative. if (count < 0) { throw new NotSupportedException("Symbolic dimensions in the tensor is not supported"); } long[] shape = new long[dimension]; NativeMethods.OrtGetDimensions(typeAndShape, shape, (UIntPtr)dimension); //Note: shape must be alive during the call _elementCount = (int)count; _dimensions = new int[dimension]; for (ulong i = 0; i < dimension; i++) { _dimensions[i] = (int)shape[i]; } if (!isStringTensor) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferPointer)); } else { if (typeof(T) != typeof(byte)) { throw new NotSupportedException(nameof(NativeOnnxTensorMemory <T>) + " T = " + nameof(T) + ". Should = byte, when isStringTensor is true"); } UIntPtr strLen; var offsets = new UIntPtr[_elementCount]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(_onnxValueHandle, out strLen)); var dataBuffer = new byte[strLen.ToUInt64()]; var dataBufferMemory = new Memory <byte>(dataBuffer); var dataBufferHandle = dataBufferMemory.Pin(); IntPtr dataBufferPointer = IntPtr.Zero; var offsetMemory = new Memory <UIntPtr>(offsets); var offsetMemoryHandle = offsetMemory.Pin(); IntPtr offsetBufferPointer = IntPtr.Zero; unsafe { dataBufferPointer = (IntPtr)dataBufferHandle.Pointer; offsetBufferPointer = (IntPtr)offsetMemoryHandle.Pointer; } NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorContent(_onnxValueHandle, dataBufferPointer, strLen, offsetBufferPointer, (UIntPtr)_elementCount)); _dataBufferPointer = dataBufferPointer; _dataBufferAsString = new string[_elementCount]; for (var i = 0; i < offsets.Length; i++) { var length = (i == offsets.Length - 1) ? strLen.ToUInt64() - offsets[i].ToUInt64() : offsets[i + 1].ToUInt64() - offsets[i].ToUInt64(); // Onnx specifies strings always in UTF-8, no trailing null, no leading BOM _dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length); } // unpin memory offsetMemoryHandle.Dispose(); dataBufferHandle.Dispose(); } } catch (Exception e) { //TODO: cleanup any partially created state //Do not call ReleaseTensor here. If the constructor has thrown exception, then this NativeOnnxTensorWrapper is not created, so caller should take appropriate action to dispose throw e; } finally { if (typeAndShape != IntPtr.Zero) { NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape); } } }
public NativeOnnxTensorMemory(IntPtr onnxValueHandle) { Type type = null; int width = 0; _onnxValueHandle = onnxValueHandle; IntPtr typeAndShape = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(onnxValueHandle, out typeAndShape)); try { TensorElementType elemType; { IntPtr el_type; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShape, out el_type)); elemType = (TensorElementType)el_type; } TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width); if (typeof(T) != type) { throw new NotSupportedException(nameof(NativeOnnxTensorMemory <T>) + " does not support T = " + nameof(T)); } _elementType = elemType; _elementWidth = width; UIntPtr dimension; long count; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(typeAndShape, out dimension)); { IntPtr el_count; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeElementCount(typeAndShape, out el_count)); // count can be negative. count = (long)el_count; } if (count < 0) { throw new NotSupportedException("Symbolic dimensions in the tensor is not supported"); } long[] shape = new long[dimension.ToUInt64()]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(typeAndShape, shape, dimension)); //Note: shape must be alive during the call _elementCount = (int)count; _dimensions = new int[dimension.ToUInt64()]; for (ulong i = 0; i < dimension.ToUInt64(); i++) { _dimensions[i] = (int)shape[i]; } if (typeof(T) != typeof(string)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferPointer)); } else { UIntPtr strLen; var offsets = new UIntPtr[_elementCount]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(_onnxValueHandle, out strLen)); var dataBuffer = new byte[strLen.ToUInt64()]; using (var dataBufferHandle = new Memory <byte>(dataBuffer).Pin()) using (var offsetMemoryHandle = new Memory <UIntPtr>(offsets).Pin()) { unsafe { _dataBufferPointer = (IntPtr)dataBufferHandle.Pointer; NativeApiStatus.VerifySuccess( NativeMethods.OrtGetStringTensorContent( _onnxValueHandle, _dataBufferPointer, strLen, (IntPtr)offsetMemoryHandle.Pointer, (UIntPtr)_elementCount)); } _dataBufferAsString = new string[_elementCount]; for (var i = 0; i < offsets.Length; i++) { var length = (i == offsets.Length - 1) ? strLen.ToUInt64() - offsets[i].ToUInt64() : offsets[i + 1].ToUInt64() - offsets[i].ToUInt64(); // Onnx specifies strings always in UTF-8, no trailing null, no leading BOM _dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length); } } } } finally { NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape); } }
private string[] _dataBufferAsString; // string tensor values copied into managed memory /// <summary> /// Constructs an instance and takes ownership of ortValue on success /// </summary> /// <param name="ortValue">ortValue that is a Tensor</param> public NativeOnnxTensorMemory(OrtValue ortValue) { Type type = null; int width = 0; IntPtr typeAndShape = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(ortValue.Handle, out typeAndShape)); try { TensorElementType elemType; { IntPtr el_type; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShape, out el_type)); elemType = (TensorElementType)el_type; } if (!TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width)) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unable to query type information for data type: " + elemType.ToString()); } if (typeof(T) != type) { var message = String.Format("The NativeOnnxTensorMemory<T> type being instantiated for T = : {0} while supplied OrtValue contains T = {1}", typeof(T), type); throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message); } ElementType = elemType; ElementWidth = width; UIntPtr dimension; long count; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(typeAndShape, out dimension)); { IntPtr el_count; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeElementCount(typeAndShape, out el_count)); // count can be negative. count = (long)el_count; } if (count < 0) { throw new NotSupportedException("Symbolic dimensions in the tensor is not supported"); } long[] shape = new long[dimension.ToUInt64()]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(typeAndShape, shape, dimension)); //Note: shape must be alive during the call Count = (int)count; Dimensions = new int[dimension.ToUInt64()]; for (ulong i = 0; i < dimension.ToUInt64(); i++) { Dimensions[i] = (int)shape[i]; } if (elemType != TensorElementType.String) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(ortValue.Handle, out _dataBufferPointer)); } else { UIntPtr strLen; var offsets = new UIntPtr[Count]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(ortValue.Handle, out strLen)); var dataBuffer = new byte[strLen.ToUInt64()]; using (var dataBufferHandle = new Memory <byte>(dataBuffer).Pin()) using (var offsetMemoryHandle = new Memory <UIntPtr>(offsets).Pin()) { unsafe { _dataBufferPointer = (IntPtr)dataBufferHandle.Pointer; NativeApiStatus.VerifySuccess( NativeMethods.OrtGetStringTensorContent( ortValue.Handle, _dataBufferPointer, strLen, (IntPtr)offsetMemoryHandle.Pointer, (UIntPtr)Count)); } _dataBufferAsString = new string[Count]; for (var i = 0; i < offsets.Length; i++) { var length = (i == offsets.Length - 1) ? strLen.ToUInt64() - offsets[i].ToUInt64() : offsets[i + 1].ToUInt64() - offsets[i].ToUInt64(); // Onnx specifies strings always in UTF-8, no trailing null, no leading BOM _dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length); } } } // Transfer ownership _ortValue = new OrtValue(ortValue.Disown()); } finally { NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape); } }