/// <summary> /// Constructor /// </summary> /// <param name="memInfo">use to accurately describe a piece of memory that this is wrapping</param> /// <param name="shape">shape of this buffer</param> /// <param name="elementType">element type</param> /// <param name="pointer">the actual pointer to memory</param> /// <param name="sizeInBytes">size of the allocation in bytes</param> public OrtExternalAllocation(OrtMemoryInfo memInfo, long[] shape, Tensors.TensorElementType elementType, IntPtr pointer, long sizeInBytes) { Type type; int width; if (!TensorElementTypeConverter.GetTypeAndWidth(elementType, out type, out width)) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Unable to query type information for data type: " + elementType.ToString()); } if (elementType == TensorElementType.String) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Strings are not supported by this API"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); var requiredBufferSize = shapeSize * width; if (requiredBufferSize > sizeInBytes) { var message = String.Format("Shape of {0} elements requires a buffer of at least {1} bytes. Provided: {2} bytes", shapeSize, requiredBufferSize, sizeInBytes); throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message); } Info = memInfo; Shape = shape; ElementType = elementType; Pointer = pointer; Size = sizeInBytes; }
/// <summary> /// Bind model output to an OrtValue as Tensor with a given type and shape. An instance of OrtMemoryAllocaiton /// owns the memory and should be alive for the time of execution.The size of the allocation can not be less than required /// by the Tensor of the given size. /// </summary> /// <param name="name">of the output</param> /// <param name="elementType">tensor element type</param> /// <param name="shape">tensor shape</param> /// <param name="allocation">allocated memory</param> public void BindOutput(string name, Tensors.TensorElementType elementType, long[] shape, OrtMemoryAllocation allocation) { using (var ortValue = OrtValue.CreateTensorValueWithData(allocation.Info, elementType, shape, allocation.Pointer, allocation.Size)) BindInputOrOutput(name, ortValue.Handle, false); }
internal static void GetTypeAndWidth(Tensors.TensorElementType elemType, out Type type, out int width) { TensorElementTypeInfo result = TensorBase.GetElementTypeInfo(elemType); if (result != null) { type = result.TensorType; width = result.TypeSize; } else { throw new ArgumentException("Unable to get information for type: " + elemType.ToString()); } }
internal static void GetTypeAndWidth(Tensors.TensorElementType elemType, out Type type, out int width) { TensorElementTypeInfo result = TensorBase.GetElementTypeInfo(elemType); if (result != null) { type = result.TensorType; width = result.TypeSize; } else { type = null; width = 0; } }
public NativeOnnxTensorMemory(IntPtr onnxValueHandle) { Type type = null; int width = 0; _onnxValueHandle = onnxValueHandle; IntPtr typeAndShape = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(onnxValueHandle, out typeAndShape)); try { TensorElementType elemType; { IntPtr el_type; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShape, out el_type)); elemType = (TensorElementType)el_type; } TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width); if (typeof(T) != type) { throw new NotSupportedException(nameof(NativeOnnxTensorMemory <T>) + " does not support T = " + nameof(T)); } _elementType = elemType; _elementWidth = width; UIntPtr dimension; long count; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(typeAndShape, out dimension)); { IntPtr el_count; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeElementCount(typeAndShape, out el_count)); // count can be negative. count = (long)el_count; } if (count < 0) { throw new NotSupportedException("Symbolic dimensions in the tensor is not supported"); } long[] shape = new long[dimension.ToUInt64()]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(typeAndShape, shape, dimension)); //Note: shape must be alive during the call _elementCount = (int)count; _dimensions = new int[dimension.ToUInt64()]; for (ulong i = 0; i < dimension.ToUInt64(); i++) { _dimensions[i] = (int)shape[i]; } if (typeof(T) != typeof(string)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferPointer)); } else { UIntPtr strLen; var offsets = new UIntPtr[_elementCount]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(_onnxValueHandle, out strLen)); var dataBuffer = new byte[strLen.ToUInt64()]; using (var dataBufferHandle = new Memory <byte>(dataBuffer).Pin()) using (var offsetMemoryHandle = new Memory <UIntPtr>(offsets).Pin()) { unsafe { _dataBufferPointer = (IntPtr)dataBufferHandle.Pointer; NativeApiStatus.VerifySuccess( NativeMethods.OrtGetStringTensorContent( _onnxValueHandle, _dataBufferPointer, strLen, (IntPtr)offsetMemoryHandle.Pointer, (UIntPtr)_elementCount)); } _dataBufferAsString = new string[_elementCount]; for (var i = 0; i < offsets.Length; i++) { var length = (i == offsets.Length - 1) ? strLen.ToUInt64() - offsets[i].ToUInt64() : offsets[i + 1].ToUInt64() - offsets[i].ToUInt64(); // Onnx specifies strings always in UTF-8, no trailing null, no leading BOM _dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length); } } } } finally { NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape); } }
/// <summary> /// Bind a piece of pre-allocated native memory as a OrtValue Tensor with a given shape /// to an input with a given name. The model will read the specified input from that memory /// possibly avoiding the need to copy between devices. OrtMemoryAllocation continues to own /// the chunk of native memory, and the allocation should be alive until the end of execution. /// </summary> /// <param name="name">of the input</param> /// <param name="elementType">Tensor element type</param> /// <param name="shape"></param> /// <param name="allocation">native memory allocation</param> public void BindInput(string name, Tensors.TensorElementType elementType, long[] shape, OrtMemoryAllocation allocation) { BindOrtAllocation(name, elementType, shape, allocation, true); }