internal static DisposableNamedOnnxValue CreateFromOrtValue(string name, OrtValue ortValue, OrtAllocator allocator)
{
DisposableNamedOnnxValue result = null;
IntPtr valueType;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValueType(ortValue.Handle, out valueType));
OnnxValueType onnxValueType = (OnnxValueType)valueType;
switch (onnxValueType)
{
case OnnxValueType.ONNX_TYPE_TENSOR:
result = CreateTensorFromOnnxValue(name, ortValue);
break;
case OnnxValueType.ONNX_TYPE_SEQUENCE:
result = DisposableNamedOnnxValueFromSequence(name, ortValue, allocator);
break;
case OnnxValueType.ONNX_TYPE_MAP:
result = DisposableNamedOnnxValueFromNativeMap(name, ortValue, allocator);
break;
default:
throw new NotSupportedException("OnnxValueType : " + onnxValueType + " is not supported");
}
return(result);
}
/// <summary>
/// This method will create an instance of DisposableNamedOnnxValue that will own ortSequenceValue
/// an all disposable native objects that are elements of the sequence
/// </summary>
/// <param name="name"></param>
/// <param name="ortValueSequence">ortValueElement that has native sequence</param>
/// <param name="allocator"> used allocator</param>
/// <returns>DisposableNamedOnnxValue</returns>
private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromSequence(string name, OrtValue ortValueSequence, OrtAllocator allocator)
{
DisposableNamedOnnxValue result = null;
IntPtr count;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValueCount(ortValueSequence.Handle, out count));
var sequence = new DisposableList <DisposableNamedOnnxValue>(count.ToInt32());
try
{
for (int i = 0; i < count.ToInt32(); i++)
{
IntPtr nativeOnnxValueSeq;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(ortValueSequence.Handle, i, allocator.Pointer, out nativeOnnxValueSeq));
using (var ortValueElement = new OrtValue(nativeOnnxValueSeq))
{
// Will take ownership or throw
sequence.Add(CreateFromOrtValue(string.Empty, ortValueElement, allocator));
}
}
// NativeOrtValueCollectionOwner will take ownership of ortValueSequence and will make sure sequence
// is also disposed.
var nativeCollectionManager = new NativeOrtValueCollectionOwner(ortValueSequence, sequence);
result = new DisposableNamedOnnxValue(name, sequence, OnnxValueType.ONNX_TYPE_SEQUENCE, TensorElementType.DataTypeMax, nativeCollectionManager);
}
catch (Exception)
{
sequence.Dispose();
throw;
}
return(result);
}
private FixedBufferOnnxValue(MemoryHandle pinnedMemory, OrtValue ortValue, OnnxValueType onnxValueType, TensorElementType elementType)
{
PinnedMemory = pinnedMemory;
Value = ortValue;
OnnxValueType = onnxValueType;
ElementType = elementType;
}
/// <summary>
/// This is a factory method that creates a disposable instance of FixedBufferOnnxValue
/// on top of a buffer. Internally, it will pin managed buffer and will create
/// an OrtValue containing a tensor that will not own the memory.
/// Such instance of FixedBufferOnnxValue can be used both as input and output in InferenceSession.Run()
/// overload. As compared to CreateFromTensor(), this allows you to pass in buffers with custom data types
/// that are blittable as defined in https://docs.microsoft.com/en-us/dotnet/framework/interop/blittable-and-non-blittable-types
/// I.e. those that have the same binary representation as the original type. This includes all existing types
/// but may also allow using custom types for Float16 and BFloat16 providing they have the same layout and size.
/// The resulting instance must be disposed of to release pinned memory and deallocate native OrtValue
/// See example below.
/// </summary>
/// <typeparam name="T">Blittable data type, compatible with supported types</typeparam>
/// <param name="memoryInfo">memoryInfo. For managed buffers simply use OrtMemoryInfo.DefaultInstance</param>
/// <param name="memory"></param>
/// <param name="elementType">TensorElementType</param>
/// <param name="shape">shape of the tensor to be created</param>
/// <param name="bytesSize">size of the allocation in bytes</param>
/// <returns>a disposable instance of FixedBufferOnnxValue</returns>
/// <example>
/// Here is an example of using a 3rd party library class for processing float16/bfloat16.
/// Currently, to pass tensor data and create a tensor one must copy data to Float16/BFloat16 structures
/// so DenseTensor can recognize it.
///
/// If you are using a library that has a class Half and it is blittable, that is its managed in memory representation
/// matches native one and its size is 16-bits, you can use the following conceptual example
/// to feed/fetch data for inference using Half array. This allows you to avoid copying data from your Half[] to Float16[]
///
/// \code{.cs}
/// unsafe { Debug.Assert(sizeof(ushort) == sizeof(Half)); }
/// Half[] input = new Half[] { 5646, 12345 };
/// var input_shape = new long[] {input.Length};
/// Half[] output = new Half[40]; // Whatever the expected len/shape is must match
/// var output_shape = new long[] {output.Length};
///
/// var memInfo = OrtMemoryInfo.DefaultInstance; // CPU
///
/// using(var fixedBufferInput = FixedBufferOnnxvalue.CreateFromMemory<Half>(memInfo,
/// input, TensorElementType.Float16, input_shape, input.Length * sizeof(ushort))
/// using(var fixedBufferOutput = FixedBufferOnnxvalue.CreateFromMemory<Half>(memInfo,
/// output, TensorElementType.Float16, output_shape, output.Length * sizeof(ushort))
/// {
/// FixedBufferOnnxvalue[] inputValues = new FixedBufferOnnxvalue[]{fixedBufferInput};
/// FixedBufferOnnxvalue[] outputValues = new FixedBufferOnnxvalue[]{fixedBufferOutput};
/// session.Run(inputNames, inputValues, outputNames, outputValues);
/// // Output is now in output[]
/// }
/// \endcode
/// </example>
public static FixedBufferOnnxValue CreateFromMemory <T>(OrtMemoryInfo memoryInfo, Memory <T> memory,
TensorElementType elementType, long[] shape, long bytesSize)
{
if (elementType == TensorElementType.String)
{
throw new ArgumentException("String data type is not supported");
}
var memHandle = memory.Pin();
try
{
IntPtr memPtr;
unsafe
{
memPtr = (IntPtr)memHandle.Pointer;
}
var ortValue = OrtValue.CreateTensorValueWithData(memoryInfo,
elementType,
shape,
memPtr, bytesSize);
return(new FixedBufferOnnxValue(memHandle, ortValue, OnnxValueType.ONNX_TYPE_TENSOR, elementType));
}
catch (Exception)
{
memHandle.Dispose();
throw;
}
}
internal static DisposableNamedOnnxValue CreateFromOrtValue(string name, OrtValue ortValue)
{
var result = CreateFromOnnxValue(name, ortValue.Handle, OrtAllocator.DefaultInstance);
ortValue.Disown();
return(result);
}
private static OrtValue CreateStringTensor(Tensor <string> tensor)
{
if (tensor == null)
{
throw new OnnxRuntimeException(ErrorCode.Fail, "Cast to Tensor<string> failed. BUG check!");
}
int totalLength = 0;
for (int i = 0; i < tensor.Length; i++)
{
totalLength += System.Text.Encoding.UTF8.GetByteCount(tensor.GetValue(i));
}
long[] shape = new long[tensor.Dimensions.Length];
for (int i = 0; i < tensor.Dimensions.Length; i++)
{
shape[i] = tensor.Dimensions[i];
}
// allocate the native tensor
IntPtr valueHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue(
OrtAllocator.DefaultInstance.Pointer,
shape,
(UIntPtr)(shape.Length),
TensorElementType.String,
out valueHandle
));
var ortValue = new OrtValue(valueHandle);
try
{
// fill the native tensor, using GetValue(index) from the Tensor<string>
var len = tensor.Length;
var nativeStrings = new IntPtr[len];
using (var pinnedHandles = new DisposableList <PinnedGCHandle>((int)len))
{
for (int i = 0; i < len; i++)
{
var utf8str = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(tensor.GetValue(i));
var gcHandle = GCHandle.Alloc(utf8str, GCHandleType.Pinned);
nativeStrings[i] = gcHandle.AddrOfPinnedObject();
pinnedHandles.Add(new PinnedGCHandle(gcHandle));
}
using (var pinnedStrings = new PinnedGCHandle(GCHandle.Alloc(nativeStrings, GCHandleType.Pinned)))
NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(ortValue.Handle, nativeStrings, (UIntPtr)len));
}
}
catch (OnnxRuntimeException)
{
ortValue.Dispose();
throw;
}
return(ortValue);
}
/// <summary>
/// Add a pre-allocated initializer to a session. If a model contains an initializer with a name
/// that is same as the name passed to this API call, ORT will use this initializer instance
/// instead of deserializing one from the model file. This is useful when you want to share
/// the same initializer across sessions.
/// \param name name of the initializer
/// \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
/// managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
/// to which it is added.
/// </summary>
public void AddInitializer(string name, OrtValue ortValue)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddInitializer(handle, pinnedName.Pointer, ortValue.Handle));
}
}
/// <summary>
/// Bind model output to an OrtValue as Tensor with a given type and shape. An instance of OrtMemoryAllocaiton
/// owns the memory and should be alive for the time of execution.The size of the allocation can not be less than required
/// by the Tensor of the given size.
/// </summary>
/// <param name="name">of the output</param>
/// <param name="elementType">tensor element type</param>
/// <param name="shape">tensor shape</param>
/// <param name="allocation">allocated memory</param>
public void BindOutput(string name, Tensors.TensorElementType elementType, long[] shape, OrtMemoryAllocation allocation)
{
using (var ortValue = OrtValue.CreateTensorValueWithData(allocation.Info,
elementType,
shape,
allocation.Pointer, allocation.Size))
BindInputOrOutput(name, ortValue.Handle, false);
}
/// <summary>
/// Bind external allocation as input or output.
/// The allocation is owned by the user code.
/// </summary>
/// <param name="name">name </param>
/// <param name="allocation">non ort allocated memory</param>
/// <param name="isInput">whether this is an input or output</param>
private void BindExternalAllocation(string name, OrtExternalAllocation allocation, bool isInput)
{
using (var ortValue = OrtValue.CreateTensorValueWithData(allocation.Info,
allocation.ElementType,
allocation.Shape,
allocation.Pointer,
allocation.Size))
BindInputOrOutput(name, ortValue.Handle, isInput);
}
/// <summary>
/// Will extract keys and values from the map and create a DisposableNamedOnnxValue from it
/// </summary>
/// <param name="name">name of the output</param>
/// <param name="ortValueMap">ortValue that represents a map.
/// This function does not take ownership of the map as it we copy all keys an values into a dictionary. We let the caller dispose of it</param>
/// <param name="allocator"></param>
/// <returns>DisposableNamedOnnxValue</returns>
private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromNativeMap(string name, OrtValue ortValueMap, OrtAllocator allocator)
{
DisposableNamedOnnxValue result = null;
// Map processing is currently not recursing. It is assumed to contain
// only primitive types and strings tensors. No sequences or maps.
// The data is being copied to a dictionary and all ortValues are being disposed.
// not mapped for client consumption.
using (var cleanUpList = new DisposableList <IDisposable>())
{
// Take possession of the map ortValueElement
IntPtr nativeOnnxValueMapKeys = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(ortValueMap.Handle, 0, allocator.Pointer, out nativeOnnxValueMapKeys));
var ortValueKeys = new OrtValue(nativeOnnxValueMapKeys);
cleanUpList.Add(ortValueKeys);
IntPtr nativeOnnxValueMapValues = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(ortValueMap.Handle, 1, allocator.Pointer, out nativeOnnxValueMapValues));
var ortValueValues = new OrtValue(nativeOnnxValueMapValues);
cleanUpList.Add(ortValueValues);
IntPtr typeAndShape = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(nativeOnnxValueMapKeys, out typeAndShape));
TensorElementType elemType = TensorElementType.DataTypeMax;
try
{
IntPtr el_type;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShape, out el_type));
elemType = (TensorElementType)el_type;
}
finally
{
NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape);
}
/// XXX: This code always assumes that the value type is float and makes no checks
/// similar to that of the key. Also Map type in general can also be another sequence or map,
/// not just a tensor
switch (elemType)
{
case TensorElementType.Int64:
result = DisposableNamedOnnxValueFromNativeMapElements <Int64, float>(string.Empty, ortValueKeys, ortValueValues);
break;
case TensorElementType.String:
result = DisposableNamedOnnxValueFromNativeMapElements <string, float>(string.Empty, ortValueKeys, ortValueValues);
break;
default:
throw new NotSupportedException("Map of element type: " + elemType + " is not supported");
}
}
return(result);
}
protected override void Dispose(bool disposing)
{
if (IsDisposed)
{
return;
}
if (_ortValue != null)
{
_ortValue.Dispose();
_ortValue = null;
}
IsDisposed = true;
}
/// <summary>
/// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="value"></param>
/// <returns></returns>
public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value)
{
MemoryHandle?memHandle;
var ortValue = OrtValue.CreateFromTensorObject(value, out memHandle, out TensorElementType elementType);
if (memHandle.HasValue)
{
return(new FixedBufferOnnxValue((MemoryHandle)memHandle, ortValue, OnnxValueType.ONNX_TYPE_TENSOR, elementType));
}
else
{
return(new FixedBufferOnnxValue(default(MemoryHandle), ortValue, OnnxValueType.ONNX_TYPE_TENSOR, elementType));
}
}
/// <summary>
/// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="value"></param>
/// <returns>a disposable instance of FixedBufferOnnxValue</returns>
public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value)
{
MemoryHandle?memHandle;
var ortValue = OrtValue.CreateFromTensorObject(value, out memHandle, out TensorElementType elementType);
// memHandle will have a value when CreateFromTensorObject() pins managed memory and that will have to be
/// disposed (unpinned) when all is said is done. This is the case for blittable types but does not
/// happen for string type where each element has its own allocation.
if (memHandle.HasValue)
{
return(new FixedBufferOnnxValue((MemoryHandle)memHandle, ortValue, OnnxValueType.ONNX_TYPE_TENSOR, elementType));
}
else
{
return(new FixedBufferOnnxValue(default(MemoryHandle), ortValue, OnnxValueType.ONNX_TYPE_TENSOR, elementType));
}
}
protected virtual void Dispose(bool disposing)
{
if (_disposed)
{
return;
}
// dispose managed state (managed objects).
if (disposing)
{
if (_disposables != null)
{
_disposables.Dispose();
_disposables = null;
}
// _ortValueHolder can be null when no native memory is involved
if (_ortValue != null)
{
_ortValue.Dispose();
_ortValue = null;
}
_disposed = true;
}
}
private string[] _dataBufferAsString; // string tensor values copied into managed memory
/// <summary>
/// Constructs an instance and takes ownership of ortValue on success
/// </summary>
/// <param name="ortValue">ortValue that is a Tensor</param>
public NativeOnnxTensorMemory(OrtValue ortValue)
{
Type type = null;
int width = 0;
IntPtr typeAndShape = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(ortValue.Handle, out typeAndShape));
try
{
TensorElementType elemType;
{
IntPtr el_type;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShape, out el_type));
elemType = (TensorElementType)el_type;
}
if (!TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width))
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument,
"Unable to query type information for data type: " + elemType.ToString());
}
if (typeof(T) != type)
{
var message = String.Format("The NativeOnnxTensorMemory<T> type being instantiated for T = : {0} while supplied OrtValue contains T = {1}",
typeof(T), type);
throw new OnnxRuntimeException(ErrorCode.InvalidArgument, message);
}
ElementType = elemType;
ElementWidth = width;
UIntPtr dimension;
long count;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(typeAndShape, out dimension));
{
IntPtr el_count;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeElementCount(typeAndShape, out el_count)); // count can be negative.
count = (long)el_count;
}
if (count < 0)
{
throw new NotSupportedException("Symbolic dimensions in the tensor is not supported");
}
long[] shape = new long[dimension.ToUInt64()];
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(typeAndShape, shape, dimension)); //Note: shape must be alive during the call
Count = (int)count;
Dimensions = new int[dimension.ToUInt64()];
for (ulong i = 0; i < dimension.ToUInt64(); i++)
{
Dimensions[i] = (int)shape[i];
}
if (elemType != TensorElementType.String)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(ortValue.Handle, out _dataBufferPointer));
}
else
{
UIntPtr strLen;
var offsets = new UIntPtr[Count];
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(ortValue.Handle, out strLen));
var dataBuffer = new byte[strLen.ToUInt64()];
using (var dataBufferHandle = new Memory <byte>(dataBuffer).Pin())
using (var offsetMemoryHandle = new Memory <UIntPtr>(offsets).Pin())
{
unsafe
{
_dataBufferPointer = (IntPtr)dataBufferHandle.Pointer;
NativeApiStatus.VerifySuccess(
NativeMethods.OrtGetStringTensorContent(
ortValue.Handle, _dataBufferPointer, strLen,
(IntPtr)offsetMemoryHandle.Pointer,
(UIntPtr)Count));
}
_dataBufferAsString = new string[Count];
for (var i = 0; i < offsets.Length; i++)
{
var length = (i == offsets.Length - 1)
? strLen.ToUInt64() - offsets[i].ToUInt64()
: offsets[i + 1].ToUInt64() - offsets[i].ToUInt64();
// Onnx specifies strings always in UTF-8, no trailing null, no leading BOM
_dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length);
}
}
}
// Transfer ownership
_ortValue = new OrtValue(ortValue.Disown());
}
finally
{
NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape);
}
}
/// <summary>
/// Add a pre-allocated initializer to a session. If a model contains an initializer with a name
/// that is same as the name passed to this API call, ORT will use this initializer instance
/// instead of deserializing one from the model file. This is useful when you want to share
/// the same initializer across sessions.
/// \param name name of the initializer
/// \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
/// managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
/// to which it is added.
/// </summary>
public void AddInitializer(string name, OrtValue ort_value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddInitializer(handle, name, ort_value.Handle));
}
/// <summary>
/// This is a factory method creates a native Onnxruntime OrtValue containing a tensor.
/// The method will attempt to pin managed memory so no copying occurs when data is passed down
/// to native code.
/// </summary>
/// <param name="value">Tensor object</param>
/// <param name="memoryHandle">For all tensor types but string tensors we endeavor to use managed memory
/// to avoid additional allocation and copy. This out parameter represents a chunk of pinned memory which will need
/// to be disposed when no longer needed. The lifespan of memoryHandle should eclipse the lifespan of the corresponding
/// OrtValue.
/// </param>
/// <param name="elementType">discovered tensor element type</param>
/// <returns>And instance of OrtValue constructed on top of the object</returns>
public static OrtValue CreateFromTensorObject(Object value, out MemoryHandle?memoryHandle,
out TensorElementType elementType)
{
// Check if this is a Tensor
if (!(value is TensorBase))
{
throw new NotSupportedException("The inference value " + nameof(value) + " is not of a supported type");
}
var tensorBase = value as TensorBase;
var typeInfo = tensorBase.GetTypeInfo();
if (typeInfo == null)
{
throw new OnnxRuntimeException(ErrorCode.RequirementNotRegistered, "BUG Check");
}
MemoryHandle?memHandle;
OrtValue ortValue = null;
int dataBufferLength = 0;
long[] shape = null;
int rank = 0;
TensorElementType elType = typeInfo.ElementType;
var typeSize = typeInfo.TypeSize;
if (typeInfo.IsString)
{
ortValue = CreateStringTensor(value as Tensor <string>);
memHandle = null;
}
else
{
switch (elType)
{
case TensorElementType.Float:
PinAsTensor(value as Tensor <float>, typeSize, out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.Double:
PinAsTensor(value as Tensor <double>, typeSize, out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.Int32:
PinAsTensor(value as Tensor <int>, typeSize, out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.UInt32:
PinAsTensor(value as Tensor <uint>, typeSize, out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.Int64:
PinAsTensor(value as Tensor <long>, typeSize, out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.UInt64:
PinAsTensor(value as Tensor <ulong>, typeSize, out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.Int16:
PinAsTensor(value as Tensor <short>, typeSize, out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.UInt16:
PinAsTensor(value as Tensor <ushort>, typeSize,
out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.UInt8:
PinAsTensor(value as Tensor <byte>, typeSize,
out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.Int8:
PinAsTensor(value as Tensor <sbyte>, typeSize,
out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.Bool:
PinAsTensor(value as Tensor <bool>, typeSize,
out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.Float16:
PinAsTensor(value as Tensor <Float16>, typeSize,
out memHandle, out dataBufferLength,
out shape, out rank);
break;
case TensorElementType.BFloat16:
PinAsTensor(value as Tensor <BFloat16>, typeSize,
out memHandle, out dataBufferLength,
out shape, out rank);
break;
default:
throw new NotSupportedException("Element type: " + elType + " is not of a supported type");
}
try
{
Debug.Assert(memHandle.HasValue);
IntPtr dataBufferPointer = IntPtr.Zero;
unsafe
{
dataBufferPointer = (IntPtr)((MemoryHandle)memHandle).Pointer;
}
IntPtr nativeValue;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue(
OrtMemoryInfo.DefaultInstance.Pointer,
dataBufferPointer,
(UIntPtr)(dataBufferLength),
shape,
(UIntPtr)rank,
elType,
out nativeValue));
ortValue = new OrtValue(nativeValue);
}
catch (Exception)
{
memHandle?.Dispose();
throw;
}
}
memoryHandle = memHandle;
elementType = elType;
return(ortValue);
}
/// <summary>
/// This method maps keys and values of the map and copies them into a Dictionary
/// and returns as an instance of DisposableNamedOnnxValue that does not own or dispose
/// any onnx/ortValueElement. The method takes possession of ortValueTensorKeys and ortValueTensorValues
/// and disposes of them. The original ortValueElement looses ownership of the Tensor. The caller is still responsible
/// for disposing these arguments. Disposing ortValueElement that does not have ownership is a no-op, however, either
/// of the arguments may still need to be disposed on exception.
/// </summary>
/// <typeparam name="K">key type</typeparam>
/// <typeparam name="V">value type</typeparam>
/// <param name="name">name of the output parameter</param>
/// <param name="ortValueTensorKeys">tensor with map keys.</param>
/// <param name="nativeOnnxValueValues">tensor with map values</param>
/// <returns>instance of DisposableNamedOnnxValue with Dictionary</returns>
private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromNativeMapElements <K, V>(string name,
OrtValue ortValueTensorKeys, OrtValue ortValueTensorValues)
{
using (var nativeTensorWrapperValues = new NativeOnnxTensorMemory <V>(ortValueTensorValues))
{
var denseTensorValues = new DenseTensor <V>(nativeTensorWrapperValues.Memory, nativeTensorWrapperValues.Dimensions);
if (typeof(K) == typeof(string))
{
var map = new Dictionary <string, V>();
using (var nativeTensorWrapper = new NativeOnnxTensorMemory <string>(ortValueTensorKeys))
{
var denseTensorKeys = new DenseTensor <string>(nativeTensorWrapper.GetBytesAsStringMemory(), nativeTensorWrapper.Dimensions);
for (var i = 0; i < denseTensorKeys.Length; i++)
{
map.Add(denseTensorKeys.GetValue(i), denseTensorValues.GetValue(i));
}
return(new DisposableNamedOnnxValue(name, map, OnnxValueType.ONNX_TYPE_MAP, TensorElementType.DataTypeMax, null));
}
}
else
{
var map = new Dictionary <K, V>();
using (var nativeTensorWrapper = new NativeOnnxTensorMemory <K>(ortValueTensorKeys))
{
var denseTensorKeys = new DenseTensor <K>(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions);
for (var i = 0; i < denseTensorKeys.Length; i++)
{
map.Add(denseTensorKeys.GetValue(i), denseTensorValues.GetValue(i));
}
return(new DisposableNamedOnnxValue(name, map, OnnxValueType.ONNX_TYPE_MAP, TensorElementType.DataTypeMax, null));
}
}
}
}
/// <summary>
/// Pin the underlying memory and create an instance of OrtValue
/// based on the pinned managed memory. The caller is responsible for Disposing
/// both OrtValue and pinnedMemoryHandle
/// </summary>
/// <param name="pinnedMemoryHandle">dispose after returned OrtValus is disposed</param>
/// <returns>an instance of OrtValue. The lifespan of OrtValue must overlap pinnedMemoryHandle</returns>
internal virtual OrtValue ToOrtValue(out MemoryHandle?pinnedMemoryHandle)
{
return(OrtValue.CreateFromTensorObject(_value, out pinnedMemoryHandle, out TensorElementType elementType));
}
private string[] _dataBufferAsString; // string tensor values copied into managed memory
/// <summary>
/// Constructs an instance and takes ownership of ortValue on success
/// </summary>
/// <param name="ortValue">ortValue that is a Tensor</param>
public NativeOnnxTensorMemory(OrtValue ortValue)
{
Type type = null;
int width = 0;
IntPtr typeAndShape = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(ortValue.Handle, out typeAndShape));
try
{
TensorElementType elemType;
{
IntPtr el_type;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShape, out el_type));
elemType = (TensorElementType)el_type;
}
TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width);
if (typeof(T) != type)
{
throw new NotSupportedException(nameof(NativeOnnxTensorMemory <T>) + " does not support T = " + nameof(T));
}
ElementType = elemType;
ElementWidth = width;
UIntPtr dimension;
long count;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(typeAndShape, out dimension));
{
IntPtr el_count;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeElementCount(typeAndShape, out el_count)); // count can be negative.
count = (long)el_count;
}
if (count < 0)
{
throw new NotSupportedException("Symbolic dimensions in the tensor is not supported");
}
long[] shape = new long[dimension.ToUInt64()];
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(typeAndShape, shape, dimension)); //Note: shape must be alive during the call
Count = (int)count;
Dimensions = new int[dimension.ToUInt64()];
for (ulong i = 0; i < dimension.ToUInt64(); i++)
{
Dimensions[i] = (int)shape[i];
}
if (typeof(T) != typeof(string))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(ortValue.Handle, out _dataBufferPointer));
}
else
{
UIntPtr strLen;
var offsets = new UIntPtr[Count];
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(ortValue.Handle, out strLen));
var dataBuffer = new byte[strLen.ToUInt64()];
using (var dataBufferHandle = new Memory <byte>(dataBuffer).Pin())
using (var offsetMemoryHandle = new Memory <UIntPtr>(offsets).Pin())
{
unsafe
{
_dataBufferPointer = (IntPtr)dataBufferHandle.Pointer;
NativeApiStatus.VerifySuccess(
NativeMethods.OrtGetStringTensorContent(
ortValue.Handle, _dataBufferPointer, strLen,
(IntPtr)offsetMemoryHandle.Pointer,
(UIntPtr)Count));
}
_dataBufferAsString = new string[Count];
for (var i = 0; i < offsets.Length; i++)
{
var length = (i == offsets.Length - 1)
? strLen.ToUInt64() - offsets[i].ToUInt64()
: offsets[i + 1].ToUInt64() - offsets[i].ToUInt64();
// Onnx specifies strings always in UTF-8, no trailing null, no leading BOM
_dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length);
}
}
}
// Transfer ownership, but only do so if the original OrtValue is already owned.
if (ortValue.IsOwned)
{
_ortValue = new OrtValue(ortValue.Disown());
}
else
{
_ortValue = new OrtValue(ortValue.Handle, false);
}
}
finally
{
NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape);
}
}
/// <summary>
/// This method creates an instance of DisposableNamedOnnxValue that has possession of ortValueElement
/// native memory Tensor and returns it to the caller. The original ortValueElement argument looses
/// ownership of the native ortValueElement handle, however, the caller is still responsible for disposing them
/// on exception. Disposing of OrtValue that has no ownership is a no-op and fine.
/// </summary>
/// <typeparam name="T">data type</typeparam>
/// <param name="name">name of the output</param>
/// <param name="ortValue">native tensor</param>
/// <returns>DisposableNamedOnnxValue instance</returns>
private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromNativeTensor <T>(string name, OrtValue ortValue)
{
if (typeof(T) == typeof(string))
{
var nativeTensorWrapper = new NativeOnnxTensorMemory <string>(ortValue);
try
{
var dt = new DenseTensor <string>(nativeTensorWrapper.GetBytesAsStringMemory(), nativeTensorWrapper.Dimensions);
return(new DisposableNamedOnnxValue(name, dt, OnnxValueType.ONNX_TYPE_TENSOR, nativeTensorWrapper.ElementType, nativeTensorWrapper));
} catch (Exception)
{
nativeTensorWrapper.Dispose();
throw;
}
}
else
{
NativeOnnxTensorMemory <T> nativeTensorWrapper = new NativeOnnxTensorMemory <T>(ortValue);
try
{
DenseTensor <T> dt = new DenseTensor <T>(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions);
return(new DisposableNamedOnnxValue(name, dt, OnnxValueType.ONNX_TYPE_TENSOR, nativeTensorWrapper.ElementType, nativeTensorWrapper));
}
catch (Exception)
{
nativeTensorWrapper.Dispose();
throw;
}
}
}
internal NativeOrtValueCollectionOwner(OrtValue ortValue, DisposableList <DisposableNamedOnnxValue> disposables)
{
Debug.Assert(ortValue.IsOwned);
_ortValue = new OrtValue(ortValue.Disown());
_disposables = disposables;
}
internal static DisposableNamedOnnxValue CreateFromOrtValue(string name, OrtValue ortValue)
{
return(CreateFromOrtValue(name, ortValue, OrtAllocator.DefaultInstance));
}
/// <summary>
/// Creates an instance of DisposableNamedOnnxValue and takes ownership of ortValueElement
/// on success.
/// </summary>
/// <param name="name">name of the value</param>
/// <param name="ortValue">underlying OrtValue</param>
/// <returns></returns>
internal static DisposableNamedOnnxValue CreateTensorFromOnnxValue(string name, OrtValue ortValue)
{
DisposableNamedOnnxValue result = null;
/* Get Tensor element type */ //TODO: Assumed value is Tensor, need to support non-tensor types in future
IntPtr typeAndShape = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(ortValue.Handle, out typeAndShape));
TensorElementType elemType = TensorElementType.DataTypeMax;
try
{
IntPtr el_type;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShape, out el_type));
elemType = (TensorElementType)el_type;
}
finally
{
NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape);
}
switch (elemType)
{
case TensorElementType.Float:
result = DisposableNamedOnnxValueFromNativeTensor <float>(name, ortValue);
break;
case TensorElementType.Double:
result = DisposableNamedOnnxValueFromNativeTensor <double>(name, ortValue);
break;
case TensorElementType.Int16:
result = DisposableNamedOnnxValueFromNativeTensor <short>(name, ortValue);
break;
case TensorElementType.UInt16:
result = DisposableNamedOnnxValueFromNativeTensor <ushort>(name, ortValue);
break;
case TensorElementType.Int32:
result = DisposableNamedOnnxValueFromNativeTensor <int>(name, ortValue);
break;
case TensorElementType.UInt32:
result = DisposableNamedOnnxValueFromNativeTensor <uint>(name, ortValue);
break;
case TensorElementType.Int64:
result = DisposableNamedOnnxValueFromNativeTensor <long>(name, ortValue);
break;
case TensorElementType.UInt64:
result = DisposableNamedOnnxValueFromNativeTensor <ulong>(name, ortValue);
break;
case TensorElementType.UInt8:
result = DisposableNamedOnnxValueFromNativeTensor <byte>(name, ortValue);
break;
case TensorElementType.Int8:
result = DisposableNamedOnnxValueFromNativeTensor <sbyte>(name, ortValue);
break;
case TensorElementType.String:
result = DisposableNamedOnnxValueFromNativeTensor <string>(name, ortValue);
break;
case TensorElementType.Bool:
result = DisposableNamedOnnxValueFromNativeTensor <bool>(name, ortValue);
break;
case TensorElementType.Float16:
result = DisposableNamedOnnxValueFromNativeTensor <Float16>(name, ortValue);
break;
case TensorElementType.BFloat16:
result = DisposableNamedOnnxValueFromNativeTensor <BFloat16>(name, ortValue);
break;
default:
throw new NotSupportedException("Tensor of element type: " + elemType + " is not supported");
}
return(result);
}
