/// <summary>
/// Append SNPE or XNNPACK execution provider
/// </summary>
/// <param name="providerName">Execution provider to add. 'SNPE' or 'XNNPACK' are currently supported.</param>
/// <param name="providerOptions">Optional key/value pairs to specify execution provider options.</param>
public void AppendExecutionProvider(string providerName, Dictionary <string, string> providerOptions = null)
{
if (providerName != "SNPE" && providerName != "XNNPACK")
{
throw new NotSupportedException(
"Only SNPE and XNNPACK execution providers can be enabled by this method.");
}
using (var cleanupList = new DisposableList <IDisposable>())
{
string[] ep = { providerName }; // put in array so we can use ConvertNamesToUtf8 for everything
var epArray = NativeOnnxValueHelper.ConvertNamesToUtf8(ep, n => n, cleanupList);
if (providerOptions == null)
{
providerOptions = new Dictionary <string, string>();
}
var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8(
providerOptions.Keys.ToArray(), n => n, cleanupList);
var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(
providerOptions.Values.ToArray(), n => n, cleanupList);
NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider(
handle, epArray[0], keysArray, valuesArray, (UIntPtr)providerOptions.Count));
}
}
/// <summary>
/// This function will bind model output with the given name to a device
/// specified by the memInfo.
/// </summary>
/// <param name="name">output name</param>
/// <param name="memInfo">instance of memory info</param>
public void BindOutputToDevice(string name, OrtMemoryInfo memInfo)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutputToDevice(handle, pinnedName.Pointer, memInfo.Pointer));
}
/// <summary>
/// Queries all the execution providers supported in the native onnxruntime shared library
/// </summary>
/// <returns>an array of strings that represent execution provider names</returns>
public string[] GetAvailableProviders()
{
IntPtr availableProvidersHandle = IntPtr.Zero;
int numProviders;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetAvailableProviders(out availableProvidersHandle, out numProviders));
var availableProviders = new string[numProviders];
try
{
for (int i = 0; i < numProviders; ++i)
{
availableProviders[i] = NativeOnnxValueHelper.StringFromNativeUtf8(Marshal.ReadIntPtr(availableProvidersHandle, IntPtr.Size * i));
}
}
finally
{
// Looks a bit weird that we might throw in finally(...)
// But the native method OrtReleaseAvailableProviders actually doesn't return a failure status
// If it does, it is BUG and we would like to propagate that to the user in the form of an exception
NativeApiStatus.VerifySuccess(NativeMethods.OrtReleaseAvailableProviders(availableProvidersHandle, numProviders));
}
return(availableProviders);
}
private static string GetErrorMessage(IntPtr /*(ONNXStatus*)*/ status)
{
// nativeString belongs to status, no need for separate release
IntPtr nativeString = NativeMethods.OrtGetErrorMessage(status);
return(NativeOnnxValueHelper.StringFromNativeUtf8(nativeString));
}
private static OrtValue CreateStringTensor(Tensor <string> tensor)
{
if (tensor == null)
{
throw new OnnxRuntimeException(ErrorCode.Fail, "Cast to Tensor<string> failed. BUG check!");
}
int totalLength = 0;
for (int i = 0; i < tensor.Length; i++)
{
totalLength += System.Text.Encoding.UTF8.GetByteCount(tensor.GetValue(i));
}
long[] shape = new long[tensor.Dimensions.Length];
for (int i = 0; i < tensor.Dimensions.Length; i++)
{
shape[i] = tensor.Dimensions[i];
}
// allocate the native tensor
IntPtr valueHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue(
OrtAllocator.DefaultInstance.Pointer,
shape,
(UIntPtr)(shape.Length),
TensorElementType.String,
out valueHandle
));
var ortValue = new OrtValue(valueHandle);
try
{
// fill the native tensor, using GetValue(index) from the Tensor<string>
var len = tensor.Length;
var nativeStrings = new IntPtr[len];
using (var pinnedHandles = new DisposableList <PinnedGCHandle>((int)len))
{
for (int i = 0; i < len; i++)
{
var utf8str = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(tensor.GetValue(i));
var gcHandle = GCHandle.Alloc(utf8str, GCHandleType.Pinned);
nativeStrings[i] = gcHandle.AddrOfPinnedObject();
pinnedHandles.Add(new PinnedGCHandle(gcHandle));
}
using (var pinnedStrings = new PinnedGCHandle(GCHandle.Alloc(nativeStrings, GCHandleType.Pinned)))
NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(ortValue.Handle, nativeStrings, (UIntPtr)len));
}
}
catch (OnnxRuntimeException)
{
ortValue.Dispose();
throw;
}
return(ortValue);
}
internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo)
{
OnnxValueType valueType;
{
IntPtr valType;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetOnnxTypeFromTypeInfo(typeInfo, out valType));
valueType = (OnnxValueType)valType;
}
if (valueType != OnnxValueType.ONNX_TYPE_TENSOR && valueType != OnnxValueType.ONNX_TYPE_SPARSETENSOR)
{
return(new NodeMetadata(valueType, new int[] { }, new string[] { }, typeof(NamedOnnxValue)));
}
// This should not be released
IntPtr tensorInfo;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo, out tensorInfo)); //(IntPtr)(int)(uint)
// Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo*
if (tensorInfo == IntPtr.Zero)
{
return(null);
}
TensorElementType type;
{
IntPtr el_type;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(tensorInfo, out el_type));
type = (TensorElementType)el_type;
}
Type dotnetType = null;
int width = 0;
TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width);
UIntPtr numDimensions;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(tensorInfo, out numDimensions));
long[] dimensions = new long[(int)numDimensions];
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(tensorInfo, dimensions, numDimensions));
int[] intDimensions = new int[(int)numDimensions];
for (var i = 0; i < (long)numDimensions; i++)
{
intDimensions[i] = (int)dimensions[i];
}
IntPtr[] dimensionNamePtrs = new IntPtr[(int)numDimensions];
NativeApiStatus.VerifySuccess(
NativeMethods.OrtGetSymbolicDimensions(tensorInfo, dimensionNamePtrs, numDimensions));
string[] symbolicDimensions = new string[(int)numDimensions];
for (var i = 0; i < (int)numDimensions; i++)
{
symbolicDimensions[i] = NativeOnnxValueHelper.StringFromNativeUtf8(dimensionNamePtrs[i]);
}
return(new NodeMetadata(valueType, intDimensions, symbolicDimensions, dotnetType));
}
/// <summary>
/// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="value"></param>
/// <returns></returns>
public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value)
{
NativeOnnxValueHelper.CreateNativeOnnxValue(value, out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType);
Debug.Assert(onnxValueType == OnnxValueType.ONNX_TYPE_TENSOR, "the value should always be a tensor");
return(new FixedBufferOnnxValue(pinnedMemoryHandle, onnxValue, onnxValueType, elementType));
}
/// <summary>
/// Override symbolic dimensions (by specific name strings) with actual values if known at session initialization time to enable
/// optimizations that can take advantage of fixed values (such as memory planning, etc)
/// </summary>
public void AddFreeDimensionOverrideByName(string dimName, long dimValue)
{
var utf8DimNamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(dimName), GCHandleType.Pinned);
using (var pinnedDimName = new PinnedGCHandle(utf8DimNamePinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddFreeDimensionOverrideByName(handle, pinnedDimName.Pointer, dimValue));
}
}
/// <summary>
/// Pin the underlying memory and create native onnx value
/// </summary>
/// <param name="onnxValue"></param>
/// <param name="pinnedMemoryHandle"></param>
/// <param name="disposeOnnxValueAfterUse"></param>
internal virtual void ToNativeOnnxValue(
out IntPtr onnxValue,
out MemoryHandle pinnedMemoryHandle,
out bool disposeOnnxValueAfterUse)
{
NativeOnnxValueHelper.CreateNativeOnnxValue(_value, out onnxValue, out pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType);
disposeOnnxValueAfterUse = true;
}
/// <summary>
/// Loads a DLL named 'libraryPath' and looks for this entry point:
/// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api);
/// It then passes in the provided session options to this function along with the api base.
/// The handle to the loaded library is returned in 'libraryHandle'.
/// It can be unloaded by the caller after all sessions using the passed in
/// session options are destroyed, or if an error occurs and it is non null.
/// Hint: .NET Core 3.1 has a 'NativeLibrary' class that can be used to free the library handle
/// </summary>
public void RegisterCustomOpLibraryV2(string libraryPath, out IntPtr libraryHandle)
{
var libraryPathPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath), GCHandleType.Pinned);
using (var pinnedlibraryPath = new PinnedGCHandle(libraryPathPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, pinnedlibraryPath.Pointer, out libraryHandle));
}
}
/// <summary>
/// Add a pre-allocated initializer to a session. If a model contains an initializer with a name
/// that is same as the name passed to this API call, ORT will use this initializer instance
/// instead of deserializing one from the model file. This is useful when you want to share
/// the same initializer across sessions.
/// \param name name of the initializer
/// \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
/// managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
/// to which it is added.
/// </summary>
public void AddInitializer(string name, OrtValue ortValue)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddInitializer(handle, pinnedName.Pointer, ortValue.Handle));
}
}
/// <summary>
/// Set a single session configuration entry as a pair of strings
/// If a configuration with same key exists, this will overwrite the configuration with the given configValue
/// </summary>
/// <param name="configKey">config key name</param>
/// <param name="configValue">config key value</param>
public void AddSessionConfigEntry(string configKey, string configValue)
{
using (var pinnedConfigKeyName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configKey), GCHandleType.Pinned)))
using (var pinnedConfigValueName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configValue), GCHandleType.Pinned)))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddSessionConfigEntry(handle,
pinnedConfigKeyName.Pointer, pinnedConfigValueName.Pointer));
}
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="settings">string with Nuphar specific settings</param>
public void AppendExecutionProvider_Nuphar(string settings = "")
{
var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(handle, 1, pinnedSettingsName.Pointer));
}
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="deviceId">device identification, default empty string</param>
public void AppendExecutionProvider_OpenVINO(string deviceId = "")
{
var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);
using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
}
}
/// <summary>
/// Updates the configuration knobs of OrtCUDAProviderOptions that will eventually be used to configure a CUDA EP
/// Please refer to the following on different key/value pairs to configure a CUDA EP and their meaning:
/// https://www.onnxruntime.ai/docs/reference/execution-providers/CUDA-ExecutionProvider.html
/// </summary>
/// <param name="providerOptions">key/value pairs used to configure a CUDA Execution Provider</param>
public void UpdateOptions(Dictionary <string, string> providerOptions)
{
using (var cleanupList = new DisposableList <IDisposable>())
{
var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Keys.ToArray(), n => n, cleanupList);
var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Values.ToArray(), n => n, cleanupList);
NativeApiStatus.VerifySuccess(NativeMethods.OrtUpdateCUDAProviderOptions(handle, keysArray, valuesArray, (UIntPtr)providerOptions.Count));
}
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="settings">string with TVM specific settings</param>
public void AppendExecutionProvider_Tvm(string settings = "")
{
#if __MOBILE__
throw new NotSupportedException("The TVM Execution Provider is not supported in this build");
#else
var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tvm(handle, pinnedSettingsName.Pointer));
}
#endif
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="deviceId">device identification, default empty string</param>
public void AppendExecutionProvider_OpenVINO(string deviceId = "")
{
#if __MOBILE__
throw new NotSupportedException("The OpenVINO Execution Provider is not supported in this build");
#else
var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);
using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
}
#endif
}
/// <summary>
/// Get TensorRT EP provider options
/// </summary>
/// <returns> return C# UTF-16 encoded string </returns>
public string GetOptions()
{
var allocator = OrtAllocator.DefaultInstance;
// Process provider options string
IntPtr providerOptions = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorRTProviderOptionsAsString(handle, allocator.Pointer, out providerOptions));
using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0))
{
return(NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions));
}
}
/// <summary>
/// A helper method to construct a SessionOptions object for Nuphar execution.
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param>
/// <returns>A SessionsOptions() object configured for execution with Nuphar</returns>
public static SessionOptions MakeSessionOptionWithNupharProvider(String settings = "")
{
SessionOptions options = new SessionOptions();
var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(options.Handle, 1, pinnedSettingsName.Pointer));
}
return(options);
}
/// <summary>
/// Ends profiling for the session. Returns the profile file name.
///
public string EndProfiling()
{
IntPtr nameHandle = IntPtr.Zero;
var allocator = OrtAllocator.DefaultInstance;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionEndProfiling(_nativeHandle,
allocator.Pointer,
out nameHandle));
using (var allocation = new OrtMemoryAllocation(allocator, nameHandle, 0))
{
return(NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle));
}
}
/// <summary>
/// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="value"></param>
/// <returns></returns>
public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value)
{
if (value is Tensor <string> )
{
throw new ArgumentException("Only numeric tensors can be used to create FixedBufferOnnxValue.", nameof(value));
}
NativeOnnxValueHelper.CreateNativeOnnxValue(value, out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType);
Debug.Assert(
onnxValueType == OnnxValueType.ONNX_TYPE_TENSOR && elementType != TensorElementType.String,
"the value should always be a numeric tensor");
return(new FixedBufferOnnxValue(pinnedMemoryHandle, onnxValue, onnxValueType, elementType));
}
/// <summary>
/// Run helper
/// </summary>
/// <param name="names">names to convert to zero terminated utf8 and pin</param>
/// <param name="cleanupList">list to add pinned memory to for later disposal</param>
/// <returns></returns>
private IntPtr[] ConvertNamesToUtf8 <T>(IReadOnlyCollection <T> inputs, NameExtractor <T> extractor,
DisposableList <IDisposable> cleanupList)
{
var result = new IntPtr[inputs.Count];
for (int i = 0; i < inputs.Count; ++i)
{
var name = extractor(inputs.ElementAt(i));
var utf8Name = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name);
var pinnedHandle = new PinnedGCHandle(GCHandle.Alloc(utf8Name, GCHandleType.Pinned));
result[i] = pinnedHandle.Pointer;
cleanupList.Add(pinnedHandle);
}
return(result);
}
/// <summary>
/// Internal helper
/// </summary>
/// <param name="name"></param>
/// <param name="ortValue"></param>
/// <param name="isInput"></param>
private void BindInputOrOutput(string name, IntPtr ortValue, bool isInput)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
{
if (isInput)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtBindInput(handle, pinnedName.Pointer, ortValue));
}
else
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutput(handle, pinnedName.Pointer, ortValue));
}
}
}
private string GetOverridableInitializerName(ulong index)
{
string str = null;
var allocator = OrtAllocator.DefaultInstance;
IntPtr nameHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOverridableInitializerName(
_nativeHandle,
(UIntPtr)index,
allocator.Pointer,
out nameHandle));
using (var ortAllocation = new OrtMemoryAllocation(allocator, nameHandle, 0))
{
str = NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle);
}
return(str);
}
/// <summary>
/// Create an instance of OrtMemoryInfo according to the specification.
/// </summary>
/// <param name="allocatorName">Allocator name</param>
/// <param name="allocatorType">Allocator type</param>
/// <param name="deviceId">Device id</param>
/// <param name="memoryType">Memory type</param>
public OrtMemoryInfo(string allocatorName, OrtAllocatorType allocatorType, int deviceId, OrtMemType memoryType)
: this(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(allocatorName), allocatorType, deviceId, memoryType)
{
}