/// <summary> /// Append SNPE or XNNPACK execution provider /// </summary> /// <param name="providerName">Execution provider to add. 'SNPE' or 'XNNPACK' are currently supported.</param> /// <param name="providerOptions">Optional key/value pairs to specify execution provider options.</param> public void AppendExecutionProvider(string providerName, Dictionary <string, string> providerOptions = null) { if (providerName != "SNPE" && providerName != "XNNPACK") { throw new NotSupportedException( "Only SNPE and XNNPACK execution providers can be enabled by this method."); } using (var cleanupList = new DisposableList <IDisposable>()) { string[] ep = { providerName }; // put in array so we can use ConvertNamesToUtf8 for everything var epArray = NativeOnnxValueHelper.ConvertNamesToUtf8(ep, n => n, cleanupList); if (providerOptions == null) { providerOptions = new Dictionary <string, string>(); } var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8( providerOptions.Keys.ToArray(), n => n, cleanupList); var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8( providerOptions.Values.ToArray(), n => n, cleanupList); NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider( handle, epArray[0], keysArray, valuesArray, (UIntPtr)providerOptions.Count)); } }
/// <summary> /// This function will bind model output with the given name to a device /// specified by the memInfo. /// </summary> /// <param name="name">output name</param> /// <param name="memInfo">instance of memory info</param> public void BindOutputToDevice(string name, OrtMemoryInfo memInfo) { var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned); using (var pinnedName = new PinnedGCHandle(utf8NamePinned)) NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutputToDevice(handle, pinnedName.Pointer, memInfo.Pointer)); }
/// <summary> /// Queries all the execution providers supported in the native onnxruntime shared library /// </summary> /// <returns>an array of strings that represent execution provider names</returns> public string[] GetAvailableProviders() { IntPtr availableProvidersHandle = IntPtr.Zero; int numProviders; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetAvailableProviders(out availableProvidersHandle, out numProviders)); var availableProviders = new string[numProviders]; try { for (int i = 0; i < numProviders; ++i) { availableProviders[i] = NativeOnnxValueHelper.StringFromNativeUtf8(Marshal.ReadIntPtr(availableProvidersHandle, IntPtr.Size * i)); } } finally { // Looks a bit weird that we might throw in finally(...) // But the native method OrtReleaseAvailableProviders actually doesn't return a failure status // If it does, it is BUG and we would like to propagate that to the user in the form of an exception NativeApiStatus.VerifySuccess(NativeMethods.OrtReleaseAvailableProviders(availableProvidersHandle, numProviders)); } return(availableProviders); }
private static string GetErrorMessage(IntPtr /*(ONNXStatus*)*/ status) { // nativeString belongs to status, no need for separate release IntPtr nativeString = NativeMethods.OrtGetErrorMessage(status); return(NativeOnnxValueHelper.StringFromNativeUtf8(nativeString)); }
private static OrtValue CreateStringTensor(Tensor <string> tensor) { if (tensor == null) { throw new OnnxRuntimeException(ErrorCode.Fail, "Cast to Tensor<string> failed. BUG check!"); } int totalLength = 0; for (int i = 0; i < tensor.Length; i++) { totalLength += System.Text.Encoding.UTF8.GetByteCount(tensor.GetValue(i)); } long[] shape = new long[tensor.Dimensions.Length]; for (int i = 0; i < tensor.Dimensions.Length; i++) { shape[i] = tensor.Dimensions[i]; } // allocate the native tensor IntPtr valueHandle = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue( OrtAllocator.DefaultInstance.Pointer, shape, (UIntPtr)(shape.Length), TensorElementType.String, out valueHandle )); var ortValue = new OrtValue(valueHandle); try { // fill the native tensor, using GetValue(index) from the Tensor<string> var len = tensor.Length; var nativeStrings = new IntPtr[len]; using (var pinnedHandles = new DisposableList <PinnedGCHandle>((int)len)) { for (int i = 0; i < len; i++) { var utf8str = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(tensor.GetValue(i)); var gcHandle = GCHandle.Alloc(utf8str, GCHandleType.Pinned); nativeStrings[i] = gcHandle.AddrOfPinnedObject(); pinnedHandles.Add(new PinnedGCHandle(gcHandle)); } using (var pinnedStrings = new PinnedGCHandle(GCHandle.Alloc(nativeStrings, GCHandleType.Pinned))) NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(ortValue.Handle, nativeStrings, (UIntPtr)len)); } } catch (OnnxRuntimeException) { ortValue.Dispose(); throw; } return(ortValue); }
internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) { OnnxValueType valueType; { IntPtr valType; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetOnnxTypeFromTypeInfo(typeInfo, out valType)); valueType = (OnnxValueType)valType; } if (valueType != OnnxValueType.ONNX_TYPE_TENSOR && valueType != OnnxValueType.ONNX_TYPE_SPARSETENSOR) { return(new NodeMetadata(valueType, new int[] { }, new string[] { }, typeof(NamedOnnxValue))); } // This should not be released IntPtr tensorInfo; NativeApiStatus.VerifySuccess(NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo, out tensorInfo)); //(IntPtr)(int)(uint) // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo* if (tensorInfo == IntPtr.Zero) { return(null); } TensorElementType type; { IntPtr el_type; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(tensorInfo, out el_type)); type = (TensorElementType)el_type; } Type dotnetType = null; int width = 0; TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width); UIntPtr numDimensions; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(tensorInfo, out numDimensions)); long[] dimensions = new long[(int)numDimensions]; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(tensorInfo, dimensions, numDimensions)); int[] intDimensions = new int[(int)numDimensions]; for (var i = 0; i < (long)numDimensions; i++) { intDimensions[i] = (int)dimensions[i]; } IntPtr[] dimensionNamePtrs = new IntPtr[(int)numDimensions]; NativeApiStatus.VerifySuccess( NativeMethods.OrtGetSymbolicDimensions(tensorInfo, dimensionNamePtrs, numDimensions)); string[] symbolicDimensions = new string[(int)numDimensions]; for (var i = 0; i < (int)numDimensions; i++) { symbolicDimensions[i] = NativeOnnxValueHelper.StringFromNativeUtf8(dimensionNamePtrs[i]); } return(new NodeMetadata(valueType, intDimensions, symbolicDimensions, dotnetType)); }
/// <summary> /// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="value"></param> /// <returns></returns> public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value) { NativeOnnxValueHelper.CreateNativeOnnxValue(value, out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType); Debug.Assert(onnxValueType == OnnxValueType.ONNX_TYPE_TENSOR, "the value should always be a tensor"); return(new FixedBufferOnnxValue(pinnedMemoryHandle, onnxValue, onnxValueType, elementType)); }
/// <summary> /// Override symbolic dimensions (by specific name strings) with actual values if known at session initialization time to enable /// optimizations that can take advantage of fixed values (such as memory planning, etc) /// </summary> public void AddFreeDimensionOverrideByName(string dimName, long dimValue) { var utf8DimNamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(dimName), GCHandleType.Pinned); using (var pinnedDimName = new PinnedGCHandle(utf8DimNamePinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtAddFreeDimensionOverrideByName(handle, pinnedDimName.Pointer, dimValue)); } }
/// <summary> /// Pin the underlying memory and create native onnx value /// </summary> /// <param name="onnxValue"></param> /// <param name="pinnedMemoryHandle"></param> /// <param name="disposeOnnxValueAfterUse"></param> internal virtual void ToNativeOnnxValue( out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle, out bool disposeOnnxValueAfterUse) { NativeOnnxValueHelper.CreateNativeOnnxValue(_value, out onnxValue, out pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType); disposeOnnxValueAfterUse = true; }
/// <summary> /// Loads a DLL named 'libraryPath' and looks for this entry point: /// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api); /// It then passes in the provided session options to this function along with the api base. /// The handle to the loaded library is returned in 'libraryHandle'. /// It can be unloaded by the caller after all sessions using the passed in /// session options are destroyed, or if an error occurs and it is non null. /// Hint: .NET Core 3.1 has a 'NativeLibrary' class that can be used to free the library handle /// </summary> public void RegisterCustomOpLibraryV2(string libraryPath, out IntPtr libraryHandle) { var libraryPathPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath), GCHandleType.Pinned); using (var pinnedlibraryPath = new PinnedGCHandle(libraryPathPinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, pinnedlibraryPath.Pointer, out libraryHandle)); } }
/// <summary> /// Add a pre-allocated initializer to a session. If a model contains an initializer with a name /// that is same as the name passed to this API call, ORT will use this initializer instance /// instead of deserializing one from the model file. This is useful when you want to share /// the same initializer across sessions. /// \param name name of the initializer /// \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be /// managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object /// to which it is added. /// </summary> public void AddInitializer(string name, OrtValue ortValue) { var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned); using (var pinnedName = new PinnedGCHandle(utf8NamePinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtAddInitializer(handle, pinnedName.Pointer, ortValue.Handle)); } }
/// <summary> /// Set a single session configuration entry as a pair of strings /// If a configuration with same key exists, this will overwrite the configuration with the given configValue /// </summary> /// <param name="configKey">config key name</param> /// <param name="configValue">config key value</param> public void AddSessionConfigEntry(string configKey, string configValue) { using (var pinnedConfigKeyName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configKey), GCHandleType.Pinned))) using (var pinnedConfigValueName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configValue), GCHandleType.Pinned))) { NativeApiStatus.VerifySuccess(NativeMethods.OrtAddSessionConfigEntry(handle, pinnedConfigKeyName.Pointer, pinnedConfigValueName.Pointer)); } }
/// <summary> /// Use only if you have the onnxruntime package specific to this Execution Provider. /// </summary> /// <param name="settings">string with Nuphar specific settings</param> public void AppendExecutionProvider_Nuphar(string settings = "") { var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned); using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(handle, 1, pinnedSettingsName.Pointer)); } }
/// <summary> /// Use only if you have the onnxruntime package specific to this Execution Provider. /// </summary> /// <param name="deviceId">device identification, default empty string</param> public void AppendExecutionProvider_OpenVINO(string deviceId = "") { var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned); using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer)); } }
/// <summary> /// Updates the configuration knobs of OrtCUDAProviderOptions that will eventually be used to configure a CUDA EP /// Please refer to the following on different key/value pairs to configure a CUDA EP and their meaning: /// https://www.onnxruntime.ai/docs/reference/execution-providers/CUDA-ExecutionProvider.html /// </summary> /// <param name="providerOptions">key/value pairs used to configure a CUDA Execution Provider</param> public void UpdateOptions(Dictionary <string, string> providerOptions) { using (var cleanupList = new DisposableList <IDisposable>()) { var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Keys.ToArray(), n => n, cleanupList); var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Values.ToArray(), n => n, cleanupList); NativeApiStatus.VerifySuccess(NativeMethods.OrtUpdateCUDAProviderOptions(handle, keysArray, valuesArray, (UIntPtr)providerOptions.Count)); } }
/// <summary> /// Use only if you have the onnxruntime package specific to this Execution Provider. /// </summary> /// <param name="settings">string with TVM specific settings</param> public void AppendExecutionProvider_Tvm(string settings = "") { #if __MOBILE__ throw new NotSupportedException("The TVM Execution Provider is not supported in this build"); #else var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned); using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tvm(handle, pinnedSettingsName.Pointer)); } #endif }
/// <summary> /// Use only if you have the onnxruntime package specific to this Execution Provider. /// </summary> /// <param name="deviceId">device identification, default empty string</param> public void AppendExecutionProvider_OpenVINO(string deviceId = "") { #if __MOBILE__ throw new NotSupportedException("The OpenVINO Execution Provider is not supported in this build"); #else var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned); using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer)); } #endif }
/// <summary> /// Get TensorRT EP provider options /// </summary> /// <returns> return C# UTF-16 encoded string </returns> public string GetOptions() { var allocator = OrtAllocator.DefaultInstance; // Process provider options string IntPtr providerOptions = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorRTProviderOptionsAsString(handle, allocator.Pointer, out providerOptions)); using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0)) { return(NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions)); } }
/// <summary> /// A helper method to construct a SessionOptions object for Nuphar execution. /// Use only if you have the onnxruntime package specific to this Execution Provider. /// </summary> /// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param> /// <returns>A SessionsOptions() object configured for execution with Nuphar</returns> public static SessionOptions MakeSessionOptionWithNupharProvider(String settings = "") { SessionOptions options = new SessionOptions(); var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned); using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned)) { NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(options.Handle, 1, pinnedSettingsName.Pointer)); } return(options); }
/// <summary> /// Ends profiling for the session. Returns the profile file name. /// public string EndProfiling() { IntPtr nameHandle = IntPtr.Zero; var allocator = OrtAllocator.DefaultInstance; NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionEndProfiling(_nativeHandle, allocator.Pointer, out nameHandle)); using (var allocation = new OrtMemoryAllocation(allocator, nameHandle, 0)) { return(NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle)); } }
/// <summary> /// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="value"></param> /// <returns></returns> public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value) { if (value is Tensor <string> ) { throw new ArgumentException("Only numeric tensors can be used to create FixedBufferOnnxValue.", nameof(value)); } NativeOnnxValueHelper.CreateNativeOnnxValue(value, out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType); Debug.Assert( onnxValueType == OnnxValueType.ONNX_TYPE_TENSOR && elementType != TensorElementType.String, "the value should always be a numeric tensor"); return(new FixedBufferOnnxValue(pinnedMemoryHandle, onnxValue, onnxValueType, elementType)); }
/// <summary> /// Run helper /// </summary> /// <param name="names">names to convert to zero terminated utf8 and pin</param> /// <param name="cleanupList">list to add pinned memory to for later disposal</param> /// <returns></returns> private IntPtr[] ConvertNamesToUtf8 <T>(IReadOnlyCollection <T> inputs, NameExtractor <T> extractor, DisposableList <IDisposable> cleanupList) { var result = new IntPtr[inputs.Count]; for (int i = 0; i < inputs.Count; ++i) { var name = extractor(inputs.ElementAt(i)); var utf8Name = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name); var pinnedHandle = new PinnedGCHandle(GCHandle.Alloc(utf8Name, GCHandleType.Pinned)); result[i] = pinnedHandle.Pointer; cleanupList.Add(pinnedHandle); } return(result); }
/// <summary> /// Internal helper /// </summary> /// <param name="name"></param> /// <param name="ortValue"></param> /// <param name="isInput"></param> private void BindInputOrOutput(string name, IntPtr ortValue, bool isInput) { var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned); using (var pinnedName = new PinnedGCHandle(utf8NamePinned)) { if (isInput) { NativeApiStatus.VerifySuccess(NativeMethods.OrtBindInput(handle, pinnedName.Pointer, ortValue)); } else { NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutput(handle, pinnedName.Pointer, ortValue)); } } }
private string GetOverridableInitializerName(ulong index) { string str = null; var allocator = OrtAllocator.DefaultInstance; IntPtr nameHandle = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOverridableInitializerName( _nativeHandle, (UIntPtr)index, allocator.Pointer, out nameHandle)); using (var ortAllocation = new OrtMemoryAllocation(allocator, nameHandle, 0)) { str = NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle); } return(str); }
/// <summary> /// Create an instance of OrtMemoryInfo according to the specification. /// </summary> /// <param name="allocatorName">Allocator name</param> /// <param name="allocatorType">Allocator type</param> /// <param name="deviceId">Device id</param> /// <param name="memoryType">Memory type</param> public OrtMemoryInfo(string allocatorName, OrtAllocatorType allocatorType, int deviceId, OrtMemType memoryType) : this(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(allocatorName), allocatorType, deviceId, memoryType) { }