Exemple #1
0
        /// <summary>
        /// Append SNPE or XNNPACK execution provider
        /// </summary>
        /// <param name="providerName">Execution provider to add. 'SNPE' or 'XNNPACK' are currently supported.</param>
        /// <param name="providerOptions">Optional key/value pairs to specify execution provider options.</param>
        public void AppendExecutionProvider(string providerName, Dictionary <string, string> providerOptions = null)
        {
            if (providerName != "SNPE" && providerName != "XNNPACK")
            {
                throw new NotSupportedException(
                          "Only SNPE and XNNPACK execution providers can be enabled by this method.");
            }

            using (var cleanupList = new DisposableList <IDisposable>())
            {
                string[] ep      = { providerName }; // put in array so we can use ConvertNamesToUtf8 for everything
                var      epArray = NativeOnnxValueHelper.ConvertNamesToUtf8(ep, n => n, cleanupList);

                if (providerOptions == null)
                {
                    providerOptions = new Dictionary <string, string>();
                }

                var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8(
                    providerOptions.Keys.ToArray(), n => n, cleanupList);

                var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(
                    providerOptions.Values.ToArray(), n => n, cleanupList);

                NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider(
                                                  handle, epArray[0], keysArray, valuesArray, (UIntPtr)providerOptions.Count));
            }
        }
        /// <summary>
        /// This function will bind model output with the given name to a device
        /// specified by the memInfo.
        /// </summary>
        /// <param name="name">output name</param>
        /// <param name="memInfo">instance of memory info</param>
        public void BindOutputToDevice(string name, OrtMemoryInfo memInfo)
        {
            var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);

            using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
                NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutputToDevice(handle, pinnedName.Pointer, memInfo.Pointer));
        }
Exemple #3
0
        /// <summary>
        /// Queries all the execution providers supported in the native onnxruntime shared library
        /// </summary>
        /// <returns>an array of strings that represent execution provider names</returns>
        public string[] GetAvailableProviders()
        {
            IntPtr availableProvidersHandle = IntPtr.Zero;
            int    numProviders;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtGetAvailableProviders(out availableProvidersHandle, out numProviders));

            var availableProviders = new string[numProviders];

            try
            {
                for (int i = 0; i < numProviders; ++i)
                {
                    availableProviders[i] = NativeOnnxValueHelper.StringFromNativeUtf8(Marshal.ReadIntPtr(availableProvidersHandle, IntPtr.Size * i));
                }
            }

            finally
            {
                // Looks a bit weird that we might throw in finally(...)
                // But the native method OrtReleaseAvailableProviders actually doesn't return a failure status
                // If it does, it is BUG and we would like to propagate that to the user in the form of an exception
                NativeApiStatus.VerifySuccess(NativeMethods.OrtReleaseAvailableProviders(availableProvidersHandle, numProviders));
            }

            return(availableProviders);
        }
Exemple #4
0
        private static string GetErrorMessage(IntPtr /*(ONNXStatus*)*/ status)
        {
            // nativeString belongs to status, no need for separate release
            IntPtr nativeString = NativeMethods.OrtGetErrorMessage(status);

            return(NativeOnnxValueHelper.StringFromNativeUtf8(nativeString));
        }
Exemple #5
0
        private static OrtValue CreateStringTensor(Tensor <string> tensor)
        {
            if (tensor == null)
            {
                throw new OnnxRuntimeException(ErrorCode.Fail, "Cast to Tensor<string> failed. BUG check!");
            }

            int totalLength = 0;

            for (int i = 0; i < tensor.Length; i++)
            {
                totalLength += System.Text.Encoding.UTF8.GetByteCount(tensor.GetValue(i));
            }

            long[] shape = new long[tensor.Dimensions.Length];
            for (int i = 0; i < tensor.Dimensions.Length; i++)
            {
                shape[i] = tensor.Dimensions[i];
            }

            // allocate the native tensor
            IntPtr valueHandle = IntPtr.Zero;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue(
                                              OrtAllocator.DefaultInstance.Pointer,
                                              shape,
                                              (UIntPtr)(shape.Length),
                                              TensorElementType.String,
                                              out valueHandle
                                              ));

            var ortValue = new OrtValue(valueHandle);

            try
            {
                // fill the native tensor, using GetValue(index) from the Tensor<string>
                var len           = tensor.Length;
                var nativeStrings = new IntPtr[len];
                using (var pinnedHandles = new DisposableList <PinnedGCHandle>((int)len))
                {
                    for (int i = 0; i < len; i++)
                    {
                        var utf8str  = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(tensor.GetValue(i));
                        var gcHandle = GCHandle.Alloc(utf8str, GCHandleType.Pinned);
                        nativeStrings[i] = gcHandle.AddrOfPinnedObject();
                        pinnedHandles.Add(new PinnedGCHandle(gcHandle));
                    }

                    using (var pinnedStrings = new PinnedGCHandle(GCHandle.Alloc(nativeStrings, GCHandleType.Pinned)))
                        NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(ortValue.Handle, nativeStrings, (UIntPtr)len));
                }
            }
            catch (OnnxRuntimeException)
            {
                ortValue.Dispose();
                throw;
            }
            return(ortValue);
        }
        internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo)
        {
            OnnxValueType valueType;

            {
                IntPtr valType;
                NativeApiStatus.VerifySuccess(NativeMethods.OrtGetOnnxTypeFromTypeInfo(typeInfo, out valType));
                valueType = (OnnxValueType)valType;
            }
            if (valueType != OnnxValueType.ONNX_TYPE_TENSOR && valueType != OnnxValueType.ONNX_TYPE_SPARSETENSOR)
            {
                return(new NodeMetadata(valueType, new int[] { }, new string[] { }, typeof(NamedOnnxValue)));
            }

            // This should not be released
            IntPtr tensorInfo;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo, out tensorInfo)); //(IntPtr)(int)(uint)
            // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo*

            if (tensorInfo == IntPtr.Zero)
            {
                return(null);
            }

            TensorElementType type;
            {
                IntPtr el_type;
                NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(tensorInfo, out el_type));
                type = (TensorElementType)el_type;
            }
            Type dotnetType = null;
            int  width      = 0;

            TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width);
            UIntPtr numDimensions;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensionsCount(tensorInfo, out numDimensions));

            long[] dimensions = new long[(int)numDimensions];
            NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDimensions(tensorInfo, dimensions, numDimensions));
            int[] intDimensions = new int[(int)numDimensions];
            for (var i = 0; i < (long)numDimensions; i++)
            {
                intDimensions[i] = (int)dimensions[i];
            }

            IntPtr[] dimensionNamePtrs = new IntPtr[(int)numDimensions];
            NativeApiStatus.VerifySuccess(
                NativeMethods.OrtGetSymbolicDimensions(tensorInfo, dimensionNamePtrs, numDimensions));

            string[] symbolicDimensions = new string[(int)numDimensions];
            for (var i = 0; i < (int)numDimensions; i++)
            {
                symbolicDimensions[i] = NativeOnnxValueHelper.StringFromNativeUtf8(dimensionNamePtrs[i]);
            }

            return(new NodeMetadata(valueType, intDimensions, symbolicDimensions, dotnetType));
        }
        /// <summary>
        /// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="value"></param>
        /// <returns></returns>
        public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value)
        {
            NativeOnnxValueHelper.CreateNativeOnnxValue(value, out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType);

            Debug.Assert(onnxValueType == OnnxValueType.ONNX_TYPE_TENSOR, "the value should always be a tensor");

            return(new FixedBufferOnnxValue(pinnedMemoryHandle, onnxValue, onnxValueType, elementType));
        }
 /// <summary>
 /// Override symbolic dimensions (by specific name strings) with actual values if known at session initialization time to enable 
 /// optimizations that can take advantage of fixed values (such as memory planning, etc)
 /// </summary>
 public void AddFreeDimensionOverrideByName(string dimName, long dimValue)
 {
     var utf8DimNamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(dimName), GCHandleType.Pinned);
     using (var pinnedDimName = new PinnedGCHandle(utf8DimNamePinned))
     {
         NativeApiStatus.VerifySuccess(NativeMethods.OrtAddFreeDimensionOverrideByName(handle, pinnedDimName.Pointer, dimValue));
     }
 }
Exemple #9
0
 /// <summary>
 /// Pin the underlying memory and create native onnx value
 /// </summary>
 /// <param name="onnxValue"></param>
 /// <param name="pinnedMemoryHandle"></param>
 /// <param name="disposeOnnxValueAfterUse"></param>
 internal virtual void ToNativeOnnxValue(
     out IntPtr onnxValue,
     out MemoryHandle pinnedMemoryHandle,
     out bool disposeOnnxValueAfterUse)
 {
     NativeOnnxValueHelper.CreateNativeOnnxValue(_value, out onnxValue, out pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType);
     disposeOnnxValueAfterUse = true;
 }
 /// <summary>
 /// Loads a DLL named 'libraryPath' and looks for this entry point:
 /// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api);
 /// It then passes in the provided session options to this function along with the api base.
 /// The handle to the loaded library is returned in 'libraryHandle'. 
 /// It can be unloaded by the caller after all sessions using the passed in
 /// session options are destroyed, or if an error occurs and it is non null.
 /// Hint: .NET Core 3.1 has a 'NativeLibrary' class that can be used to free the library handle
 /// </summary>
 public void RegisterCustomOpLibraryV2(string libraryPath, out IntPtr libraryHandle)
 {
     var libraryPathPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath), GCHandleType.Pinned);
     using (var pinnedlibraryPath = new PinnedGCHandle(libraryPathPinned))
     {
         NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, pinnedlibraryPath.Pointer, out libraryHandle));
     }
 }
 /// <summary>
 /// Add a pre-allocated initializer to a session. If a model contains an initializer with a name
 /// that is same as the name passed to this API call, ORT will use this initializer instance
 /// instead of deserializing one from the model file. This is useful when you want to share
 /// the same initializer across sessions.
 /// \param name name of the initializer
 /// \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
 /// managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
 /// to which it is added.
 /// </summary>
 public void AddInitializer(string name, OrtValue ortValue)
 {
     var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
     using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
     {
         NativeApiStatus.VerifySuccess(NativeMethods.OrtAddInitializer(handle, pinnedName.Pointer, ortValue.Handle));
     }
 }
Exemple #12
0
 /// <summary>
 /// Set a single session configuration entry as a pair of strings
 /// If a configuration with same key exists, this will overwrite the configuration with the given configValue
 /// </summary>
 /// <param name="configKey">config key name</param>
 /// <param name="configValue">config key value</param>
 public void AddSessionConfigEntry(string configKey, string configValue)
 {
     using (var pinnedConfigKeyName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configKey), GCHandleType.Pinned)))
         using (var pinnedConfigValueName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configValue), GCHandleType.Pinned)))
         {
             NativeApiStatus.VerifySuccess(NativeMethods.OrtAddSessionConfigEntry(handle,
                                                                                  pinnedConfigKeyName.Pointer, pinnedConfigValueName.Pointer));
         }
 }
Exemple #13
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="settings">string with Nuphar specific settings</param>
        public void AppendExecutionProvider_Nuphar(string settings = "")
        {
            var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);

            using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(handle, 1, pinnedSettingsName.Pointer));
            }
        }
Exemple #14
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="deviceId">device identification, default empty string</param>
        public void AppendExecutionProvider_OpenVINO(string deviceId = "")
        {
            var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);

            using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
            }
        }
Exemple #15
0
        /// <summary>
        /// Updates  the configuration knobs of OrtCUDAProviderOptions that will eventually be used to configure a CUDA EP
        /// Please refer to the following on different key/value pairs to configure a CUDA EP and their meaning:
        /// https://www.onnxruntime.ai/docs/reference/execution-providers/CUDA-ExecutionProvider.html
        /// </summary>
        /// <param name="providerOptions">key/value pairs used to configure a CUDA Execution Provider</param>
        public void UpdateOptions(Dictionary <string, string> providerOptions)
        {
            using (var cleanupList = new DisposableList <IDisposable>())
            {
                var keysArray   = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Keys.ToArray(), n => n, cleanupList);
                var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Values.ToArray(), n => n, cleanupList);

                NativeApiStatus.VerifySuccess(NativeMethods.OrtUpdateCUDAProviderOptions(handle, keysArray, valuesArray, (UIntPtr)providerOptions.Count));
            }
        }
Exemple #16
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="settings">string with TVM specific settings</param>
        public void AppendExecutionProvider_Tvm(string settings = "")
        {
#if __MOBILE__
            throw new NotSupportedException("The TVM Execution Provider is not supported in this build");
#else
            var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
            using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tvm(handle, pinnedSettingsName.Pointer));
            }
#endif
        }
Exemple #17
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="deviceId">device identification, default empty string</param>
        public void AppendExecutionProvider_OpenVINO(string deviceId = "")
        {
#if __MOBILE__
            throw new NotSupportedException("The OpenVINO Execution Provider is not supported in this build");
#else
            var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);
            using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
            }
#endif
        }
Exemple #18
0
        /// <summary>
        /// Get TensorRT EP provider options
        /// </summary>
        /// <returns> return C# UTF-16 encoded string </returns>
        public string GetOptions()
        {
            var allocator = OrtAllocator.DefaultInstance;

            // Process provider options string
            IntPtr providerOptions = IntPtr.Zero;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorRTProviderOptionsAsString(handle, allocator.Pointer, out providerOptions));
            using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0))
            {
                return(NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions));
            }
        }
Exemple #19
0
        /// <summary>
        /// A helper method to construct a SessionOptions object for Nuphar execution.
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param>
        /// <returns>A SessionsOptions() object configured for execution with Nuphar</returns>
        public static SessionOptions MakeSessionOptionWithNupharProvider(String settings = "")
        {
            SessionOptions options = new SessionOptions();

            var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);

            using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(options.Handle, 1, pinnedSettingsName.Pointer));
            }

            return(options);
        }
        /// <summary>
        /// Ends profiling for the session. Returns the profile file name.
        ///
        public string EndProfiling()
        {
            IntPtr nameHandle = IntPtr.Zero;
            var    allocator  = OrtAllocator.DefaultInstance;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionEndProfiling(_nativeHandle,
                                                                               allocator.Pointer,
                                                                               out nameHandle));
            using (var allocation = new OrtMemoryAllocation(allocator, nameHandle, 0))
            {
                return(NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle));
            }
        }
Exemple #21
0
        /// <summary>
        /// Creates a <see cref="FixedBufferOnnxValue"/> object from the tensor and pins its underlying buffer.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="value"></param>
        /// <returns></returns>
        public static FixedBufferOnnxValue CreateFromTensor <T>(Tensor <T> value)
        {
            if (value is Tensor <string> )
            {
                throw new ArgumentException("Only numeric tensors can be used to create FixedBufferOnnxValue.", nameof(value));
            }

            NativeOnnxValueHelper.CreateNativeOnnxValue(value, out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle, out OnnxValueType onnxValueType, out TensorElementType elementType);

            Debug.Assert(
                onnxValueType == OnnxValueType.ONNX_TYPE_TENSOR && elementType != TensorElementType.String,
                "the value should always be a numeric tensor");

            return(new FixedBufferOnnxValue(pinnedMemoryHandle, onnxValue, onnxValueType, elementType));
        }
        /// <summary>
        /// Run helper
        /// </summary>
        /// <param name="names">names to convert to zero terminated utf8 and pin</param>
        /// <param name="cleanupList">list to add pinned memory to for later disposal</param>
        /// <returns></returns>
        private IntPtr[] ConvertNamesToUtf8 <T>(IReadOnlyCollection <T> inputs, NameExtractor <T> extractor,
                                                DisposableList <IDisposable> cleanupList)
        {
            var result = new IntPtr[inputs.Count];

            for (int i = 0; i < inputs.Count; ++i)
            {
                var name         = extractor(inputs.ElementAt(i));
                var utf8Name     = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name);
                var pinnedHandle = new PinnedGCHandle(GCHandle.Alloc(utf8Name, GCHandleType.Pinned));
                result[i] = pinnedHandle.Pointer;
                cleanupList.Add(pinnedHandle);
            }
            return(result);
        }
Exemple #23
0
        /// <summary>
        /// Internal helper
        /// </summary>
        /// <param name="name"></param>
        /// <param name="ortValue"></param>
        /// <param name="isInput"></param>
        private void BindInputOrOutput(string name, IntPtr ortValue, bool isInput)
        {
            var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);

            using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
            {
                if (isInput)
                {
                    NativeApiStatus.VerifySuccess(NativeMethods.OrtBindInput(handle, pinnedName.Pointer, ortValue));
                }
                else
                {
                    NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutput(handle, pinnedName.Pointer, ortValue));
                }
            }
        }
        private string GetOverridableInitializerName(ulong index)
        {
            string str        = null;
            var    allocator  = OrtAllocator.DefaultInstance;
            IntPtr nameHandle = IntPtr.Zero;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOverridableInitializerName(
                                              _nativeHandle,
                                              (UIntPtr)index,
                                              allocator.Pointer,
                                              out nameHandle));
            using (var ortAllocation = new OrtMemoryAllocation(allocator, nameHandle, 0))
            {
                str = NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle);
            }
            return(str);
        }
Exemple #25
0
 /// <summary>
 /// Create an instance of OrtMemoryInfo according to the specification.
 /// </summary>
 /// <param name="allocatorName">Allocator name</param>
 /// <param name="allocatorType">Allocator type</param>
 /// <param name="deviceId">Device id</param>
 /// <param name="memoryType">Memory type</param>
 public OrtMemoryInfo(string allocatorName, OrtAllocatorType allocatorType, int deviceId, OrtMemType memoryType)
     : this(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(allocatorName), allocatorType, deviceId, memoryType)
 {
 }