Example #1
0
        /// <summary>
        /// This function will bind model output with the given name to a device
        /// specified by the memInfo.
        /// </summary>
        /// <param name="name">output name</param>
        /// <param name="memInfo">instance of memory info</param>
        public void BindOutputToDevice(string name, OrtMemoryInfo memInfo)
        {
            var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);

            using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
                NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutputToDevice(handle, pinnedName.Pointer, memInfo.Pointer));
        }
Example #2
0
        private static OrtValue CreateStringTensor(Tensor <string> tensor)
        {
            if (tensor == null)
            {
                throw new OnnxRuntimeException(ErrorCode.Fail, "Cast to Tensor<string> failed. BUG check!");
            }

            int totalLength = 0;

            for (int i = 0; i < tensor.Length; i++)
            {
                totalLength += System.Text.Encoding.UTF8.GetByteCount(tensor.GetValue(i));
            }

            long[] shape = new long[tensor.Dimensions.Length];
            for (int i = 0; i < tensor.Dimensions.Length; i++)
            {
                shape[i] = tensor.Dimensions[i];
            }

            // allocate the native tensor
            IntPtr valueHandle = IntPtr.Zero;

            NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue(
                                              OrtAllocator.DefaultInstance.Pointer,
                                              shape,
                                              (UIntPtr)(shape.Length),
                                              TensorElementType.String,
                                              out valueHandle
                                              ));

            var ortValue = new OrtValue(valueHandle);

            try
            {
                // fill the native tensor, using GetValue(index) from the Tensor<string>
                var len           = tensor.Length;
                var nativeStrings = new IntPtr[len];
                using (var pinnedHandles = new DisposableList <PinnedGCHandle>((int)len))
                {
                    for (int i = 0; i < len; i++)
                    {
                        var utf8str  = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(tensor.GetValue(i));
                        var gcHandle = GCHandle.Alloc(utf8str, GCHandleType.Pinned);
                        nativeStrings[i] = gcHandle.AddrOfPinnedObject();
                        pinnedHandles.Add(new PinnedGCHandle(gcHandle));
                    }

                    using (var pinnedStrings = new PinnedGCHandle(GCHandle.Alloc(nativeStrings, GCHandleType.Pinned)))
                        NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(ortValue.Handle, nativeStrings, (UIntPtr)len));
                }
            }
            catch (OnnxRuntimeException)
            {
                ortValue.Dispose();
                throw;
            }
            return(ortValue);
        }
Example #3
0
 /// <summary>
 /// Override symbolic dimensions (by specific name strings) with actual values if known at session initialization time to enable 
 /// optimizations that can take advantage of fixed values (such as memory planning, etc)
 /// </summary>
 public void AddFreeDimensionOverrideByName(string dimName, long dimValue)
 {
     var utf8DimNamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(dimName), GCHandleType.Pinned);
     using (var pinnedDimName = new PinnedGCHandle(utf8DimNamePinned))
     {
         NativeApiStatus.VerifySuccess(NativeMethods.OrtAddFreeDimensionOverrideByName(handle, pinnedDimName.Pointer, dimValue));
     }
 }
Example #4
0
 /// <summary>
 /// Add a pre-allocated initializer to a session. If a model contains an initializer with a name
 /// that is same as the name passed to this API call, ORT will use this initializer instance
 /// instead of deserializing one from the model file. This is useful when you want to share
 /// the same initializer across sessions.
 /// \param name name of the initializer
 /// \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
 /// managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
 /// to which it is added.
 /// </summary>
 public void AddInitializer(string name, OrtValue ortValue)
 {
     var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
     using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
     {
         NativeApiStatus.VerifySuccess(NativeMethods.OrtAddInitializer(handle, pinnedName.Pointer, ortValue.Handle));
     }
 }
Example #5
0
 /// <summary>
 /// Loads a DLL named 'libraryPath' and looks for this entry point:
 /// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api);
 /// It then passes in the provided session options to this function along with the api base.
 /// The handle to the loaded library is returned in 'libraryHandle'. 
 /// It can be unloaded by the caller after all sessions using the passed in
 /// session options are destroyed, or if an error occurs and it is non null.
 /// Hint: .NET Core 3.1 has a 'NativeLibrary' class that can be used to free the library handle
 /// </summary>
 public void RegisterCustomOpLibraryV2(string libraryPath, out IntPtr libraryHandle)
 {
     var libraryPathPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath), GCHandleType.Pinned);
     using (var pinnedlibraryPath = new PinnedGCHandle(libraryPathPinned))
     {
         NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, pinnedlibraryPath.Pointer, out libraryHandle));
     }
 }
Example #6
0
 /// <summary>
 /// Set a single session configuration entry as a pair of strings
 /// If a configuration with same key exists, this will overwrite the configuration with the given configValue
 /// </summary>
 /// <param name="configKey">config key name</param>
 /// <param name="configValue">config key value</param>
 public void AddSessionConfigEntry(string configKey, string configValue)
 {
     using (var pinnedConfigKeyName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configKey), GCHandleType.Pinned)))
         using (var pinnedConfigValueName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configValue), GCHandleType.Pinned)))
         {
             NativeApiStatus.VerifySuccess(NativeMethods.OrtAddSessionConfigEntry(handle,
                                                                                  pinnedConfigKeyName.Pointer, pinnedConfigValueName.Pointer));
         }
 }
Example #7
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="settings">string with Nuphar specific settings</param>
        public void AppendExecutionProvider_Nuphar(string settings = "")
        {
            var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);

            using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(handle, 1, pinnedSettingsName.Pointer));
            }
        }
Example #8
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="deviceId">device identification, default empty string</param>
        public void AppendExecutionProvider_OpenVINO(string deviceId = "")
        {
            var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);

            using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
            }
        }
Example #9
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="settings">string with TVM specific settings</param>
        public void AppendExecutionProvider_Tvm(string settings = "")
        {
#if __MOBILE__
            throw new NotSupportedException("The TVM Execution Provider is not supported in this build");
#else
            var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
            using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tvm(handle, pinnedSettingsName.Pointer));
            }
#endif
        }
Example #10
0
        /// <summary>
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="deviceId">device identification, default empty string</param>
        public void AppendExecutionProvider_OpenVINO(string deviceId = "")
        {
#if __MOBILE__
            throw new NotSupportedException("The OpenVINO Execution Provider is not supported in this build");
#else
            var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);
            using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
            }
#endif
        }
Example #11
0
 /// <summary>
 /// Create an instance of OrtMemoryInfo according to the specification
 /// Memory info instances are usually used to get a handle of a native allocator
 /// that is present within the current inference session object. That, in turn, depends
 /// of what execution providers are available within the binary that you are using and are
 /// registered with Add methods.
 /// </summary>
 /// <param name="utf8AllocatorName">Allocator name. Use of the predefined above.</param>
 /// <param name="allocatorType">Allocator type</param>
 /// <param name="deviceId">Device id</param>
 /// <param name="memoryType">Memory type</param>
 public OrtMemoryInfo(byte[] utf8AllocatorName, OrtAllocatorType allocatorType, int deviceId, OrtMemType memoryType)
 {
     using (var pinnedName = new PinnedGCHandle(GCHandle.Alloc(utf8AllocatorName, GCHandleType.Pinned)))
     {
         NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateMemoryInfo(pinnedName.Pointer,
                                                                         allocatorType,
                                                                         deviceId,
                                                                         memoryType,
                                                                         out _pointer));
     }
     _owned = true;
 }
Example #12
0
        /// <summary>
        /// A helper method to construct a SessionOptions object for Nuphar execution.
        /// Use only if you have the onnxruntime package specific to this Execution Provider.
        /// </summary>
        /// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param>
        /// <returns>A SessionsOptions() object configured for execution with Nuphar</returns>
        public static SessionOptions MakeSessionOptionWithNupharProvider(String settings = "")
        {
            SessionOptions options = new SessionOptions();

            var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);

            using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
            {
                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(options.Handle, 1, pinnedSettingsName.Pointer));
            }

            return(options);
        }
        /// <summary>
        /// Run helper
        /// </summary>
        /// <param name="names">names to convert to zero terminated utf8 and pin</param>
        /// <param name="cleanupList">list to add pinned memory to for later disposal</param>
        /// <returns></returns>
        private IntPtr[] ConvertNamesToUtf8 <T>(IReadOnlyCollection <T> inputs, NameExtractor <T> extractor,
                                                DisposableList <IDisposable> cleanupList)
        {
            var result = new IntPtr[inputs.Count];

            for (int i = 0; i < inputs.Count; ++i)
            {
                var name         = extractor(inputs.ElementAt(i));
                var utf8Name     = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name);
                var pinnedHandle = new PinnedGCHandle(GCHandle.Alloc(utf8Name, GCHandleType.Pinned));
                result[i] = pinnedHandle.Pointer;
                cleanupList.Add(pinnedHandle);
            }
            return(result);
        }
Example #14
0
        /// <summary>
        /// Internal helper
        /// </summary>
        /// <param name="name"></param>
        /// <param name="ortValue"></param>
        /// <param name="isInput"></param>
        private void BindInputOrOutput(string name, IntPtr ortValue, bool isInput)
        {
            var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);

            using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
            {
                if (isInput)
                {
                    NativeApiStatus.VerifySuccess(NativeMethods.OrtBindInput(handle, pinnedName.Pointer, ortValue));
                }
                else
                {
                    NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutput(handle, pinnedName.Pointer, ortValue));
                }
            }
        }