/// <summary> /// Initializes the session object with a native session handle /// </summary> /// <param name="session">Handle of a native session object</param> /// <param name="options">Session options</param> private void InitWithSessionHandle(IntPtr session, SessionOptions options) { _nativeHandle = session; try { // Initialize input/output metadata _inputMetadata = new Dictionary <string, NodeMetadata>(); _outputMetadata = new Dictionary <string, NodeMetadata>(); // get input count UIntPtr inputCount = UIntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetInputCount(_nativeHandle, out inputCount)); // get all the output names for (ulong i = 0; i < (ulong)inputCount; i++) { var iname = GetInputName(i); _inputMetadata[iname] = GetInputMetadata(i); } // get output count UIntPtr outputCount = UIntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOutputCount(_nativeHandle, out outputCount)); // get all the output names for (ulong i = 0; i < (ulong)outputCount; i++) { _outputMetadata[GetOutputName(i)] = GetOutputMetadata(i); } } catch (OnnxRuntimeException e) { if (_nativeHandle != IntPtr.Zero) { NativeMethods.OrtReleaseSession(_nativeHandle); _nativeHandle = IntPtr.Zero; } throw e; } _builtInRunOptions = new RunOptions(); // create a default built-in run option, and avoid creating a new one every run() call }
/// <summary> /// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run. /// </summary> /// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param> /// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param> /// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param> /// <param name="options"></param> /// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns> public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run( IReadOnlyCollection <string> inputNames, IReadOnlyCollection <FixedBufferOnnxValue> inputValues, IReadOnlyCollection <string> outputNames, RunOptions options) { if (inputNames.Count != inputValues.Count) { throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count})."); } using (var cleanupList = new DisposableList <IDisposable>()) { var inputNamesArray = ConvertNamesToUtf8(inputNames, n => n, cleanupList); IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true); var outputNamesArray = ConvertNamesToUtf8(outputNames, n => n, cleanupList); var ortValues = RunImpl(options, inputNamesArray, inputValuesArray, outputNamesArray, cleanupList); return(CreateDisposableResult(ortValues, outputNames)); } }
/// <summary> /// Runs the loaded model for the given inputs and outputs. Uses the given RunOptions for this run. /// /// Outputs need to be created with correct type and dimension to accept the fetched data. /// </summary> /// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param> /// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param> /// <param name="outputNames">Specify a collection of string that indicates the output names. Should match <paramref name="outputValues"/>.</param> /// <param name="outputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the output values.</param> /// <param name="options"></param> public void Run( IReadOnlyCollection <string> inputNames, IReadOnlyCollection <FixedBufferOnnxValue> inputValues, IReadOnlyCollection <string> outputNames, IReadOnlyCollection <FixedBufferOnnxValue> outputValues, RunOptions options) { if (inputNames.Count != inputValues.Count) { throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count})."); } if (outputNames.Count != outputValues.Count) { throw new ArgumentException($"Length of {nameof(outputNames)} ({outputNames.Count}) must match that of {nameof(outputValues)} ({outputValues.Count})."); } using (var cleanupList = new DisposableList <IDisposable>()) { // prepare inputs var inputNamesArray = ConvertNamesToUtf8(inputNames, n => n, cleanupList); IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true); // prepare outputs var outputNamesArray = ConvertNamesToUtf8(outputNames, n => n, cleanupList); IntPtr[] outputValuesArray = GetOrtValuesHandles(outputValues, false); NativeApiStatus.VerifySuccess(NativeMethods.OrtRun( _nativeHandle, options.Handle, inputNamesArray, inputValuesArray, (UIntPtr)inputNames.Count, outputNamesArray, (UIntPtr)outputNames.Count, outputValuesArray /* pointers to Pre-allocated OrtValue instances */ )); } }
/// <summary> /// Runs the loaded model for the given inputs and outputs. Uses the given RunOptions for this run. /// /// Outputs need to be created with correct type and dimension to receive the fetched data. /// </summary> /// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param> /// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param> /// <param name="output">Specify a collection of <see cref="NamedOnnxValue"/> that indicates the output values.</param> /// <param name="options"></param> public void Run( IReadOnlyCollection <string> inputNames, IReadOnlyCollection <FixedBufferOnnxValue> inputValues, IReadOnlyCollection <NamedOnnxValue> outputs, RunOptions options) { if (inputNames.Count != inputValues.Count) { throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count})."); } var outputNamesArray = new string[outputs.Count]; var outputValuesArray = new IntPtr[outputs.Count]; var pinnedOutputBufferHandles = new System.Buffers.MemoryHandle[outputs.Count]; var disposeOutputs = new bool[outputs.Count]; try { // prepare inputs string[] inputNamesArray = inputNames as string[] ?? inputNames.ToArray(); IntPtr[] inputValuesArray = new IntPtr[inputNames.Count]; int inputIndex = 0; foreach (var input in inputValues) { inputValuesArray[inputIndex] = input.Value; inputIndex++; } // prepare outputs int outputIndex = 0; foreach (var output in outputs) { outputNamesArray[outputIndex] = output.Name; // create native OrtValue from the output if feasible, else throw notsupported exception for now output.ToNativeOnnxValue( out outputValuesArray[outputIndex], out pinnedOutputBufferHandles[outputIndex], out disposeOutputs[outputIndex]); outputIndex++; } IntPtr status = NativeMethods.OrtRun( _nativeHandle, options.Handle, inputNamesArray, inputValuesArray, (UIntPtr)inputNames.Count, outputNamesArray, (UIntPtr)outputs.Count, outputValuesArray /* pointers to Pre-allocated OrtValue instances */ ); NativeApiStatus.VerifySuccess(status); } finally { for (int i = 0; i < outputs.Count; i++) { if (disposeOutputs[i]) { NativeMethods.OrtReleaseValue(outputValuesArray[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object. // For string tensors, this releases the native memory allocated for the tensor, including the buffer pinnedOutputBufferHandles[i].Dispose(); } } } }
/// <summary> /// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run. /// </summary> /// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param> /// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param> /// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param> /// <param name="options"></param> /// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns> public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run( IReadOnlyCollection <string> inputNames, IReadOnlyCollection <FixedBufferOnnxValue> inputValues, IReadOnlyCollection <string> outputNames, RunOptions options) { if (inputNames.Count != inputValues.Count) { throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count})."); } // prepare inputs string[] inputNamesArray = inputNames as string[] ?? inputNames.ToArray(); IntPtr[] inputValuesArray = new IntPtr[inputNames.Count]; int inputIndex = 0; foreach (var input in inputValues) { inputValuesArray[inputIndex] = input.Value; inputIndex++; } // prepare outputs string[] outputNamesArray = outputNames as string[] ?? outputNames.ToArray(); IntPtr[] outputValuesArray = new IntPtr[outputNames.Count]; IntPtr status = NativeMethods.OrtRun( _nativeHandle, options.Handle, inputNamesArray, inputValuesArray, (UIntPtr)inputNames.Count, outputNamesArray, (UIntPtr)outputNames.Count, outputValuesArray /* Empty array is passed in to receive output OrtValue pointers */ ); try { NativeApiStatus.VerifySuccess(status); var result = new DisposableList <DisposableNamedOnnxValue>(outputValuesArray.Length); for (int i = 0; i < outputValuesArray.Length; i++) { result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValuesArray[i])); } return(result); } catch (OnnxRuntimeException e) { //clean up the individual output tensors if it is not null; for (uint i = 0; i < outputValuesArray.Length; i++) { if (outputValuesArray[i] != IntPtr.Zero) { NativeMethods.OrtReleaseValue(outputValuesArray[i]); } } throw e; } }
/// <summary> /// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run. /// </summary> /// <param name="inputs">Specify a collection of <see cref="NamedOnnxValue"/> that indicates the input values.</param> /// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param> /// <param name="options"></param> /// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns> public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options) { // prepare inputs var inputNamesArray = new string[inputs.Count]; var inputValuesArray = new IntPtr[inputs.Count]; var pinnedInputBufferHandles = new System.Buffers.MemoryHandle[inputs.Count]; var disposeInputs = new bool[inputs.Count]; int inputIndex = 0; foreach (var input in inputs) { inputNamesArray[inputIndex] = input.Name; // create Tensor from the input if feasible, else throw notsupported exception for now input.ToNativeOnnxValue( out inputValuesArray[inputIndex], out pinnedInputBufferHandles[inputIndex], out disposeInputs[inputIndex]); inputIndex++; } // prepare outputs string[] outputNamesArray = outputNames as string[] ?? outputNames.ToArray(); IntPtr[] outputValuesArray = new IntPtr[outputNames.Count]; IntPtr status = NativeMethods.OrtRun( _nativeHandle, options.Handle, inputNamesArray, inputValuesArray, (UIntPtr)inputs.Count, outputNamesArray, (UIntPtr)outputNames.Count, outputValuesArray /* Empty array is passed in to receive output OrtValue pointers */ ); try { NativeApiStatus.VerifySuccess(status); var result = new DisposableList <DisposableNamedOnnxValue>(outputValuesArray.Length); for (int i = 0; i < outputValuesArray.Length; i++) { result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValuesArray[i])); } return(result); } catch (OnnxRuntimeException e) { //clean up the individual output tensors if it is not null; for (int i = 0; i < outputValuesArray.Length; i++) { if (outputValuesArray[i] != IntPtr.Zero) { NativeMethods.OrtReleaseValue(outputValuesArray[i]); } } throw e; } finally { for (int i = 0; i < inputs.Count; i++) { if (disposeInputs[i]) { NativeMethods.OrtReleaseValue(inputValuesArray[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object. // For string tensors, this releases the native memory allocated for the tensor, including the buffer pinnedInputBufferHandles[i].Dispose(); } } } }
/// <summary> /// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. /// </summary> /// <param name="inputs"></param> /// <param name="outputNames"></param> /// <param name="options"></param> /// <returns>Output Tensors in a Dictionary</returns> //TODO: kept internal until RunOptions is made public internal IReadOnlyCollection <NamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options) { var inputNames = new string[inputs.Count]; var inputTensors = new IntPtr[inputs.Count]; var pinnedBufferHandles = new System.Buffers.MemoryHandle[inputs.Count]; int offset = 0; foreach (var input in inputs) { inputNames[offset] = input.Name; // create Tensor from the input if feasible, else throw notsupported exception for now input.ToNativeOnnxValue(out inputTensors[offset], out pinnedBufferHandles[offset]); offset++; } string[] outputNamesArray = outputNames.ToArray(); IntPtr[] outputValueArray = new IntPtr[outputNames.Count]; IntPtr status = NativeMethods.OrtRunInference( this._nativeHandle, IntPtr.Zero, // TODO: use Run options when Run options creation API is available // Passing null uses the default run options in the C-api inputNames, inputTensors, (ulong)(inputTensors.Length), /* TODO: size_t, make it portable for x86 arm */ outputNamesArray, (ulong)outputNames.Count, /* TODO: size_t, make it portable for x86 and arm */ outputValueArray /* An array of output value pointers. Array must be allocated by the caller */ ); try { NativeApiStatus.VerifySuccess(status); var result = new List <NamedOnnxValue>(); for (uint i = 0; i < outputValueArray.Length; i++) { result.Add(NamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i])); } return(result); } catch (OnnxRuntimeException e) { //clean up the individual output tensors if it is not null; for (uint i = 0; i < outputValueArray.Length; i++) { if (outputValueArray[i] != IntPtr.Zero) { NativeMethods.OrtReleaseValue(outputValueArray[i]); } } throw e; } finally { // always unpin the input buffers, and delete the native Onnx value objects for (int i = 0; i < inputs.Count; i++) { NativeMethods.OrtReleaseValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object pinnedBufferHandles[i].Dispose(); } } }
/// <summary> /// This method return a collection of DisposableNamedOnnxValue as in other interfaces /// Query names from OrtIoBinding object and pair then with the array of OrtValues returned /// from OrtIoBinding.GetOutputValues() /// /// </summary> /// <param name="runOptions">RunOptions</param> /// <param name="ioBinding">OrtIoBinding instance with bindings</param> /// <param name="names">optional parameter. If you already know the names of the outputs you can save a native /// call to retrieve output names. They will be paired with the returned OrtValues and combined into DisposbleNamedOnnxValues. /// Otherwise, the method will retrieve output names from the OrtIoBinding instance. /// It is an error if you supply a different number of names than the returned outputs</param> public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> RunWithBindingAndNames(RunOptions runOptions, OrtIoBinding ioBinding, string[] names = null) { NativeApiStatus.VerifySuccess(NativeMethods.OrtRunWithBinding(Handle, runOptions.Handle, ioBinding.Handle)); using (var ortValues = ioBinding.GetOutputValues()) { string[] outputNames = names; if (outputNames == null) { outputNames = ioBinding.GetOutputNames(); } if (outputNames.Length != ortValues.Count) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Number of specified names: " + names.Length + " does not match the output number: " + ortValues.Count); } var result = new DisposableList <DisposableNamedOnnxValue>(outputNames.Length); try { for (int i = 0; i < outputNames.Length; ++i) { var ortValue = ortValues.ElementAt(i); result.Add(DisposableNamedOnnxValue.CreateTensorFromOnnxValue(outputNames[i], ortValue.Handle)); ortValue.Disown(); } } catch (Exception e) { result.Dispose(); throw e; } return(result); } }
/// <summary> /// This method runs inference on the OrtIoBinding instance /// The method does not return anything. This is a lightweight version of /// RunWithBindingAndNames(). When you bind pre-allocated buffers to the output values /// you may not want to fetch the outputs since you already have access to them so you can spare /// the expense of fetching them and pairing with names. /// You can still fetch the outputs by calling OrtIOBinding.GetOutputValues() /// </summary> /// <param name="runOptions"></param> /// <param name="ioBinding"></param> public void RunWithBinding(RunOptions runOptions, OrtIoBinding ioBinding) { NativeApiStatus.VerifySuccess(NativeMethods.OrtRunWithBinding(Handle, runOptions.Handle, ioBinding.Handle)); }
/// <summary> /// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run. /// </summary> /// <param name="inputs">Specify a collection of <see cref="NamedOnnxValue"/> that indicates the input values.</param> /// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param> /// <param name="options"></param> /// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns> public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options) { using (var cleanupList = new DisposableList <IDisposable>()) { var inputNamesArray = ConvertNamesToUtf8(inputs, v => v.Name, cleanupList); var inputValuesArray = GetOrtValuesHandles(inputs, cleanupList); var outputNamesArray = ConvertNamesToUtf8(outputNames, n => n, cleanupList); var ortValues = RunImpl(options, inputNamesArray, inputValuesArray, outputNamesArray, cleanupList); return(CreateDisposableResult(ortValues, outputNames)); } }
public unsafe int Run(IReadOnlyCollection <NamedOnnxValue> inputs, RunOptions options, string[] outputNames, IntPtr *outputValues) { var inputCount = inputs.Count; // HACK: Only handling IReadOnlyLists for now! var inputsList = (IReadOnlyList <NamedOnnxValue>)inputs; var inputNames = GetInputNames(inputsList); var inputTensors = stackalloc IntPtr[inputCount]; // Ugly hack to avoid having to create this array var pinnedBufferHandles = m_threadLocalMemoryHandles.Value == null || m_threadLocalMemoryHandles.Value.Length != inputCount ? new MemoryHandle[inputCount] : m_threadLocalMemoryHandles.Value; m_threadLocalMemoryHandles.Value = pinnedBufferHandles; int inputIndex = 0; for (int i = 0; i < inputCount; i++) { var input = inputsList[i]; inputNames[inputIndex] = input.Name; // create Tensor from the input if feasible, else throw notsupported exception for now input.ToNativeOnnxValue(out inputTensors[inputIndex], out pinnedBufferHandles[inputIndex]); inputIndex++; } var outputCount = outputNames.Length; IntPtr status = NativeMethods.OrtRunFast( this._nativeHandle, options.Handle, inputNames, inputTensors, (UIntPtr)(inputCount), outputNames, (UIntPtr)outputCount, outputValues /* An array of output value pointers. Array must be allocated by the caller */ ); try { NativeApiStatus.VerifySuccess(status); } catch (OnnxRuntimeException e) { // clean up the individual output tensors if it is not null; OrtReleaseValues(outputValues, outputCount); throw e; } finally { // always unpin the input buffers, and delete the native Onnx value objects for (int i = 0; i < inputs.Count; i++) { NativeMethods.OrtReleaseValue(inputTensors[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object. // For string tensors, this releases the native memory allocated for the tensor, including the buffer pinnedBufferHandles[i].Dispose(); pinnedBufferHandles[i] = default; } } return(outputCount); }
/// <summary> /// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames". Uses the given RunOptions for this run./>. /// </summary> /// <param name="inputs"></param> /// <param name="outputNames"></param> /// <param name="options"></param> /// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns> public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options) { var inputNames = new string[inputs.Count]; var inputTensors = new IntPtr[inputs.Count]; var pinnedBufferHandles = new System.Buffers.MemoryHandle[inputs.Count]; int inputIndex = 0; foreach (var input in inputs) { inputNames[inputIndex] = input.Name; // create Tensor from the input if feasible, else throw notsupported exception for now input.ToNativeOnnxValue(out inputTensors[inputIndex], out pinnedBufferHandles[inputIndex]); inputIndex++; } string[] outputNamesArray = outputNames.ToArray(); IntPtr[] outputValueArray = new IntPtr[outputNames.Count]; IntPtr status = NativeMethods.OrtRun( this._nativeHandle, options.Handle, inputNames, inputTensors, (UIntPtr)(inputTensors.Length), outputNamesArray, (UIntPtr)outputNames.Count, outputValueArray /* An array of output value pointers. Array must be allocated by the caller */ ); try { NativeApiStatus.VerifySuccess(status); var result = new DisposableList <DisposableNamedOnnxValue>(); for (uint i = 0; i < outputValueArray.Length; i++) { result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i])); } return(result); } catch (OnnxRuntimeException e) { //clean up the individual output tensors if it is not null; for (uint i = 0; i < outputValueArray.Length; i++) { if (outputValueArray[i] != IntPtr.Zero) { NativeMethods.OrtReleaseValue(outputValueArray[i]); } } throw e; } finally { inputIndex = 0; foreach (var input in inputs) { // For NamedOnnxValue, always unpin the input buffers, and delete the native Onnx value objects // For DisposableNamedOnnxValue, the user needs to do this by invoking Dispose if (input.GetType() == typeof(NamedOnnxValue)) { NativeMethods.OrtReleaseValue(inputTensors[inputIndex]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object. // For string tensors, this releases the native memory allocated for the tensor, including the buffer pinnedBufferHandles[inputIndex].Dispose(); } inputIndex++; } } }