/// <summary>
/// Initializes the session object with a native session handle
/// </summary>
/// <param name="session">Handle of a native session object</param>
/// <param name="options">Session options</param>
private void InitWithSessionHandle(IntPtr session, SessionOptions options)
{
_nativeHandle = session;
try
{
// Initialize input/output metadata
_inputMetadata = new Dictionary <string, NodeMetadata>();
_outputMetadata = new Dictionary <string, NodeMetadata>();
// get input count
UIntPtr inputCount = UIntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetInputCount(_nativeHandle, out inputCount));
// get all the output names
for (ulong i = 0; i < (ulong)inputCount; i++)
{
var iname = GetInputName(i);
_inputMetadata[iname] = GetInputMetadata(i);
}
// get output count
UIntPtr outputCount = UIntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOutputCount(_nativeHandle, out outputCount));
// get all the output names
for (ulong i = 0; i < (ulong)outputCount; i++)
{
_outputMetadata[GetOutputName(i)] = GetOutputMetadata(i);
}
}
catch (OnnxRuntimeException e)
{
if (_nativeHandle != IntPtr.Zero)
{
NativeMethods.OrtReleaseSession(_nativeHandle);
_nativeHandle = IntPtr.Zero;
}
throw e;
}
_builtInRunOptions = new RunOptions(); // create a default built-in run option, and avoid creating a new one every run() call
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run.
/// </summary>
/// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param>
/// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param>
/// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(
IReadOnlyCollection <string> inputNames,
IReadOnlyCollection <FixedBufferOnnxValue> inputValues,
IReadOnlyCollection <string> outputNames,
RunOptions options)
{
if (inputNames.Count != inputValues.Count)
{
throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count}).");
}
using (var cleanupList = new DisposableList <IDisposable>())
{
var inputNamesArray = ConvertNamesToUtf8(inputNames, n => n, cleanupList);
IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true);
var outputNamesArray = ConvertNamesToUtf8(outputNames, n => n, cleanupList);
var ortValues = RunImpl(options, inputNamesArray, inputValuesArray, outputNamesArray, cleanupList);
return(CreateDisposableResult(ortValues, outputNames));
}
}
/// <summary>
/// Runs the loaded model for the given inputs and outputs. Uses the given RunOptions for this run.
///
/// Outputs need to be created with correct type and dimension to accept the fetched data.
/// </summary>
/// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param>
/// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param>
/// <param name="outputNames">Specify a collection of string that indicates the output names. Should match <paramref name="outputValues"/>.</param>
/// <param name="outputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the output values.</param>
/// <param name="options"></param>
public void Run(
IReadOnlyCollection <string> inputNames,
IReadOnlyCollection <FixedBufferOnnxValue> inputValues,
IReadOnlyCollection <string> outputNames,
IReadOnlyCollection <FixedBufferOnnxValue> outputValues,
RunOptions options)
{
if (inputNames.Count != inputValues.Count)
{
throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count}).");
}
if (outputNames.Count != outputValues.Count)
{
throw new ArgumentException($"Length of {nameof(outputNames)} ({outputNames.Count}) must match that of {nameof(outputValues)} ({outputValues.Count}).");
}
using (var cleanupList = new DisposableList <IDisposable>())
{
// prepare inputs
var inputNamesArray = ConvertNamesToUtf8(inputNames, n => n, cleanupList);
IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true);
// prepare outputs
var outputNamesArray = ConvertNamesToUtf8(outputNames, n => n, cleanupList);
IntPtr[] outputValuesArray = GetOrtValuesHandles(outputValues, false);
NativeApiStatus.VerifySuccess(NativeMethods.OrtRun(
_nativeHandle,
options.Handle,
inputNamesArray,
inputValuesArray,
(UIntPtr)inputNames.Count,
outputNamesArray,
(UIntPtr)outputNames.Count,
outputValuesArray /* pointers to Pre-allocated OrtValue instances */
));
}
}
/// <summary>
/// Runs the loaded model for the given inputs and outputs. Uses the given RunOptions for this run.
///
/// Outputs need to be created with correct type and dimension to receive the fetched data.
/// </summary>
/// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param>
/// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param>
/// <param name="output">Specify a collection of <see cref="NamedOnnxValue"/> that indicates the output values.</param>
/// <param name="options"></param>
public void Run(
IReadOnlyCollection <string> inputNames,
IReadOnlyCollection <FixedBufferOnnxValue> inputValues,
IReadOnlyCollection <NamedOnnxValue> outputs,
RunOptions options)
{
if (inputNames.Count != inputValues.Count)
{
throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count}).");
}
var outputNamesArray = new string[outputs.Count];
var outputValuesArray = new IntPtr[outputs.Count];
var pinnedOutputBufferHandles = new System.Buffers.MemoryHandle[outputs.Count];
var disposeOutputs = new bool[outputs.Count];
try
{
// prepare inputs
string[] inputNamesArray = inputNames as string[] ?? inputNames.ToArray();
IntPtr[] inputValuesArray = new IntPtr[inputNames.Count];
int inputIndex = 0;
foreach (var input in inputValues)
{
inputValuesArray[inputIndex] = input.Value;
inputIndex++;
}
// prepare outputs
int outputIndex = 0;
foreach (var output in outputs)
{
outputNamesArray[outputIndex] = output.Name;
// create native OrtValue from the output if feasible, else throw notsupported exception for now
output.ToNativeOnnxValue(
out outputValuesArray[outputIndex],
out pinnedOutputBufferHandles[outputIndex],
out disposeOutputs[outputIndex]);
outputIndex++;
}
IntPtr status = NativeMethods.OrtRun(
_nativeHandle,
options.Handle,
inputNamesArray,
inputValuesArray,
(UIntPtr)inputNames.Count,
outputNamesArray,
(UIntPtr)outputs.Count,
outputValuesArray /* pointers to Pre-allocated OrtValue instances */
);
NativeApiStatus.VerifySuccess(status);
}
finally
{
for (int i = 0; i < outputs.Count; i++)
{
if (disposeOutputs[i])
{
NativeMethods.OrtReleaseValue(outputValuesArray[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object.
// For string tensors, this releases the native memory allocated for the tensor, including the buffer
pinnedOutputBufferHandles[i].Dispose();
}
}
}
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run.
/// </summary>
/// <param name="inputNames">Specify a collection of string that indicates the input names. Should match <paramref name="inputValues"/>.</param>
/// <param name="inputValues">Specify a collection of <see cref="FixedBufferOnnxValue"/> that indicates the input values.</param>
/// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(
IReadOnlyCollection <string> inputNames,
IReadOnlyCollection <FixedBufferOnnxValue> inputValues,
IReadOnlyCollection <string> outputNames,
RunOptions options)
{
if (inputNames.Count != inputValues.Count)
{
throw new ArgumentException($"Length of {nameof(inputNames)} ({inputNames.Count}) must match that of {nameof(inputValues)} ({inputValues.Count}).");
}
// prepare inputs
string[] inputNamesArray = inputNames as string[] ?? inputNames.ToArray();
IntPtr[] inputValuesArray = new IntPtr[inputNames.Count];
int inputIndex = 0;
foreach (var input in inputValues)
{
inputValuesArray[inputIndex] = input.Value;
inputIndex++;
}
// prepare outputs
string[] outputNamesArray = outputNames as string[] ?? outputNames.ToArray();
IntPtr[] outputValuesArray = new IntPtr[outputNames.Count];
IntPtr status = NativeMethods.OrtRun(
_nativeHandle,
options.Handle,
inputNamesArray,
inputValuesArray,
(UIntPtr)inputNames.Count,
outputNamesArray,
(UIntPtr)outputNames.Count,
outputValuesArray /* Empty array is passed in to receive output OrtValue pointers */
);
try
{
NativeApiStatus.VerifySuccess(status);
var result = new DisposableList <DisposableNamedOnnxValue>(outputValuesArray.Length);
for (int i = 0; i < outputValuesArray.Length; i++)
{
result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValuesArray[i]));
}
return(result);
}
catch (OnnxRuntimeException e)
{
//clean up the individual output tensors if it is not null;
for (uint i = 0; i < outputValuesArray.Length; i++)
{
if (outputValuesArray[i] != IntPtr.Zero)
{
NativeMethods.OrtReleaseValue(outputValuesArray[i]);
}
}
throw e;
}
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run.
/// </summary>
/// <param name="inputs">Specify a collection of <see cref="NamedOnnxValue"/> that indicates the input values.</param>
/// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options)
{
// prepare inputs
var inputNamesArray = new string[inputs.Count];
var inputValuesArray = new IntPtr[inputs.Count];
var pinnedInputBufferHandles = new System.Buffers.MemoryHandle[inputs.Count];
var disposeInputs = new bool[inputs.Count];
int inputIndex = 0;
foreach (var input in inputs)
{
inputNamesArray[inputIndex] = input.Name;
// create Tensor from the input if feasible, else throw notsupported exception for now
input.ToNativeOnnxValue(
out inputValuesArray[inputIndex],
out pinnedInputBufferHandles[inputIndex],
out disposeInputs[inputIndex]);
inputIndex++;
}
// prepare outputs
string[] outputNamesArray = outputNames as string[] ?? outputNames.ToArray();
IntPtr[] outputValuesArray = new IntPtr[outputNames.Count];
IntPtr status = NativeMethods.OrtRun(
_nativeHandle,
options.Handle,
inputNamesArray,
inputValuesArray,
(UIntPtr)inputs.Count,
outputNamesArray,
(UIntPtr)outputNames.Count,
outputValuesArray /* Empty array is passed in to receive output OrtValue pointers */
);
try
{
NativeApiStatus.VerifySuccess(status);
var result = new DisposableList <DisposableNamedOnnxValue>(outputValuesArray.Length);
for (int i = 0; i < outputValuesArray.Length; i++)
{
result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValuesArray[i]));
}
return(result);
}
catch (OnnxRuntimeException e)
{
//clean up the individual output tensors if it is not null;
for (int i = 0; i < outputValuesArray.Length; i++)
{
if (outputValuesArray[i] != IntPtr.Zero)
{
NativeMethods.OrtReleaseValue(outputValuesArray[i]);
}
}
throw e;
}
finally
{
for (int i = 0; i < inputs.Count; i++)
{
if (disposeInputs[i])
{
NativeMethods.OrtReleaseValue(inputValuesArray[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object.
// For string tensors, this releases the native memory allocated for the tensor, including the buffer
pinnedInputBufferHandles[i].Dispose();
}
}
}
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>.
/// </summary>
/// <param name="inputs"></param>
/// <param name="outputNames"></param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Dictionary</returns>
//TODO: kept internal until RunOptions is made public
internal IReadOnlyCollection <NamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options)
{
var inputNames = new string[inputs.Count];
var inputTensors = new IntPtr[inputs.Count];
var pinnedBufferHandles = new System.Buffers.MemoryHandle[inputs.Count];
int offset = 0;
foreach (var input in inputs)
{
inputNames[offset] = input.Name;
// create Tensor from the input if feasible, else throw notsupported exception for now
input.ToNativeOnnxValue(out inputTensors[offset], out pinnedBufferHandles[offset]);
offset++;
}
string[] outputNamesArray = outputNames.ToArray();
IntPtr[] outputValueArray = new IntPtr[outputNames.Count];
IntPtr status = NativeMethods.OrtRunInference(
this._nativeHandle,
IntPtr.Zero, // TODO: use Run options when Run options creation API is available
// Passing null uses the default run options in the C-api
inputNames,
inputTensors,
(ulong)(inputTensors.Length), /* TODO: size_t, make it portable for x86 arm */
outputNamesArray,
(ulong)outputNames.Count, /* TODO: size_t, make it portable for x86 and arm */
outputValueArray /* An array of output value pointers. Array must be allocated by the caller */
);
try
{
NativeApiStatus.VerifySuccess(status);
var result = new List <NamedOnnxValue>();
for (uint i = 0; i < outputValueArray.Length; i++)
{
result.Add(NamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i]));
}
return(result);
}
catch (OnnxRuntimeException e)
{
//clean up the individual output tensors if it is not null;
for (uint i = 0; i < outputValueArray.Length; i++)
{
if (outputValueArray[i] != IntPtr.Zero)
{
NativeMethods.OrtReleaseValue(outputValueArray[i]);
}
}
throw e;
}
finally
{
// always unpin the input buffers, and delete the native Onnx value objects
for (int i = 0; i < inputs.Count; i++)
{
NativeMethods.OrtReleaseValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object
pinnedBufferHandles[i].Dispose();
}
}
}
/// <summary>
/// This method return a collection of DisposableNamedOnnxValue as in other interfaces
/// Query names from OrtIoBinding object and pair then with the array of OrtValues returned
/// from OrtIoBinding.GetOutputValues()
///
/// </summary>
/// <param name="runOptions">RunOptions</param>
/// <param name="ioBinding">OrtIoBinding instance with bindings</param>
/// <param name="names">optional parameter. If you already know the names of the outputs you can save a native
/// call to retrieve output names. They will be paired with the returned OrtValues and combined into DisposbleNamedOnnxValues.
/// Otherwise, the method will retrieve output names from the OrtIoBinding instance.
/// It is an error if you supply a different number of names than the returned outputs</param>
public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> RunWithBindingAndNames(RunOptions runOptions, OrtIoBinding ioBinding, string[] names = null)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunWithBinding(Handle, runOptions.Handle, ioBinding.Handle));
using (var ortValues = ioBinding.GetOutputValues())
{
string[] outputNames = names;
if (outputNames == null)
{
outputNames = ioBinding.GetOutputNames();
}
if (outputNames.Length != ortValues.Count)
{
throw new OnnxRuntimeException(ErrorCode.InvalidArgument,
"Number of specified names: " + names.Length + " does not match the output number: " +
ortValues.Count);
}
var result = new DisposableList <DisposableNamedOnnxValue>(outputNames.Length);
try
{
for (int i = 0; i < outputNames.Length; ++i)
{
var ortValue = ortValues.ElementAt(i);
result.Add(DisposableNamedOnnxValue.CreateTensorFromOnnxValue(outputNames[i], ortValue.Handle));
ortValue.Disown();
}
} catch (Exception e)
{
result.Dispose();
throw e;
}
return(result);
}
}
/// <summary>
/// This method runs inference on the OrtIoBinding instance
/// The method does not return anything. This is a lightweight version of
/// RunWithBindingAndNames(). When you bind pre-allocated buffers to the output values
/// you may not want to fetch the outputs since you already have access to them so you can spare
/// the expense of fetching them and pairing with names.
/// You can still fetch the outputs by calling OrtIOBinding.GetOutputValues()
/// </summary>
/// <param name="runOptions"></param>
/// <param name="ioBinding"></param>
public void RunWithBinding(RunOptions runOptions, OrtIoBinding ioBinding)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunWithBinding(Handle, runOptions.Handle, ioBinding.Handle));
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>. Uses the given RunOptions for this run.
/// </summary>
/// <param name="inputs">Specify a collection of <see cref="NamedOnnxValue"/> that indicates the input values.</param>
/// <param name="outputNames">Specify a collection of string that indicates the output names to fetch.</param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options)
{
using (var cleanupList = new DisposableList <IDisposable>())
{
var inputNamesArray = ConvertNamesToUtf8(inputs, v => v.Name, cleanupList);
var inputValuesArray = GetOrtValuesHandles(inputs, cleanupList);
var outputNamesArray = ConvertNamesToUtf8(outputNames, n => n, cleanupList);
var ortValues = RunImpl(options, inputNamesArray, inputValuesArray, outputNamesArray, cleanupList);
return(CreateDisposableResult(ortValues, outputNames));
}
}
public unsafe int Run(IReadOnlyCollection <NamedOnnxValue> inputs, RunOptions options,
string[] outputNames, IntPtr *outputValues)
{
var inputCount = inputs.Count;
// HACK: Only handling IReadOnlyLists for now!
var inputsList = (IReadOnlyList <NamedOnnxValue>)inputs;
var inputNames = GetInputNames(inputsList);
var inputTensors = stackalloc IntPtr[inputCount];
// Ugly hack to avoid having to create this array
var pinnedBufferHandles =
m_threadLocalMemoryHandles.Value == null ||
m_threadLocalMemoryHandles.Value.Length != inputCount
? new MemoryHandle[inputCount]
: m_threadLocalMemoryHandles.Value;
m_threadLocalMemoryHandles.Value = pinnedBufferHandles;
int inputIndex = 0;
for (int i = 0; i < inputCount; i++)
{
var input = inputsList[i];
inputNames[inputIndex] = input.Name;
// create Tensor from the input if feasible, else throw notsupported exception for now
input.ToNativeOnnxValue(out inputTensors[inputIndex], out pinnedBufferHandles[inputIndex]);
inputIndex++;
}
var outputCount = outputNames.Length;
IntPtr status = NativeMethods.OrtRunFast(
this._nativeHandle,
options.Handle,
inputNames,
inputTensors,
(UIntPtr)(inputCount),
outputNames,
(UIntPtr)outputCount,
outputValues /* An array of output value pointers. Array must be allocated by the caller */
);
try
{
NativeApiStatus.VerifySuccess(status);
}
catch (OnnxRuntimeException e)
{
// clean up the individual output tensors if it is not null;
OrtReleaseValues(outputValues, outputCount);
throw e;
}
finally
{
// always unpin the input buffers, and delete the native Onnx value objects
for (int i = 0; i < inputs.Count; i++)
{
NativeMethods.OrtReleaseValue(inputTensors[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object.
// For string tensors, this releases the native memory allocated for the tensor, including the buffer
pinnedBufferHandles[i].Dispose();
pinnedBufferHandles[i] = default;
}
}
return(outputCount);
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames". Uses the given RunOptions for this run./>.
/// </summary>
/// <param name="inputs"></param>
/// <param name="outputNames"></param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection <DisposableNamedOnnxValue> Run(IReadOnlyCollection <NamedOnnxValue> inputs, IReadOnlyCollection <string> outputNames, RunOptions options)
{
var inputNames = new string[inputs.Count];
var inputTensors = new IntPtr[inputs.Count];
var pinnedBufferHandles = new System.Buffers.MemoryHandle[inputs.Count];
int inputIndex = 0;
foreach (var input in inputs)
{
inputNames[inputIndex] = input.Name;
// create Tensor from the input if feasible, else throw notsupported exception for now
input.ToNativeOnnxValue(out inputTensors[inputIndex],
out pinnedBufferHandles[inputIndex]);
inputIndex++;
}
string[] outputNamesArray = outputNames.ToArray();
IntPtr[] outputValueArray = new IntPtr[outputNames.Count];
IntPtr status = NativeMethods.OrtRun(
this._nativeHandle,
options.Handle,
inputNames,
inputTensors,
(UIntPtr)(inputTensors.Length),
outputNamesArray,
(UIntPtr)outputNames.Count,
outputValueArray /* An array of output value pointers. Array must be allocated by the caller */
);
try
{
NativeApiStatus.VerifySuccess(status);
var result = new DisposableList <DisposableNamedOnnxValue>();
for (uint i = 0; i < outputValueArray.Length; i++)
{
result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i]));
}
return(result);
}
catch (OnnxRuntimeException e)
{
//clean up the individual output tensors if it is not null;
for (uint i = 0; i < outputValueArray.Length; i++)
{
if (outputValueArray[i] != IntPtr.Zero)
{
NativeMethods.OrtReleaseValue(outputValueArray[i]);
}
}
throw e;
}
finally
{
inputIndex = 0;
foreach (var input in inputs)
{
// For NamedOnnxValue, always unpin the input buffers, and delete the native Onnx value objects
// For DisposableNamedOnnxValue, the user needs to do this by invoking Dispose
if (input.GetType() == typeof(NamedOnnxValue))
{
NativeMethods.OrtReleaseValue(inputTensors[inputIndex]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object.
// For string tensors, this releases the native memory allocated for the tensor, including the buffer
pinnedBufferHandles[inputIndex].Dispose();
}
inputIndex++;
}
}
}
