/// <summary>
/// This fetches bound outputs after running the model with RunWithBinding()
/// </summary>
/// <returns>IDisposableReadOnlyCollection<OrtValue></returns>
public IDisposableReadOnlyCollection <OrtValue> GetOutputValues()
{
IntPtr ortValues = IntPtr.Zero;
UIntPtr count = UIntPtr.Zero;
var allocator = OrtAllocator.DefaultInstance;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetBoundOutputValues(handle, allocator.Pointer, out ortValues, out count));
if (count.Equals(UIntPtr.Zero))
{
return(new DisposableList <OrtValue>());
}
using (var ortValuesAllocation = new OrtMemoryAllocation(allocator, ortValues, 0))
{
int outputCount = (int)count;
var ortList = new DisposableList <OrtValue>(outputCount);
try
{
for (int i = 0; i < outputCount; ++i)
{
IntPtr ortValue = Marshal.ReadIntPtr(ortValues, IntPtr.Size * i);
ortList.Add(new OrtValue(ortValue));
}
} catch (Exception e)
{
ortList.Dispose();
throw e;
}
return(ortList);
}
}
/// <summary>
/// Bind allocation obtained from an Ort allocator
/// </summary>
/// <param name="name">name </param>
/// <param name="elementType">data type</param>
/// <param name="shape">tensor shape</param>
/// <param name="allocation">ort allocation</param>
/// <param name="isInput">whether this is input or output</param>
private void BindOrtAllocation(string name, Tensors.TensorElementType elementType, long[] shape,
OrtMemoryAllocation allocation, bool isInput)
{
using (var ortValue = OrtValue.CreateTensorValueWithData(allocation.Info,
elementType,
shape,
allocation.Pointer, allocation.Size))
BindInputOrOutput(name, ortValue.Handle, isInput);
}
/// <summary>
/// Get TensorRT EP provider options
/// </summary>
/// <returns> return C# UTF-16 encoded string </returns>
public string GetOptions()
{
var allocator = OrtAllocator.DefaultInstance;
// Process provider options string
IntPtr providerOptions = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorRTProviderOptionsAsString(handle, allocator.Pointer, out providerOptions));
using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0))
{
return(NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions));
}
}
/// <summary>
/// Ends profiling for the session. Returns the profile file name.
///
public string EndProfiling()
{
IntPtr nameHandle = IntPtr.Zero;
var allocator = OrtAllocator.DefaultInstance;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionEndProfiling(_nativeHandle,
allocator.Pointer,
out nameHandle));
using (var allocation = new OrtMemoryAllocation(allocator, nameHandle, 0))
{
return(NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle));
}
}
private string GetOverridableInitializerName(ulong index)
{
string str = null;
var allocator = OrtAllocator.DefaultInstance;
IntPtr nameHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOverridableInitializerName(
_nativeHandle,
(UIntPtr)index,
allocator.Pointer,
out nameHandle));
using (var ortAllocation = new OrtMemoryAllocation(allocator, nameHandle, 0))
{
str = NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle);
}
return(str);
}
/// <summary>
/// Returns an array of output names in the same order they were bound
/// </summary>
/// <returns>array of output names</returns>
public string[] GetOutputNames()
{
IntPtr buffer = IntPtr.Zero;
IntPtr lengths = IntPtr.Zero;
UIntPtr count = UIntPtr.Zero;
var allocator = OrtAllocator.DefaultInstance;
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetBoundOutputNames(handle, allocator.Pointer, out buffer, out lengths, out count));
if (count.Equals(UIntPtr.Zero))
{
return(new string[0]);
}
using (var bufferAllocation = new OrtMemoryAllocation(allocator, buffer, 0))
using (var lengthsAllocation = new OrtMemoryAllocation(allocator, lengths, 0))
{
int outputCount = (int)count;
var lens = new int[outputCount];
int totalLength = 0;
for (int i = 0; i < outputCount; ++i)
{
var len = (int)Marshal.ReadIntPtr(lengths, IntPtr.Size * i);
lens[i] = len;
totalLength += len;
}
var stringData = new byte[totalLength];
Marshal.Copy(buffer, stringData, 0, stringData.Length);
string[] result = new string[outputCount];
int readOffset = 0;
for (int i = 0; i < outputCount; ++i)
{
var strLen = lens[i];
result[i] = Encoding.UTF8.GetString(stringData, readOffset, strLen);
readOffset += strLen;
}
return(result);
}
}
/// <summary>
/// Bind a piece of pre-allocated native memory as a OrtValue Tensor with a given shape
/// to an input with a given name. The model will read the specified input from that memory
/// possibly avoiding the need to copy between devices. OrtMemoryAllocation continues to own
/// the chunk of native memory, and the allocation should be alive until the end of execution.
/// </summary>
/// <param name="name">of the input</param>
/// <param name="elementType">Tensor element type</param>
/// <param name="shape"></param>
/// <param name="allocation">native memory allocation</param>
public void BindInput(string name, Tensors.TensorElementType elementType, long[] shape, OrtMemoryAllocation allocation)
{
BindOrtAllocation(name, elementType, shape, allocation, true);
}