/// <summary>
/// This function will bind model output with the given name to a device
/// specified by the memInfo.
/// </summary>
/// <param name="name">output name</param>
/// <param name="memInfo">instance of memory info</param>
public void BindOutputToDevice(string name, OrtMemoryInfo memInfo)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutputToDevice(handle, pinnedName.Pointer, memInfo.Pointer));
}
private static OrtValue CreateStringTensor(Tensor <string> tensor)
{
if (tensor == null)
{
throw new OnnxRuntimeException(ErrorCode.Fail, "Cast to Tensor<string> failed. BUG check!");
}
int totalLength = 0;
for (int i = 0; i < tensor.Length; i++)
{
totalLength += System.Text.Encoding.UTF8.GetByteCount(tensor.GetValue(i));
}
long[] shape = new long[tensor.Dimensions.Length];
for (int i = 0; i < tensor.Dimensions.Length; i++)
{
shape[i] = tensor.Dimensions[i];
}
// allocate the native tensor
IntPtr valueHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue(
OrtAllocator.DefaultInstance.Pointer,
shape,
(UIntPtr)(shape.Length),
TensorElementType.String,
out valueHandle
));
var ortValue = new OrtValue(valueHandle);
try
{
// fill the native tensor, using GetValue(index) from the Tensor<string>
var len = tensor.Length;
var nativeStrings = new IntPtr[len];
using (var pinnedHandles = new DisposableList <PinnedGCHandle>((int)len))
{
for (int i = 0; i < len; i++)
{
var utf8str = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(tensor.GetValue(i));
var gcHandle = GCHandle.Alloc(utf8str, GCHandleType.Pinned);
nativeStrings[i] = gcHandle.AddrOfPinnedObject();
pinnedHandles.Add(new PinnedGCHandle(gcHandle));
}
using (var pinnedStrings = new PinnedGCHandle(GCHandle.Alloc(nativeStrings, GCHandleType.Pinned)))
NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(ortValue.Handle, nativeStrings, (UIntPtr)len));
}
}
catch (OnnxRuntimeException)
{
ortValue.Dispose();
throw;
}
return(ortValue);
}
/// <summary>
/// Override symbolic dimensions (by specific name strings) with actual values if known at session initialization time to enable
/// optimizations that can take advantage of fixed values (such as memory planning, etc)
/// </summary>
public void AddFreeDimensionOverrideByName(string dimName, long dimValue)
{
var utf8DimNamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(dimName), GCHandleType.Pinned);
using (var pinnedDimName = new PinnedGCHandle(utf8DimNamePinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddFreeDimensionOverrideByName(handle, pinnedDimName.Pointer, dimValue));
}
}
/// <summary>
/// Add a pre-allocated initializer to a session. If a model contains an initializer with a name
/// that is same as the name passed to this API call, ORT will use this initializer instance
/// instead of deserializing one from the model file. This is useful when you want to share
/// the same initializer across sessions.
/// \param name name of the initializer
/// \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
/// managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
/// to which it is added.
/// </summary>
public void AddInitializer(string name, OrtValue ortValue)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddInitializer(handle, pinnedName.Pointer, ortValue.Handle));
}
}
/// <summary>
/// Loads a DLL named 'libraryPath' and looks for this entry point:
/// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api);
/// It then passes in the provided session options to this function along with the api base.
/// The handle to the loaded library is returned in 'libraryHandle'.
/// It can be unloaded by the caller after all sessions using the passed in
/// session options are destroyed, or if an error occurs and it is non null.
/// Hint: .NET Core 3.1 has a 'NativeLibrary' class that can be used to free the library handle
/// </summary>
public void RegisterCustomOpLibraryV2(string libraryPath, out IntPtr libraryHandle)
{
var libraryPathPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath), GCHandleType.Pinned);
using (var pinnedlibraryPath = new PinnedGCHandle(libraryPathPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, pinnedlibraryPath.Pointer, out libraryHandle));
}
}
/// <summary>
/// Set a single session configuration entry as a pair of strings
/// If a configuration with same key exists, this will overwrite the configuration with the given configValue
/// </summary>
/// <param name="configKey">config key name</param>
/// <param name="configValue">config key value</param>
public void AddSessionConfigEntry(string configKey, string configValue)
{
using (var pinnedConfigKeyName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configKey), GCHandleType.Pinned)))
using (var pinnedConfigValueName = new PinnedGCHandle(GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(configValue), GCHandleType.Pinned)))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtAddSessionConfigEntry(handle,
pinnedConfigKeyName.Pointer, pinnedConfigValueName.Pointer));
}
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="settings">string with Nuphar specific settings</param>
public void AppendExecutionProvider_Nuphar(string settings = "")
{
var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(handle, 1, pinnedSettingsName.Pointer));
}
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="deviceId">device identification, default empty string</param>
public void AppendExecutionProvider_OpenVINO(string deviceId = "")
{
var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);
using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
}
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="settings">string with TVM specific settings</param>
public void AppendExecutionProvider_Tvm(string settings = "")
{
#if __MOBILE__
throw new NotSupportedException("The TVM Execution Provider is not supported in this build");
#else
var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tvm(handle, pinnedSettingsName.Pointer));
}
#endif
}
/// <summary>
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="deviceId">device identification, default empty string</param>
public void AppendExecutionProvider_OpenVINO(string deviceId = "")
{
#if __MOBILE__
throw new NotSupportedException("The OpenVINO Execution Provider is not supported in this build");
#else
var deviceIdPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(deviceId), GCHandleType.Pinned);
using (var pinnedDeviceIdName = new PinnedGCHandle(deviceIdPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_OpenVINO(handle, pinnedDeviceIdName.Pointer));
}
#endif
}
/// <summary>
/// Create an instance of OrtMemoryInfo according to the specification
/// Memory info instances are usually used to get a handle of a native allocator
/// that is present within the current inference session object. That, in turn, depends
/// of what execution providers are available within the binary that you are using and are
/// registered with Add methods.
/// </summary>
/// <param name="utf8AllocatorName">Allocator name. Use of the predefined above.</param>
/// <param name="allocatorType">Allocator type</param>
/// <param name="deviceId">Device id</param>
/// <param name="memoryType">Memory type</param>
public OrtMemoryInfo(byte[] utf8AllocatorName, OrtAllocatorType allocatorType, int deviceId, OrtMemType memoryType)
{
using (var pinnedName = new PinnedGCHandle(GCHandle.Alloc(utf8AllocatorName, GCHandleType.Pinned)))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateMemoryInfo(pinnedName.Pointer,
allocatorType,
deviceId,
memoryType,
out _pointer));
}
_owned = true;
}
/// <summary>
/// A helper method to construct a SessionOptions object for Nuphar execution.
/// Use only if you have the onnxruntime package specific to this Execution Provider.
/// </summary>
/// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param>
/// <returns>A SessionsOptions() object configured for execution with Nuphar</returns>
public static SessionOptions MakeSessionOptionWithNupharProvider(String settings = "")
{
SessionOptions options = new SessionOptions();
var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(options.Handle, 1, pinnedSettingsName.Pointer));
}
return(options);
}
/// <summary>
/// Run helper
/// </summary>
/// <param name="names">names to convert to zero terminated utf8 and pin</param>
/// <param name="cleanupList">list to add pinned memory to for later disposal</param>
/// <returns></returns>
private IntPtr[] ConvertNamesToUtf8 <T>(IReadOnlyCollection <T> inputs, NameExtractor <T> extractor,
DisposableList <IDisposable> cleanupList)
{
var result = new IntPtr[inputs.Count];
for (int i = 0; i < inputs.Count; ++i)
{
var name = extractor(inputs.ElementAt(i));
var utf8Name = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name);
var pinnedHandle = new PinnedGCHandle(GCHandle.Alloc(utf8Name, GCHandleType.Pinned));
result[i] = pinnedHandle.Pointer;
cleanupList.Add(pinnedHandle);
}
return(result);
}
/// <summary>
/// Internal helper
/// </summary>
/// <param name="name"></param>
/// <param name="ortValue"></param>
/// <param name="isInput"></param>
private void BindInputOrOutput(string name, IntPtr ortValue, bool isInput)
{
var utf8NamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name), GCHandleType.Pinned);
using (var pinnedName = new PinnedGCHandle(utf8NamePinned))
{
if (isInput)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtBindInput(handle, pinnedName.Pointer, ortValue));
}
else
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtBindOutput(handle, pinnedName.Pointer, ortValue));
}
}
}