/// <summary> /// This fetches bound outputs after running the model with RunWithBinding() /// </summary> /// <returns>IDisposableReadOnlyCollection<OrtValue></returns> public IDisposableReadOnlyCollection <OrtValue> GetOutputValues() { IntPtr ortValues = IntPtr.Zero; UIntPtr count = UIntPtr.Zero; var allocator = OrtAllocator.DefaultInstance; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetBoundOutputValues(handle, allocator.Pointer, out ortValues, out count)); if (count.Equals(UIntPtr.Zero)) { return(new DisposableList <OrtValue>()); } using (var ortValuesAllocation = new OrtMemoryAllocation(allocator, ortValues, 0)) { int outputCount = (int)count; var ortList = new DisposableList <OrtValue>(outputCount); try { for (int i = 0; i < outputCount; ++i) { IntPtr ortValue = Marshal.ReadIntPtr(ortValues, IntPtr.Size * i); ortList.Add(new OrtValue(ortValue)); } } catch (Exception e) { ortList.Dispose(); throw e; } return(ortList); } }
/// <summary> /// Bind allocation obtained from an Ort allocator /// </summary> /// <param name="name">name </param> /// <param name="elementType">data type</param> /// <param name="shape">tensor shape</param> /// <param name="allocation">ort allocation</param> /// <param name="isInput">whether this is input or output</param> private void BindOrtAllocation(string name, Tensors.TensorElementType elementType, long[] shape, OrtMemoryAllocation allocation, bool isInput) { using (var ortValue = OrtValue.CreateTensorValueWithData(allocation.Info, elementType, shape, allocation.Pointer, allocation.Size)) BindInputOrOutput(name, ortValue.Handle, isInput); }
/// <summary> /// Get TensorRT EP provider options /// </summary> /// <returns> return C# UTF-16 encoded string </returns> public string GetOptions() { var allocator = OrtAllocator.DefaultInstance; // Process provider options string IntPtr providerOptions = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorRTProviderOptionsAsString(handle, allocator.Pointer, out providerOptions)); using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0)) { return(NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions)); } }
/// <summary> /// Ends profiling for the session. Returns the profile file name. /// public string EndProfiling() { IntPtr nameHandle = IntPtr.Zero; var allocator = OrtAllocator.DefaultInstance; NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionEndProfiling(_nativeHandle, allocator.Pointer, out nameHandle)); using (var allocation = new OrtMemoryAllocation(allocator, nameHandle, 0)) { return(NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle)); } }
private string GetOverridableInitializerName(ulong index) { string str = null; var allocator = OrtAllocator.DefaultInstance; IntPtr nameHandle = IntPtr.Zero; NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOverridableInitializerName( _nativeHandle, (UIntPtr)index, allocator.Pointer, out nameHandle)); using (var ortAllocation = new OrtMemoryAllocation(allocator, nameHandle, 0)) { str = NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle); } return(str); }
/// <summary> /// Returns an array of output names in the same order they were bound /// </summary> /// <returns>array of output names</returns> public string[] GetOutputNames() { IntPtr buffer = IntPtr.Zero; IntPtr lengths = IntPtr.Zero; UIntPtr count = UIntPtr.Zero; var allocator = OrtAllocator.DefaultInstance; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetBoundOutputNames(handle, allocator.Pointer, out buffer, out lengths, out count)); if (count.Equals(UIntPtr.Zero)) { return(new string[0]); } using (var bufferAllocation = new OrtMemoryAllocation(allocator, buffer, 0)) using (var lengthsAllocation = new OrtMemoryAllocation(allocator, lengths, 0)) { int outputCount = (int)count; var lens = new int[outputCount]; int totalLength = 0; for (int i = 0; i < outputCount; ++i) { var len = (int)Marshal.ReadIntPtr(lengths, IntPtr.Size * i); lens[i] = len; totalLength += len; } var stringData = new byte[totalLength]; Marshal.Copy(buffer, stringData, 0, stringData.Length); string[] result = new string[outputCount]; int readOffset = 0; for (int i = 0; i < outputCount; ++i) { var strLen = lens[i]; result[i] = Encoding.UTF8.GetString(stringData, readOffset, strLen); readOffset += strLen; } return(result); } }
/// <summary> /// Bind a piece of pre-allocated native memory as a OrtValue Tensor with a given shape /// to an input with a given name. The model will read the specified input from that memory /// possibly avoiding the need to copy between devices. OrtMemoryAllocation continues to own /// the chunk of native memory, and the allocation should be alive until the end of execution. /// </summary> /// <param name="name">of the input</param> /// <param name="elementType">Tensor element type</param> /// <param name="shape"></param> /// <param name="allocation">native memory allocation</param> public void BindInput(string name, Tensors.TensorElementType elementType, long[] shape, OrtMemoryAllocation allocation) { BindOrtAllocation(name, elementType, shape, allocation, true); }