public void TestIOBindingWithOrtAllocation()
{
var inputName = "data_0";
var outputName = "softmaxout_1";
var allocator = OrtAllocator.DefaultInstance;
// From the model
using (var dispList = new DisposableListTest <IDisposable>())
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var inputTensor = tuple.Item3;
var outputData = tuple.Item4;
dispList.Add(session);
var runOptions = new RunOptions();
dispList.Add(runOptions);
var inputMeta = session.InputMetadata;
var outputMeta = session.OutputMetadata;
var outputTensor = new DenseTensor <float>(outputData, outputMeta[outputName].Dimensions);
var ioBinding = session.CreateIoBinding();
dispList.Add(ioBinding);
var ortAllocationInput = allocator.Allocate((uint)inputData.Length * sizeof(float));
dispList.Add(ortAllocationInput);
var inputShape = Array.ConvertAll <int, long>(inputMeta[inputName].Dimensions, d => d);
var shapeSize = ArrayUtilities.GetSizeForShape(inputShape);
Assert.Equal(shapeSize, inputData.Length);
PopulateNativeBufferFloat(ortAllocationInput, inputData);
// Create an external allocation for testing OrtExternalAllocation
var cpuMemInfo = OrtMemoryInfo.DefaultInstance;
var sizeInBytes = shapeSize * sizeof(float);
IntPtr allocPtr = Marshal.AllocHGlobal((int)sizeInBytes);
dispList.Add(new OrtSafeMemoryHandle(allocPtr));
PopulateNativeBuffer(allocPtr, inputData);
var ortAllocationOutput = allocator.Allocate((uint)outputData.Length * sizeof(float));
dispList.Add(ortAllocationOutput);
var outputShape = Array.ConvertAll <int, long>(outputMeta[outputName].Dimensions, i => i);
// Test 1. Bind the output to OrtAllocated buffer
using (FixedBufferOnnxValue fixedInputBuffer = FixedBufferOnnxValue.CreateFromTensor(inputTensor))
{
ioBinding.BindInput(inputName, fixedInputBuffer);
ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput);
ioBinding.SynchronizeBoundInputs();
using (var outputs = session.RunWithBindingAndNames(runOptions, ioBinding))
{
ioBinding.SynchronizeBoundOutputs();
Assert.Equal(1, outputs.Count);
var output = outputs.ElementAt(0);
Assert.Equal(outputName, output.Name);
var tensor = output.AsTensor <float>();
Assert.True(tensor.IsFixedSize);
Assert.Equal(outputData, tensor.ToArray <float>(), new FloatComparer());
}
}
// Test 2. Bind the input to memory allocation and output to a fixedBuffer
{
ioBinding.BindInput(inputName, Tensors.TensorElementType.Float, inputShape, ortAllocationInput);
ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput);
ioBinding.SynchronizeBoundInputs();
using (var outputs = session.RunWithBindingAndNames(runOptions, ioBinding))
{
ioBinding.SynchronizeBoundOutputs();
Assert.Equal(1, outputs.Count);
var output = outputs.ElementAt(0);
Assert.Equal(outputName, output.Name);
var tensor = output.AsTensor <float>();
Assert.True(tensor.IsFixedSize);
Assert.Equal(outputData, tensor.ToArray <float>(), new FloatComparer());
}
}
// 3. Test external allocation
{
var externalInputAllocation = new OrtExternalAllocation(cpuMemInfo, inputShape,
Tensors.TensorElementType.Float, allocPtr, sizeInBytes);
ioBinding.BindInput(inputName, externalInputAllocation);
ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput);
ioBinding.SynchronizeBoundInputs();
using (var outputs = session.RunWithBindingAndNames(runOptions, ioBinding))
{
ioBinding.SynchronizeBoundOutputs();
Assert.Equal(1, outputs.Count);
var output = outputs.ElementAt(0);
Assert.Equal(outputName, output.Name);
var tensor = output.AsTensor <float>();
Assert.True(tensor.IsFixedSize);
Assert.Equal(outputData, tensor.ToArray <float>(), new FloatComparer());
}
}
// 4. Some negative tests for external allocation
{
// Small buffer size
Action smallBuffer = delegate()
{
new OrtExternalAllocation(cpuMemInfo, inputShape,
Tensors.TensorElementType.Float, allocPtr, sizeInBytes - 10);
};
Assert.Throws <OnnxRuntimeException>(smallBuffer);
Action stringType = delegate()
{
new OrtExternalAllocation(cpuMemInfo, inputShape,
Tensors.TensorElementType.String, allocPtr, sizeInBytes);
};
Assert.Throws <OnnxRuntimeException>(stringType);
}
}
}
public void TestIOBindingWithOrtAllocation()
{
var inputName = "data_0";
var outputName = "softmaxout_1";
var allocator = OrtAllocator.DefaultInstance;
// From the model
using (var dispList = new DisposableListTest <IDisposable>())
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var inputTensor = tuple.Item3;
var outputData = tuple.Item4;
dispList.Add(session);
var inputMeta = session.InputMetadata;
var outputMeta = session.OutputMetadata;
var outputTensor = new DenseTensor <float>(outputData, outputMeta[outputName].Dimensions);
var ioBinding = session.CreateIoBinding();
dispList.Add(ioBinding);
var ortAllocationInput = allocator.Allocate((uint)inputData.Length * sizeof(float));
dispList.Add(ortAllocationInput);
var inputShape = Array.ConvertAll <int, long>(inputMeta[inputName].Dimensions, d => d);
PopulateNativeBufferFloat(ortAllocationInput, inputData);
var ortAllocationOutput = allocator.Allocate((uint)outputData.Length * sizeof(float));
dispList.Add(ortAllocationOutput);
var outputShape = Array.ConvertAll <int, long>(outputMeta[outputName].Dimensions, i => i);
// Test 1. Bind the output to OrtAllocated buffer
using (FixedBufferOnnxValue fixedInputBuffer = FixedBufferOnnxValue.CreateFromTensor(inputTensor))
{
ioBinding.BindInput(inputName, fixedInputBuffer);
ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput);
using (var outputs = session.RunWithBindingAndNames(new RunOptions(), ioBinding))
{
Assert.Equal(1, outputs.Count);
var output = outputs.ElementAt(0);
Assert.Equal(outputName, output.Name);
var tensor = output.AsTensor <float>();
Assert.True(tensor.IsFixedSize);
Assert.Equal(outputData, tensor.ToArray <float>(), new floatComparer());
}
}
// Test 2. Bind the input to memory allocation and output to a fixedBuffer
{
ioBinding.BindInput(inputName, Tensors.TensorElementType.Float, inputShape, ortAllocationInput);
ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput);
using (var outputs = session.RunWithBindingAndNames(new RunOptions(), ioBinding))
{
Assert.Equal(1, outputs.Count);
var output = outputs.ElementAt(0);
Assert.Equal(outputName, output.Name);
var tensor = output.AsTensor <float>();
Assert.True(tensor.IsFixedSize);
Assert.Equal(outputData, tensor.ToArray <float>(), new floatComparer());
}
}
}
}
