private void TestCUDAProviderOptions() { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); using (var cleanUp = new DisposableListTest <IDisposable>()) { var cudaProviderOptions = new OrtCUDAProviderOptions(); cleanUp.Add(cudaProviderOptions); var providerOptionsDict = new Dictionary <string, string>(); providerOptionsDict["device_id"] = "0"; providerOptionsDict["gpu_mem_limit"] = "20971520"; providerOptionsDict["arena_extend_strategy"] = "kSameAsRequested"; providerOptionsDict["cudnn_conv_algo_search"] = "DEFAULT"; providerOptionsDict["do_copy_in_default_stream"] = "1"; providerOptionsDict["cudnn_conv_use_max_workspace"] = "1"; providerOptionsDict["cudnn_conv1d_pad_to_nc1d"] = "1"; cudaProviderOptions.UpdateOptions(providerOptionsDict); var resultProviderOptionsDict = new Dictionary <string, string>(); ProviderOptionsValueHelper.StringToDict(cudaProviderOptions.GetOptions(), resultProviderOptionsDict); // test provider options configuration string value; value = resultProviderOptionsDict["device_id"]; Assert.Equal("0", value); value = resultProviderOptionsDict["gpu_mem_limit"]; Assert.Equal("20971520", value); value = resultProviderOptionsDict["arena_extend_strategy"]; Assert.Equal("kSameAsRequested", value); value = resultProviderOptionsDict["cudnn_conv_algo_search"]; Assert.Equal("DEFAULT", value); value = resultProviderOptionsDict["do_copy_in_default_stream"]; Assert.Equal("1", value); value = resultProviderOptionsDict["cudnn_conv_use_max_workspace"]; Assert.Equal("1", value); value = resultProviderOptionsDict["cudnn_conv1d_pad_to_nc1d"]; Assert.Equal("1", value); // test correctness of provider options SessionOptions options = SessionOptions.MakeSessionOptionWithCudaProvider(cudaProviderOptions); cleanUp.Add(options); var session = new InferenceSession(modelPath, options); cleanUp.Add(session); var inputMeta = session.InputMetadata; var container = new List <NamedOnnxValue>(); float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model foreach (var name in inputMeta.Keys) { Assert.Equal(typeof(float), inputMeta[name].ElementType); Assert.True(inputMeta[name].IsTensor); var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions); container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor)); } session.Run(container); } }
private void CanRunInferenceOnAModelWithTensorRT() { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); using (var cleanUp = new DisposableListTest <IDisposable>()) { SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(0); cleanUp.Add(options); var session = new InferenceSession(modelPath, options); cleanUp.Add(session); var inputMeta = session.InputMetadata; var container = new List <NamedOnnxValue>(); float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model foreach (var name in inputMeta.Keys) { Assert.Equal(typeof(float), inputMeta[name].ElementType); Assert.True(inputMeta[name].IsTensor); var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions); container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor)); } using (var results = session.Run(container)) { ValidateRunResults(results); } } }
private void TestTensorRTProviderOptions() { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); string calTablePath = "squeezenet_calibration.flatbuffers"; string enginePath = "./"; string engineDecrptLibPath = "engine_decryp"; using (var cleanUp = new DisposableListTest <IDisposable>()) { var trtProviderOptions = new OrtTensorRTProviderOptions(); cleanUp.Add(trtProviderOptions); var providerOptionsDict = new Dictionary <string, string>(); providerOptionsDict["device_id"] = "0"; providerOptionsDict["trt_fp16_enable"] = "1"; providerOptionsDict["trt_int8_enable"] = "1"; providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath; providerOptionsDict["trt_engine_cache_enable"] = "1"; providerOptionsDict["trt_engine_cache_path"] = enginePath; providerOptionsDict["trt_engine_decryption_enable"] = "0"; providerOptionsDict["trt_engine_decryption_lib_path"] = engineDecrptLibPath; trtProviderOptions.UpdateOptions(providerOptionsDict); var resultProviderOptionsDict = new Dictionary <string, string>(); ProviderOptionsValueHelper.StringToDict(trtProviderOptions.GetOptions(), resultProviderOptionsDict); // test provider options configuration string value; value = resultProviderOptionsDict["device_id"]; Assert.Equal("0", value); value = resultProviderOptionsDict["trt_fp16_enable"]; Assert.Equal("1", value); value = resultProviderOptionsDict["trt_int8_enable"]; Assert.Equal("1", value); value = resultProviderOptionsDict["trt_int8_calibration_table_name"]; Assert.Equal(calTablePath, value); value = resultProviderOptionsDict["trt_engine_cache_enable"]; Assert.Equal("1", value); value = resultProviderOptionsDict["trt_engine_cache_path"]; Assert.Equal(enginePath, value); value = resultProviderOptionsDict["trt_engine_decryption_enable"]; Assert.Equal("0", value); value = resultProviderOptionsDict["trt_engine_decryption_lib_path"]; Assert.Equal(engineDecrptLibPath, value); // test correctness of provider options SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(trtProviderOptions); cleanUp.Add(options); var session = new InferenceSession(modelPath, options); cleanUp.Add(session); var inputMeta = session.InputMetadata; var container = new List <NamedOnnxValue>(); float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model foreach (var name in inputMeta.Keys) { Assert.Equal(typeof(float), inputMeta[name].ElementType); Assert.True(inputMeta[name].IsTensor); var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions); container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor)); } session.Run(container); } }
public void TestIOBindingWithOrtAllocation() { var inputName = "data_0"; var outputName = "softmaxout_1"; var allocator = OrtAllocator.DefaultInstance; // From the model using (var dispList = new DisposableListTest <IDisposable>()) { var tuple = OpenSessionSqueezeNet(); var session = tuple.Item1; var inputData = tuple.Item2; var inputTensor = tuple.Item3; var outputData = tuple.Item4; dispList.Add(session); var runOptions = new RunOptions(); dispList.Add(runOptions); var inputMeta = session.InputMetadata; var outputMeta = session.OutputMetadata; var outputTensor = new DenseTensor <float>(outputData, outputMeta[outputName].Dimensions); var ioBinding = session.CreateIoBinding(); dispList.Add(ioBinding); var ortAllocationInput = allocator.Allocate((uint)inputData.Length * sizeof(float)); dispList.Add(ortAllocationInput); var inputShape = Array.ConvertAll <int, long>(inputMeta[inputName].Dimensions, d => d); var shapeSize = ArrayUtilities.GetSizeForShape(inputShape); Assert.Equal(shapeSize, inputData.Length); PopulateNativeBufferFloat(ortAllocationInput, inputData); // Create an external allocation for testing OrtExternalAllocation var cpuMemInfo = OrtMemoryInfo.DefaultInstance; var sizeInBytes = shapeSize * sizeof(float); IntPtr allocPtr = Marshal.AllocHGlobal((int)sizeInBytes); dispList.Add(new OrtSafeMemoryHandle(allocPtr)); PopulateNativeBuffer(allocPtr, inputData); var ortAllocationOutput = allocator.Allocate((uint)outputData.Length * sizeof(float)); dispList.Add(ortAllocationOutput); var outputShape = Array.ConvertAll <int, long>(outputMeta[outputName].Dimensions, i => i); // Test 1. Bind the output to OrtAllocated buffer using (FixedBufferOnnxValue fixedInputBuffer = FixedBufferOnnxValue.CreateFromTensor(inputTensor)) { ioBinding.BindInput(inputName, fixedInputBuffer); ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput); ioBinding.SynchronizeBoundInputs(); using (var outputs = session.RunWithBindingAndNames(runOptions, ioBinding)) { ioBinding.SynchronizeBoundOutputs(); Assert.Equal(1, outputs.Count); var output = outputs.ElementAt(0); Assert.Equal(outputName, output.Name); var tensor = output.AsTensor <float>(); Assert.True(tensor.IsFixedSize); Assert.Equal(outputData, tensor.ToArray <float>(), new FloatComparer()); } } // Test 2. Bind the input to memory allocation and output to a fixedBuffer { ioBinding.BindInput(inputName, Tensors.TensorElementType.Float, inputShape, ortAllocationInput); ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput); ioBinding.SynchronizeBoundInputs(); using (var outputs = session.RunWithBindingAndNames(runOptions, ioBinding)) { ioBinding.SynchronizeBoundOutputs(); Assert.Equal(1, outputs.Count); var output = outputs.ElementAt(0); Assert.Equal(outputName, output.Name); var tensor = output.AsTensor <float>(); Assert.True(tensor.IsFixedSize); Assert.Equal(outputData, tensor.ToArray <float>(), new FloatComparer()); } } // 3. Test external allocation { var externalInputAllocation = new OrtExternalAllocation(cpuMemInfo, inputShape, Tensors.TensorElementType.Float, allocPtr, sizeInBytes); ioBinding.BindInput(inputName, externalInputAllocation); ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput); ioBinding.SynchronizeBoundInputs(); using (var outputs = session.RunWithBindingAndNames(runOptions, ioBinding)) { ioBinding.SynchronizeBoundOutputs(); Assert.Equal(1, outputs.Count); var output = outputs.ElementAt(0); Assert.Equal(outputName, output.Name); var tensor = output.AsTensor <float>(); Assert.True(tensor.IsFixedSize); Assert.Equal(outputData, tensor.ToArray <float>(), new FloatComparer()); } } // 4. Some negative tests for external allocation { // Small buffer size Action smallBuffer = delegate() { new OrtExternalAllocation(cpuMemInfo, inputShape, Tensors.TensorElementType.Float, allocPtr, sizeInBytes - 10); }; Assert.Throws <OnnxRuntimeException>(smallBuffer); Action stringType = delegate() { new OrtExternalAllocation(cpuMemInfo, inputShape, Tensors.TensorElementType.String, allocPtr, sizeInBytes); }; Assert.Throws <OnnxRuntimeException>(stringType); } } }
public void TestIOBindingWithOrtAllocation() { var inputName = "data_0"; var outputName = "softmaxout_1"; var allocator = OrtAllocator.DefaultInstance; // From the model using (var dispList = new DisposableListTest <IDisposable>()) { var tuple = OpenSessionSqueezeNet(); var session = tuple.Item1; var inputData = tuple.Item2; var inputTensor = tuple.Item3; var outputData = tuple.Item4; dispList.Add(session); var inputMeta = session.InputMetadata; var outputMeta = session.OutputMetadata; var outputTensor = new DenseTensor <float>(outputData, outputMeta[outputName].Dimensions); var ioBinding = session.CreateIoBinding(); dispList.Add(ioBinding); var ortAllocationInput = allocator.Allocate((uint)inputData.Length * sizeof(float)); dispList.Add(ortAllocationInput); var inputShape = Array.ConvertAll <int, long>(inputMeta[inputName].Dimensions, d => d); PopulateNativeBufferFloat(ortAllocationInput, inputData); var ortAllocationOutput = allocator.Allocate((uint)outputData.Length * sizeof(float)); dispList.Add(ortAllocationOutput); var outputShape = Array.ConvertAll <int, long>(outputMeta[outputName].Dimensions, i => i); // Test 1. Bind the output to OrtAllocated buffer using (FixedBufferOnnxValue fixedInputBuffer = FixedBufferOnnxValue.CreateFromTensor(inputTensor)) { ioBinding.BindInput(inputName, fixedInputBuffer); ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput); using (var outputs = session.RunWithBindingAndNames(new RunOptions(), ioBinding)) { Assert.Equal(1, outputs.Count); var output = outputs.ElementAt(0); Assert.Equal(outputName, output.Name); var tensor = output.AsTensor <float>(); Assert.True(tensor.IsFixedSize); Assert.Equal(outputData, tensor.ToArray <float>(), new floatComparer()); } } // Test 2. Bind the input to memory allocation and output to a fixedBuffer { ioBinding.BindInput(inputName, Tensors.TensorElementType.Float, inputShape, ortAllocationInput); ioBinding.BindOutput(outputName, Tensors.TensorElementType.Float, outputShape, ortAllocationOutput); using (var outputs = session.RunWithBindingAndNames(new RunOptions(), ioBinding)) { Assert.Equal(1, outputs.Count); var output = outputs.ElementAt(0); Assert.Equal(outputName, output.Name); var tensor = output.AsTensor <float>(); Assert.True(tensor.IsFixedSize); Assert.Equal(outputData, tensor.ToArray <float>(), new floatComparer()); } } } }