private void CanRunInferenceOnAModelWithTensorRT() { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); using (var cleanUp = new DisposableListTest <IDisposable>()) { SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(0); cleanUp.Add(options); var session = new InferenceSession(modelPath, options); cleanUp.Add(session); var inputMeta = session.InputMetadata; var container = new List <NamedOnnxValue>(); float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model foreach (var name in inputMeta.Keys) { Assert.Equal(typeof(float), inputMeta[name].ElementType); Assert.True(inputMeta[name].IsTensor); var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions); container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor)); } using (var results = session.Run(container)) { ValidateRunResults(results); } } }
private void TestTensorRTProviderOptions() { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); string calTablePath = "squeezenet_calibration.flatbuffers"; string enginePath = "./"; string engineDecrptLibPath = "engine_decryp"; using (var cleanUp = new DisposableListTest <IDisposable>()) { var trtProviderOptions = new OrtTensorRTProviderOptions(); cleanUp.Add(trtProviderOptions); var providerOptionsDict = new Dictionary <string, string>(); providerOptionsDict["device_id"] = "0"; providerOptionsDict["trt_fp16_enable"] = "1"; providerOptionsDict["trt_int8_enable"] = "1"; providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath; providerOptionsDict["trt_engine_cache_enable"] = "1"; providerOptionsDict["trt_engine_cache_path"] = enginePath; providerOptionsDict["trt_engine_decryption_enable"] = "0"; providerOptionsDict["trt_engine_decryption_lib_path"] = engineDecrptLibPath; trtProviderOptions.UpdateOptions(providerOptionsDict); var resultProviderOptionsDict = new Dictionary <string, string>(); ProviderOptionsValueHelper.StringToDict(trtProviderOptions.GetOptions(), resultProviderOptionsDict); // test provider options configuration string value; value = resultProviderOptionsDict["device_id"]; Assert.Equal("0", value); value = resultProviderOptionsDict["trt_fp16_enable"]; Assert.Equal("1", value); value = resultProviderOptionsDict["trt_int8_enable"]; Assert.Equal("1", value); value = resultProviderOptionsDict["trt_int8_calibration_table_name"]; Assert.Equal(calTablePath, value); value = resultProviderOptionsDict["trt_engine_cache_enable"]; Assert.Equal("1", value); value = resultProviderOptionsDict["trt_engine_cache_path"]; Assert.Equal(enginePath, value); value = resultProviderOptionsDict["trt_engine_decryption_enable"]; Assert.Equal("0", value); value = resultProviderOptionsDict["trt_engine_decryption_lib_path"]; Assert.Equal(engineDecrptLibPath, value); // test correctness of provider options SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(trtProviderOptions); cleanUp.Add(options); var session = new InferenceSession(modelPath, options); cleanUp.Add(session); var inputMeta = session.InputMetadata; var container = new List <NamedOnnxValue>(); float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model foreach (var name in inputMeta.Keys) { Assert.Equal(typeof(float), inputMeta[name].ElementType); Assert.True(inputMeta[name].IsTensor); var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions); container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor)); } session.Run(container); } }