private void TestCUDAProviderOptions()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
using (var cleanUp = new DisposableListTest <IDisposable>())
{
var cudaProviderOptions = new OrtCUDAProviderOptions();
cleanUp.Add(cudaProviderOptions);
var providerOptionsDict = new Dictionary <string, string>();
providerOptionsDict["device_id"] = "0";
providerOptionsDict["gpu_mem_limit"] = "20971520";
providerOptionsDict["arena_extend_strategy"] = "kSameAsRequested";
providerOptionsDict["cudnn_conv_algo_search"] = "DEFAULT";
providerOptionsDict["do_copy_in_default_stream"] = "1";
providerOptionsDict["cudnn_conv_use_max_workspace"] = "1";
providerOptionsDict["cudnn_conv1d_pad_to_nc1d"] = "1";
cudaProviderOptions.UpdateOptions(providerOptionsDict);
var resultProviderOptionsDict = new Dictionary <string, string>();
ProviderOptionsValueHelper.StringToDict(cudaProviderOptions.GetOptions(), resultProviderOptionsDict);
// test provider options configuration
string value;
value = resultProviderOptionsDict["device_id"];
Assert.Equal("0", value);
value = resultProviderOptionsDict["gpu_mem_limit"];
Assert.Equal("20971520", value);
value = resultProviderOptionsDict["arena_extend_strategy"];
Assert.Equal("kSameAsRequested", value);
value = resultProviderOptionsDict["cudnn_conv_algo_search"];
Assert.Equal("DEFAULT", value);
value = resultProviderOptionsDict["do_copy_in_default_stream"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["cudnn_conv_use_max_workspace"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["cudnn_conv1d_pad_to_nc1d"];
Assert.Equal("1", value);
// test correctness of provider options
SessionOptions options = SessionOptions.MakeSessionOptionWithCudaProvider(cudaProviderOptions);
cleanUp.Add(options);
var session = new InferenceSession(modelPath, options);
cleanUp.Add(session);
var inputMeta = session.InputMetadata;
var container = new List <NamedOnnxValue>();
float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor));
}
session.Run(container);
}
}
private void CanRunInferenceOnAModelWithTensorRT()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
using (var cleanUp = new DisposableListTest <IDisposable>())
{
SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(0);
cleanUp.Add(options);
var session = new InferenceSession(modelPath, options);
cleanUp.Add(session);
var inputMeta = session.InputMetadata;
var container = new List <NamedOnnxValue>();
float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor));
}
using (var results = session.Run(container))
{
ValidateRunResults(results);
}
}
}
private void TestGpu()
{
var tuple = OpenSessionSqueezeNet(0); // run on deviceID 0
float[] expectedOutput = TestDataLoader.LoadTensorFromFile(@"bench.expected_out");
using (var session = tuple.Item1)
{
var inputData = tuple.Item2;
var tensor = tuple.Item3;
var inputMeta = session.InputMetadata;
var container = new List <NamedOnnxValue>();
container.Add(NamedOnnxValue.CreateFromTensor <float>("data_0", tensor));
var res = session.Run(container);
var resultArray = res.First().AsTensor <float>().ToArray();
Assert.Equal(expectedOutput, resultArray, new FloatComparer());
}
}
private void TestTensorRTProviderOptions()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
string calTablePath = "squeezenet_calibration.flatbuffers";
string enginePath = "./";
string engineDecrptLibPath = "engine_decryp";
using (var cleanUp = new DisposableListTest <IDisposable>())
{
var trtProviderOptions = new OrtTensorRTProviderOptions();
cleanUp.Add(trtProviderOptions);
var providerOptionsDict = new Dictionary <string, string>();
providerOptionsDict["device_id"] = "0";
providerOptionsDict["trt_fp16_enable"] = "1";
providerOptionsDict["trt_int8_enable"] = "1";
providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath;
providerOptionsDict["trt_engine_cache_enable"] = "1";
providerOptionsDict["trt_engine_cache_path"] = enginePath;
providerOptionsDict["trt_engine_decryption_enable"] = "0";
providerOptionsDict["trt_engine_decryption_lib_path"] = engineDecrptLibPath;
trtProviderOptions.UpdateOptions(providerOptionsDict);
var resultProviderOptionsDict = new Dictionary <string, string>();
ProviderOptionsValueHelper.StringToDict(trtProviderOptions.GetOptions(), resultProviderOptionsDict);
// test provider options configuration
string value;
value = resultProviderOptionsDict["device_id"];
Assert.Equal("0", value);
value = resultProviderOptionsDict["trt_fp16_enable"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["trt_int8_enable"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["trt_int8_calibration_table_name"];
Assert.Equal(calTablePath, value);
value = resultProviderOptionsDict["trt_engine_cache_enable"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["trt_engine_cache_path"];
Assert.Equal(enginePath, value);
value = resultProviderOptionsDict["trt_engine_decryption_enable"];
Assert.Equal("0", value);
value = resultProviderOptionsDict["trt_engine_decryption_lib_path"];
Assert.Equal(engineDecrptLibPath, value);
// test correctness of provider options
SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(trtProviderOptions);
cleanUp.Add(options);
var session = new InferenceSession(modelPath, options);
cleanUp.Add(session);
var inputMeta = session.InputMetadata;
var container = new List <NamedOnnxValue>();
float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor));
}
session.Run(container);
}
}