private void CanRunInferenceOnAModel(GraphOptimizationLevel graphOptimizationLevel, bool disableSequentialExecution)
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
// Set the graph optimization level for this session.
SessionOptions options = new SessionOptions();
options.GraphOptimizationLevel = graphOptimizationLevel;
if (disableSequentialExecution)
{
options.EnableSequentialExecution = false;
}
using (var session = new InferenceSession(modelPath, options))
{
var inputMeta = session.InputMetadata;
var container = new List <NamedOnnxValue>();
float[] inputData = LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor));
}
// Run the inference
using (var results = session.Run(container)) // results is an IReadOnlyList<NamedOnnxValue> container
{
validateRunResults(results);
}
// Run Inference with RunOptions
using (var runOptions = new RunOptions())
{
runOptions.LogTag = "CsharpTest";
runOptions.Terminate = true;
runOptions.LogVerbosityLevel = LogLevel.Error;
IReadOnlyCollection <string> outputNames = session.OutputMetadata.Keys.ToList();
using (var results = session.Run(container, outputNames, runOptions)) // results is an IReadOnlyList<NamedOnnxValue> container
{
validateRunResults(results);
}
}
}
}
static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteration, DateTime[] timestamps,
double[] timecosts, bool parallelExecution, GraphOptimizationLevel optLevel)
{
if (timestamps.Length != (int)TimingPoint.TotalCount)
{
throw new ArgumentException("Timestamps array must have " + (int)TimingPoint.TotalCount + " size");
}
Random random = new Random();
timestamps[(int)TimingPoint.Start] = DateTime.Now;
SessionOptions options = new SessionOptions();
if (parallelExecution)
{
options.ExecutionMode = ExecutionMode.ORT_PARALLEL;
}
options.GraphOptimizationLevel = optLevel;
using (var session = new InferenceSession(modelPath, options))
{
timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now;
var containers = LoadTestData(modelPath, inputPath, session.InputMetadata);
timestamps[(int)TimingPoint.InputLoaded] = DateTime.Now;
// Warm-up
{
var container = containers[random.Next(0, containers.Count)];
session.Run(container);
}
timestamps[(int)TimingPoint.WarmUp] = DateTime.Now;
// Run the inference
for (int i = 0; i < iteration; i++)
{
var next = random.Next(0, containers.Count);
var container = containers[next];
var startTime = DateTime.Now;
var results = session.Run(container); // results is an IReadOnlyList<NamedOnnxValue> container
timecosts[i] = (DateTime.Now - startTime).TotalMilliseconds;
Debug.Assert(results != null);
Debug.Assert(results.Count == 1);
}
timestamps[(int)TimingPoint.RunComplete] = DateTime.Now;
}
}
public static void Run(CommandOptions options)
{
string modelPath = options.ModelFile;
string inputPath = options.InputFile;
int iteration = options.IterationCount;
bool parallelExecution = options.ParallelExecution;
GraphOptimizationLevel optLevel = options.OptimizationLevel;
Console.WriteLine("Running model {0} in OnnxRuntime:", modelPath);
Console.WriteLine("input:{0}", inputPath);
Console.WriteLine("iteration count:{0}", iteration);
Console.WriteLine("parallel execution:{0}", parallelExecution);
Console.WriteLine("optimization level:{0}", optLevel);
DateTime[] timestamps = new DateTime[(int)TimingPoint.TotalCount];
RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps, parallelExecution, optLevel);
PrintReport(timestamps, iteration);
Console.WriteLine("Done");
}
static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteration, DateTime[] timestamps, bool parallelExecution, GraphOptimizationLevel optLevel)
{
if (timestamps.Length != (int)TimingPoint.TotalCount)
{
throw new ArgumentException("Timestamps array must have " + (int)TimingPoint.TotalCount + " size");
}
timestamps[(int)TimingPoint.Start] = DateTime.Now;
SessionOptions options = new SessionOptions();
if (parallelExecution)
{
options.EnableSequentialExecution = false;
}
options.GraphOptimizationLevel = optLevel;
using (var session = new InferenceSession(modelPath, options))
{
timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now;
var inputMeta = session.InputMetadata;
var container = new List <NamedOnnxValue>();
foreach (var name in inputMeta.Keys)
{
float[] rawData = LoadTensorFromFile(inputPath);
var tensor = new DenseTensor <float>(rawData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor));
}
timestamps[(int)TimingPoint.InputLoaded] = DateTime.Now;
// Run the inference
for (int i = 0; i < iteration; i++)
{
var results = session.Run(container); // results is an IReadOnlyList<NamedOnnxValue> container
Debug.Assert(results != null);
Debug.Assert(results.Count == 1);
//results = null;
//GC.Collect();
//GC.WaitForPendingFinalizers();
}
timestamps[(int)TimingPoint.RunComplete] = DateTime.Now;
}
}
