private void CanRunInferenceOnAModel(uint graphOptimizationLevel, bool disableSequentialExecution) { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); // Set the graph optimization level for this session. SessionOptions options = new SessionOptions(); Assert.True(options.SetSessionGraphOptimizationLevel(graphOptimizationLevel)); if (disableSequentialExecution) { options.DisableSequentialExecution(); } using (var session = new InferenceSession(modelPath, options)) { var inputMeta = session.InputMetadata; var container = new List <NamedOnnxValue>(); float[] inputData = LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model foreach (var name in inputMeta.Keys) { Assert.Equal(typeof(float), inputMeta[name].ElementType); Assert.True(inputMeta[name].IsTensor); var tensor = new DenseTensor <float>(inputData, inputMeta[name].Dimensions); container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor)); } // Run the inference using (var results = session.Run(container)) // results is an IReadOnlyList<NamedOnnxValue> container { Assert.Equal(1, results.Count); float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out"); // validate the results foreach (var r in results) { Assert.Equal("softmaxout_1", r.Name); var resultTensor = r.AsTensor <float>(); int[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data Assert.Equal(expectedDimensions.Length, resultTensor.Rank); var resultDimensions = resultTensor.Dimensions; for (int i = 0; i < expectedDimensions.Length; i++) { Assert.Equal(expectedDimensions[i], resultDimensions[i]); } var resultArray = r.AsTensor <float>().ToArray(); Assert.Equal(expectedOutput.Length, resultArray.Length); Assert.Equal(expectedOutput, resultArray, new floatComparer()); } } } }
static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteration, DateTime[] timestamps, bool parallelExecution, uint optLevel) { if (timestamps.Length != (int)TimingPoint.TotalCount) { throw new ArgumentException("Timestamps array must have " + (int)TimingPoint.TotalCount + " size"); } timestamps[(int)TimingPoint.Start] = DateTime.Now; SessionOptions options = new SessionOptions(); if (parallelExecution) { options.DisableSequentialExecution(); } options.SetSessionGraphOptimizationLevel(optLevel); using (var session = new InferenceSession(modelPath, options)) { timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now; var inputMeta = session.InputMetadata; var container = new List <NamedOnnxValue>(); foreach (var name in inputMeta.Keys) { float[] rawData = LoadTensorFromFile(inputPath); var tensor = new DenseTensor <float>(rawData, inputMeta[name].Dimensions); container.Add(NamedOnnxValue.CreateFromTensor <float>(name, tensor)); } timestamps[(int)TimingPoint.InputLoaded] = DateTime.Now; // Run the inference for (int i = 0; i < iteration; i++) { var results = session.Run(container); // results is an IReadOnlyList<NamedOnnxValue> container Debug.Assert(results != null); Debug.Assert(results.Count == 1); //results = null; //GC.Collect(); //GC.WaitForPendingFinalizers(); } timestamps[(int)TimingPoint.RunComplete] = DateTime.Now; } }