private void TestPreTrainedModels(string opset, string modelName)
{
var modelsDir = GetTestModelsDir();
string onnxModelFileName = null;
var modelDir = new DirectoryInfo(Path.Combine(modelsDir, opset, modelName));
try
{
var onnxModelNames = modelDir.GetFiles("*.onnx");
bool validModelFound = false;
if (onnxModelNames.Length > 0)
{
// TODO remove file "._resnet34v2.onnx" from test set
for (int i = 0; i < onnxModelNames.Length; i++)
{
if (onnxModelNames[i].Name != "._resnet34v2.onnx")
{
onnxModelNames[0] = onnxModelNames[i];
validModelFound = true;
}
}
}
if (validModelFound)
{
onnxModelFileName = Path.Combine(modelDir.FullName, onnxModelNames[0].Name);
}
else
{
var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString()));
throw new Exception($"Opset {opset} Model {modelName}. Can't determine model file name. Found these :{modelNamesList}");
}
using (var session = new InferenceSession(onnxModelFileName))
{
var inMeta = session.InputMetadata;
string testDataDirNamePattern = "test_data*";
if (opset == "opset9" && modelName == "LSTM_Seq_lens_unpacked")
{
testDataDirNamePattern = "seq_lens*"; // discrepancy in data directory
}
foreach (var testDataDir in modelDir.EnumerateDirectories(testDataDirNamePattern))
{
var inputContainer = new List <NamedOnnxValue>();
var outputContainer = new List <NamedOnnxValue>();
foreach (var f in testDataDir.EnumerateFiles("input_*.pb"))
{
inputContainer.Add(TestDataLoader.LoadTensorFromFilePb(f.FullName, inMeta));
}
foreach (var f in testDataDir.EnumerateFiles("output_*.pb"))
{
outputContainer.Add(TestDataLoader.LoadTensorFromFilePb(f.FullName, session.OutputMetadata));
}
using (var resultCollection = session.Run(inputContainer))
{
foreach (var result in resultCollection)
{
Assert.True(session.OutputMetadata.ContainsKey(result.Name));
var outputMeta = session.OutputMetadata[result.Name];
NamedOnnxValue outputValue = null;
foreach (var o in outputContainer)
{
if (o.Name == result.Name)
{
outputValue = o;
break;
}
}
if (outputValue == null)
{
outputValue = outputContainer.First(); // in case the output data file does not contain the name
}
if (outputMeta.IsTensor)
{
if (outputMeta.ElementType == typeof(float))
{
Assert.Equal(result.AsTensor <float>(), outputValue.AsTensor <float>(), new FloatComparer());
}
else if (outputMeta.ElementType == typeof(double))
{
Assert.Equal(result.AsTensor <double>(), outputValue.AsTensor <double>(), new DoubleComparer());
}
else if (outputMeta.ElementType == typeof(int))
{
Assert.Equal(result.AsTensor <int>(), outputValue.AsTensor <int>(), new ExactComparer <int>());
}
else if (outputMeta.ElementType == typeof(uint))
{
Assert.Equal(result.AsTensor <uint>(), outputValue.AsTensor <uint>(), new ExactComparer <uint>());
}
else if (outputMeta.ElementType == typeof(short))
{
Assert.Equal(result.AsTensor <short>(), outputValue.AsTensor <short>(), new ExactComparer <short>());
}
else if (outputMeta.ElementType == typeof(ushort))
{
Assert.Equal(result.AsTensor <ushort>(), outputValue.AsTensor <ushort>(), new ExactComparer <ushort>());
}
else if (outputMeta.ElementType == typeof(long))
{
Assert.Equal(result.AsTensor <long>(), outputValue.AsTensor <long>(), new ExactComparer <long>());
}
else if (outputMeta.ElementType == typeof(ulong))
{
Assert.Equal(result.AsTensor <ulong>(), outputValue.AsTensor <ulong>(), new ExactComparer <ulong>());
}
else if (outputMeta.ElementType == typeof(byte))
{
Assert.Equal(result.AsTensor <byte>(), outputValue.AsTensor <byte>(), new ExactComparer <byte>());
}
else if (outputMeta.ElementType == typeof(bool))
{
Assert.Equal(result.AsTensor <bool>(), outputValue.AsTensor <bool>(), new ExactComparer <bool>());
}
else if (outputMeta.ElementType == typeof(Float16))
{
Assert.Equal(result.AsTensor <Float16>(), outputValue.AsTensor <Float16>(), new Float16Comparer {
tolerance = 2
});
}
else if (outputMeta.ElementType == typeof(BFloat16))
{
Assert.Equal(result.AsTensor <BFloat16>(), outputValue.AsTensor <BFloat16>(), new BFloat16Comparer {
tolerance = 2
});
}
else
{
Assert.True(false, $"{nameof(TestPreTrainedModels)} does not yet support output of type {outputMeta.ElementType}");
}
}
else
{
Assert.True(false, $"{nameof(TestPreTrainedModels)} cannot handle non-tensor outputs yet");
}
}
}
}
}
}
catch (Exception ex)
{
var msg = $"Opset {opset}, Model {modelName}: ModelFile = {onnxModelFileName} error = {ex.Message}";
if (ex.Message.Contains("ONNX Runtime only *guarantees* support for models stamped with official released onnx opset versions"))
{
// If the exception is thrown because the opset version of the test model is
// not supported by ONNXRuntime yet, then ignore the test and proceed.
// ORT allows commits from ONNX master and in such cases we do come across new opsets which are
// not supported in ORT yet. In order to force these tests to run set env var ALLOW_RELEASED_ONNX_OPSET_ONLY=0
output.WriteLine("Skipping the model test as the latest ONNX opset is not supported yet. Error Message: " + msg);
}
else
{
throw new Exception(msg + "\n" + ex.StackTrace);
}
}
}