public void debugModelIO()
{
string _inName, _outName;
foreach (var inputF in _learningModel.InputFeatures)
{
Debug.WriteLine($"input | kind:{inputF.Kind}, name:{inputF.Name}, type:{inputF.GetType()}");
int i = 0;
ImageFeatureDescriptor imgDesc = inputF as ImageFeatureDescriptor;
TensorFeatureDescriptor tfDesc = inputF as TensorFeatureDescriptor;
_inWidth = (uint)(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width);
_inHeight = (uint)(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height);
_inName = inputF.Name;
Debug.WriteLine($"N: {(imgDesc == null ? tfDesc.Shape[0] : 1)}, " +
$"Channel: {(imgDesc == null ? tfDesc.Shape[1].ToString() : imgDesc.BitmapPixelFormat.ToString())}, " +
$"Height:{(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height)}, " +
$"Width: {(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width)}");
}
foreach (var outputF in _learningModel.OutputFeatures)
{
Debug.WriteLine($"output | kind:{outputF.Kind}, name:{outputF.Name}, type:{outputF.GetType()}");
int i = 0;
ImageFeatureDescriptor imgDesc = outputF as ImageFeatureDescriptor;
TensorFeatureDescriptor tfDesc = outputF as TensorFeatureDescriptor;
_outWidth = (uint)(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width);
_outHeight = (uint)(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height);
_outName = outputF.Name;
Debug.WriteLine($"N: {(imgDesc == null ? tfDesc.Shape[0] : 1)}, " +
$"Channel: {(imgDesc == null ? tfDesc.Shape[1].ToString() : imgDesc.BitmapPixelFormat.ToString())}, " +
$"Height:{(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height)}, " +
$"Width: {(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width)}");
}
}
private async Task LoadAndEvaluateModelAsync(VideoFrame _inputFrame, string _modelFileName)
{
LearningModelBinding _binding = null;
VideoFrame _outputFrame = null;
LearningModelSession _session;
try
{
//Load and create the model
if (_model == null)
{
var modelFile =
await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///{_modelFileName}"));
_model = await LearningModel.LoadFromStorageFileAsync(modelFile);
}
// Create the evaluation session with the model
_session = new LearningModelSession(_model);
// Get input and output features of the model
var inputFeatures = _model.InputFeatures.ToList();
var outputFeatures = _model.OutputFeatures.ToList();
// Create binding and then bind input/ output features
_binding = new LearningModelBinding(_session);
_inputImageDescriptor =
inputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
_outputTensorDescriptor =
outputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape);
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(_inputFrame);
// Bind inputs +outputs
_binding.Bind(_inputImageDescriptor.Name, imageTensor);
_binding.Bind(_outputTensorDescriptor.Name, outputTensor);
// Evaluate and get the results
var results = await _session.EvaluateAsync(_binding, "test");
Debug.WriteLine("ResultsEvaluated: " + results.ToString());
var outputTensorList = outputTensor.GetAsVectorView();
var resultsList = new List <float>(outputTensorList.Count);
for (int i = 0; i < outputTensorList.Count; i++)
{
resultsList.Add(outputTensorList[i]);
}
}
catch (Exception ex)
{
Debug.WriteLine($"error: {ex.Message}");
_model = null;
}
}
private async Task <bool> LoadModelAsync()
{
var modelStorageFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri(Constants.MODEL_PATH));
try
{
_model = await LearningModel.LoadFromStorageFileAsync(modelStorageFile);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
// since we do not specify the device, we are using the default CPU option
_session = new LearningModelSession(_model);
List <ILearningModelFeatureDescriptor> inputFeatures;
List <ILearningModelFeatureDescriptor> outputFeatures;
if (_model.InputFeatures == null)
{
return(false);
}
else
{
inputFeatures = _model.InputFeatures.ToList();
}
if (_model.OutputFeatures == null)
{
return(false);
}
else
{
outputFeatures = _model.OutputFeatures.ToList();
}
_inputImageDescriptor =
inputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
_outputTensorDescriptor =
outputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
return(true);
}
public static async Task <CustomVisionModel> CreateFromStorageFile(StorageFile file)
{
CustomVisionModel learningModel = new CustomVisionModel();
learningModel.model = await LearningModel.LoadFromStorageFileAsync(file);
IReadOnlyList <ILearningModelFeatureDescriptor> input = learningModel.model.InputFeatures.ToList();
MapFeatureDescriptor imgDesc = input[0] as MapFeatureDescriptor;
TensorFeatureDescriptor tfDesc = input[0] as TensorFeatureDescriptor;
learningModel.inputParameterName = input[0].Name;
learningModel.inputWidth = (int)tfDesc.Shape[2];
learningModel.inputHeight = (int)tfDesc.Shape[3];
IReadOnlyList <ILearningModelFeatureDescriptor> output = learningModel.model.OutputFeatures.ToList();
MapFeatureDescriptor imgDesc1 = output[0] as MapFeatureDescriptor;
TensorFeatureDescriptor tfDesc1 = output[0] as TensorFeatureDescriptor;
learningModel.outputParameterName = output[0].Name;
learningModel.session = new LearningModelSession(learningModel.model);
learningModel.binding = new LearningModelBinding(learningModel.session);
return(learningModel);
}
/// <summary>
/// Evaluate the VideoFrame passed in as arg
/// </summary>
/// <param name="inputFrame"></param>
/// <returns></returns>
private async Task EvaluateVideoFrameAsync(VideoFrame inputFrame)
{
if (inputFrame != null)
{
try
{
StatusBlock.Text = "Binding image...";
// create a binding object from the session
LearningModelBinding binding = new LearningModelBinding(_session);
// bind the input image
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(inputFrame);
binding.Bind("data_0", imageTensor);
// temp: there is a bug where winml doesn't allow unbound outputs yet, prebind the output!
{
TensorFeatureDescriptor outputTensorDescription = _model.OutputFeatures.FirstOrDefault(
feature => feature.Name == "softmaxout_1"
) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(outputTensorDescription.Shape);
binding.Bind("softmaxout_1", outputTensor);
}
StatusBlock.Text = "Running model...";
int ticks = Environment.TickCount;
// Process the frame with the model
var results = await _session.EvaluateAsync(binding, $"Run { ++_runCount } ");
ticks = Environment.TickCount - ticks;
// retrieve results from evaluation
var resultTensor = results.Outputs["softmaxout_1"] as TensorFloat;
var resultVector = resultTensor.GetAsVectorView();
// Find the top 3 probabilities
List <float> topProbabilities = new List <float>()
{
0.0f, 0.0f, 0.0f
};
List <int> topProbabilityLabelIndexes = new List <int>()
{
0, 0, 0
};
// SqueezeNet returns a list of 1000 options, with probabilities for each, loop through all
for (int i = 0; i < resultVector.Count(); i++)
{
// is it one of the top 3?
for (int j = 0; j < 3; j++)
{
if (resultVector[i] > topProbabilities[j])
{
topProbabilityLabelIndexes[j] = i;
topProbabilities[j] = resultVector[i];
break;
}
}
}
// Display the result
string message = $"Run took { ticks } ticks";
for (int i = 0; i < 3; i++)
{
message += $"\n\"{ _labels[topProbabilityLabelIndexes[i]]}\" with confidence of { topProbabilities[i]}";
}
StatusBlock.Text = message;
}
catch (Exception ex)
{
StatusBlock.Text = $"error: {ex.Message}";
}
ButtonRun.IsEnabled = true;
}
}
/// <summary>
/// Load the labels and model and initialize WinML
/// </summary>
/// <returns></returns>
private async Task LoadModelAsync(string modelFileName)
{
Debug.WriteLine("LoadModelAsync");
_evaluationLock.Wait();
{
m_binding = null;
m_model = null;
m_session = null;
_isReadyForEval = false;
try
{
// Start stopwatch
_perfStopwatch.Restart();
// Load Model
StorageFile modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/{modelFileName}.onnx"));
m_model = await LearningModel.LoadFromStorageFileAsync(modelFile);
// Stop stopwatch
_perfStopwatch.Stop();
// Setting preferred inference device given user's intent
m_inferenceDeviceSelected = _useGPU ? LearningModelDeviceKind.DirectXHighPerformance : LearningModelDeviceKind.Cpu;
m_session = new LearningModelSession(m_model, new LearningModelDevice(m_inferenceDeviceSelected));
// Debugging logic to see the input and output of ther model and retrieve dimensions of input/output variables
// ### DEBUG ###
foreach (var inputF in m_model.InputFeatures)
{
Debug.WriteLine($"input | kind:{inputF.Kind}, name:{inputF.Name}, type:{inputF.GetType()}");
int i = 0;
ImageFeatureDescriptor imgDesc = inputF as ImageFeatureDescriptor;
TensorFeatureDescriptor tfDesc = inputF as TensorFeatureDescriptor;
m_inWidth = (uint)(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width);
m_inHeight = (uint)(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height);
m_inName = inputF.Name;
Debug.WriteLine($"N: {(imgDesc == null ? tfDesc.Shape[0] : 1)}, " +
$"Channel: {(imgDesc == null ? tfDesc.Shape[1].ToString() : imgDesc.BitmapPixelFormat.ToString())}, " +
$"Height:{(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height)}, " +
$"Width: {(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width)}");
}
foreach (var outputF in m_model.OutputFeatures)
{
Debug.WriteLine($"output | kind:{outputF.Kind}, name:{outputF.Name}, type:{outputF.GetType()}");
int i = 0;
ImageFeatureDescriptor imgDesc = outputF as ImageFeatureDescriptor;
TensorFeatureDescriptor tfDesc = outputF as TensorFeatureDescriptor;
m_outWidth = (uint)(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width);
m_outHeight = (uint)(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height);
m_outName = outputF.Name;
Debug.WriteLine($"N: {(imgDesc == null ? tfDesc.Shape[0] : 1)}, " +
$"Channel: {(imgDesc == null ? tfDesc.Shape[1].ToString() : imgDesc.BitmapPixelFormat.ToString())}, " +
$"Height:{(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height)}, " +
$"Width: {(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width)}");
}
// ### END OF DEBUG ###
// Create output frame
_outputFrame?.Dispose();
_outputFrame = new VideoFrame(BitmapPixelFormat.Bgra8, (int)m_outWidth, (int)m_outHeight);
Debug.WriteLine($"Elapsed time: {_perfStopwatch.ElapsedMilliseconds} ms");
_isReadyForEval = true;
}
catch (Exception ex)
{
NotifyUser($"error: {ex.Message}", NotifyType.ErrorMessage);
Debug.WriteLine($"error: {ex.Message}");
}
}
_evaluationLock.Release();
}
public async Task LoadModelAsync(bool shouldUseGpu, bool resourceLoad)
{
try
{
// Parse labels from label file
var labelsTextAsset = Resources.Load(LabelsFileName) as TextAsset;
using (var streamReader = new StringReader(labelsTextAsset.text))
{
string line = "";
char[] charToTrim = { '\"', ' ' };
while (streamReader.Peek() >= 0)
{
line = streamReader.ReadLine();
line.Trim(charToTrim);
var indexAndLabel = line.Split(':');
if (indexAndLabel.Count() == 2)
{
_labels.Add(indexAndLabel[1]);
}
}
}
#if ENABLE_WINMD_SUPPORT
if (resourceLoad)
{
// Load from Unity Resources via awkward UWP streams and initialize model
using (var modelStream = new InMemoryRandomAccessStream())
{
var dataWriter = new DataWriter(modelStream);
var modelResource = Resources.Load(ModelFileName) as TextAsset;
dataWriter.WriteBytes(modelResource.bytes);
await dataWriter.StoreAsync();
var randomAccessStream = RandomAccessStreamReference.CreateFromStream(modelStream);
_model = await LearningModel.LoadFromStreamAsync(randomAccessStream);
var deviceKind = shouldUseGpu ? LearningModelDeviceKind.DirectXHighPerformance : LearningModelDeviceKind.Cpu;
_session = new LearningModelSession(_model, new LearningModelDevice(deviceKind));
}
}
else
{
try
{
var modelFile = await StorageFile.GetFileFromApplicationUriAsync(
new Uri($"ms-appx:///Data/StreamingAssets/SqueezeNet.onnx"));
_model = await LearningModel.LoadFromStorageFileAsync(modelFile);
var deviceKind = shouldUseGpu ? LearningModelDeviceKind.DirectXHighPerformance : LearningModelDeviceKind.Cpu;
_session = new LearningModelSession(_model, new LearningModelDevice(deviceKind));
}
catch (Exception e)
{
var exceptionStr = e.ToString();
//StatusBlock.text = exceptionStr;
}
}
// Get model input and output descriptions
var inputImageDescription =
_model.InputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Image)
as ImageFeatureDescriptor;
// Check if input is passed as image if not try to interpret it as generic tensor
if (inputImageDescription != null)
{
InputWidth = inputImageDescription.Width;
InputHeight = inputImageDescription.Height;
}
else
{
var inputTensorDescription =
_model.InputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor)
as TensorFeatureDescriptor;
InputWidth = (uint)inputTensorDescription.Shape[3];
InputHeight = (uint)inputTensorDescription.Shape[2];
}
_outputDescription =
_model.OutputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor)
as TensorFeatureDescriptor;
#endif
}
catch
{
#if ENABLE_WINMD_SUPPORT
_model = null;
#endif
throw;
}
}
