private unsafe LearningModelBinding EvaluateContrastAndBrightnessSession(object input, object output)
{
var slope = Math.Tan(ContrastMaxSlider.Value * 3.14159 / 2);
var yintercept = -255 * (ContrastMinSlider.Value * 2 - 1);
if (yintercept < 0)
{
// it was the x-intercept
yintercept = slope * yintercept;
}
var binding = new LearningModelBinding(contrastEffectSession_);
binding.Bind("Input", input);
binding.Bind("Slope", TensorFloat.CreateFromArray(new long[] { 1 }, new float[] { (float)slope }));
binding.Bind("YIntercept", TensorFloat.CreateFromArray(new long[] { 1 }, new float[] { (float)yintercept }));
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
binding.Bind("Output", output, outputBindProperties);
EvaluateInternal(contrastEffectSession_, binding);
return(binding);
}
private async Task LoadModelAsync()
{
Debug.Write("LoadModelBegin | ");
Debug.Write("LoadModel Lock | ");
_binding?.Clear();
_session?.Dispose();
StorageFile modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/{_appModel.ModelSource}.onnx"));
_learningModel = await LearningModel.LoadFromStorageFileAsync(modelFile);
_inferenceDeviceSelected = UseGpu ? LearningModelDeviceKind.DirectX : LearningModelDeviceKind.Cpu;
// Lock so can't create a new session or binding while also being disposed
lock (_processLock)
{
_session = new LearningModelSession(_learningModel, new LearningModelDevice(_inferenceDeviceSelected));
_binding = new LearningModelBinding(_session);
}
debugModelIO();
_inputImageDescription = _learningModel.InputFeatures.ToList().First().Name;
_outputImageDescription = _learningModel.OutputFeatures.ToList().First().Name;
Debug.Write("LoadModel Unlock\n");
}
private async Task LoadAndEvaluateModelAsync(VideoFrame _inputFrame, string _modelFileName)
{
LearningModelBinding _binding = null;
VideoFrame _outputFrame = null;
LearningModelSession _session;
try
{
//Load and create the model
if (_model == null)
{
var modelFile =
await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///{_modelFileName}"));
_model = await LearningModel.LoadFromStorageFileAsync(modelFile);
}
// Create the evaluation session with the model
_session = new LearningModelSession(_model);
// Get input and output features of the model
var inputFeatures = _model.InputFeatures.ToList();
var outputFeatures = _model.OutputFeatures.ToList();
// Create binding and then bind input/ output features
_binding = new LearningModelBinding(_session);
_inputImageDescriptor =
inputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
_outputTensorDescriptor =
outputFeatures.FirstOrDefault(feature => feature.Kind == LearningModelFeatureKind.Tensor) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(_outputTensorDescriptor.Shape);
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(_inputFrame);
// Bind inputs +outputs
_binding.Bind(_inputImageDescriptor.Name, imageTensor);
_binding.Bind(_outputTensorDescriptor.Name, outputTensor);
// Evaluate and get the results
var results = await _session.EvaluateAsync(_binding, "test");
Debug.WriteLine("ResultsEvaluated: " + results.ToString());
var outputTensorList = outputTensor.GetAsVectorView();
var resultsList = new List <float>(outputTensorList.Count);
for (int i = 0; i < outputTensorList.Count; i++)
{
resultsList.Add(outputTensorList[i]);
}
}
catch (Exception ex)
{
Debug.WriteLine($"error: {ex.Message}");
_model = null;
}
}
public async Task InitializeAsync(ModelType modelType, params string[] parameters)
{
var file = await StorageFile.GetFileFromApplicationUriAsync(new Uri(parameters[0]));
model = await LearningModel.LoadFromStreamAsync(file);
session = new LearningModelSession(model);
binding = new LearningModelBinding(session);
}
private void SampleInputsGridView_SelectionChanged(object sender, SelectionChangedEventArgs e)
{
var gridView = sender as GridView;
var thumbnail = gridView.SelectedItem as WinMLSamplesGallery.Controls.Thumbnail;
if (thumbnail != null)
{
var image = thumbnail.ImageUri;
var file = StorageFile.GetFileFromApplicationUriAsync(new Uri(image)).GetAwaiter().GetResult();
var softwareBitmap = CreateSoftwareBitmapFromStorageFile(file);
tensorizationSession_ =
CreateLearningModelSession(
TensorizationModels.ReshapeFlatBufferNHWC(
1,
4,
softwareBitmap.PixelHeight,
softwareBitmap.PixelWidth,
416,
416));
// Tensorize
var stream = file.OpenAsync(FileAccessMode.Read).GetAwaiter().GetResult();
var decoder = BitmapDecoder.CreateAsync(stream).GetAwaiter().GetResult();
var bitmap = decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied).GetAwaiter().GetResult();
var pixelDataProvider = decoder.GetPixelDataAsync().GetAwaiter().GetResult();
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
// 3 channel NCHW
var tensorizeOutput = TensorFloat.Create(new long[] { 1, 416, 416, 3 });
var b = new LearningModelBinding(tensorizationSession_);
b.Bind(tensorizationSession_.Model.InputFeatures[0].Name, inputRawTensor);
b.Bind(tensorizationSession_.Model.OutputFeatures[0].Name, tensorizeOutput);
tensorizationSession_.Evaluate(b, "");
// Resize
var resizeBinding = new LearningModelBinding(_session);
resizeBinding.Bind(_session.Model.InputFeatures[0].Name, tensorizeOutput);
var results = _session.Evaluate(resizeBinding, "");
var output1 = results.Output(0) as TensorFloat;
var data = output1.GetAsVectorView();
var detections = ParseResult(data.ToList <float>().ToArray());
Comparer cp = new Comparer();
detections.Sort(cp);
var final = NMS(detections);
RenderImageInMainPanel(softwareBitmap);
}
}
/// <summary>
/// NeuralStyleTransformerBinding constructor
/// </summary>
internal NeuralStyleTransformerBinding(
ISkillDescriptor descriptor,
ISkillExecutionDevice device,
LearningModelSession session)
{
m_bindingHelper = new VisionSkillBindingHelper(descriptor, device);
// Create WinML binding
m_winmlBinding = new LearningModelBinding(session);
}
/// <summary>
/// FaceSentimentAnalyzerBinding constructor
/// </summary>
internal FaceSentimentAnalyzerBinding(
ISkillDescriptor descriptor,
ISkillExecutionDevice device,
LearningModelSession session)
{
m_bindingHelper = new VisionSkillBindingHelper(descriptor, device);
// Create WinML binding
m_winmlBinding = new LearningModelBinding(session);
}
/// <summary>
/// Detect objects from the given image.
/// The input image must be 416x416.
/// </summary>
public async Task <IList <PredictionModel> > PredictImageAsync(VideoFrame image)
{
var imageFeature = ImageFeatureValue.CreateFromVideoFrame(image);
var bindings = new LearningModelBinding(this.session);
bindings.Bind("data", imageFeature);
var result = await this.session.EvaluateAsync(bindings, "");
return(Postprocess(result.Outputs["model_outputs0"] as TensorFloat));
}
internal async Task InitModelAsync()
{
var model_file = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///Assets//Yolo.onnx"));
_model = await LearningModel.LoadFromStorageFileAsync(model_file);
var device = new LearningModelDevice(LearningModelDeviceKind.Cpu);
_session = new LearningModelSession(_model, device);
_binding = new LearningModelBinding(_session);
}
public async Task <ModelOutput> EvaluateAsync(OnnxModelInput input)
{
var output = new ModelOutput();
var binding = new LearningModelBinding(_session);
binding.Bind("data", input.Data);
binding.Bind("model_outputs0", output.Model_outputs0);
var evalResult = await _session.EvaluateAsync(binding, "0");
return(output);
}
/// <summary>
/// Detect objects from the given image.
/// The input image must be 416x416.
/// </summary>
public async Task <IList <PredictionModel> > PredictImageAsync(VideoFrame image)
{
var output = new modelOutput();
var imageFeature = ImageFeatureValue.CreateFromVideoFrame(image);
var bindings = new LearningModelBinding(Session);
bindings.Bind("data", imageFeature);
bindings.Bind("model_outputs0", output.Model_outputs0);
var result = await Session.EvaluateAsync(bindings, "0");
return(Postprocess(output.Model_outputs0));
}
/// <summary>
/// Initialize
/// </summary>
/// <param name="file">The ONNX file</param>
///
public async Task Init()
{
StorageFile ModelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/" + "model.onnx"));
this.model = await LearningModel.LoadFromStorageFileAsync(ModelFile);
this.session = new LearningModelSession(this.model);
this.binding = new LearningModelBinding(this.session);
Debug.Assert(this.model.InputFeatures.Count == 1, "The number of input must be 1");
Debug.Assert(this.model.OutputFeatures.Count == 1, "The number of output must be 1");
}
public async Task <ModelOutput> EvaluateAsync(OnnxModelInput input)
{
var output = new ModelOutput();
var binding = new LearningModelBinding(_session);
binding.Bind("data", input.Data);
//binding.Bind("classLabel", output.ClassLabel);
//binding.Bind("loss", output.Loss);
LearningModelEvaluationResult evalResult = await _session.EvaluateAsync(binding, "0");
return(output);
}
/// <summary>
/// Evaluate the model
/// </summary>
/// <param name="input">The VideoFrame to evaluate</param>
/// <returns></returns>
public async Task <ONNXModelOutput> EvaluateAsync(ONNXModelInput input)
{
var output = new ONNXModelOutput();
var binding = new LearningModelBinding(_session);
binding.Bind("data", input.data);
binding.Bind("classLabel", output.classLabel);
binding.Bind("loss", output.loss);
LearningModelEvaluationResult result = await _session.EvaluateAsync(binding, "0");
return(output);
}
public async Task <IList <PredictionModel> > EvaluateAsync(ONNXModelInput input)
{
var binding = new LearningModelBinding(_session);
System.Diagnostics.Debug.WriteLine("start binding");
binding.Bind("data", input.Data);
System.Diagnostics.Debug.WriteLine("loss binded");
var result = await _session.EvaluateAsync(binding, "");
System.Diagnostics.Debug.WriteLine("result created");
return(Postprocess(result.Outputs["model_outputs0"] as TensorFloat));
}
private void ThisAddIn_Startup(object sender, System.EventArgs e)
{
richTextBox1 = new RichTextBox();
richTextBox1.Dock = DockStyle.Fill;
richTextBox1.SelectionFont = new Font("Verdana", 12, FontStyle.Bold);
richTextBox1.SelectionColor = Color.Red;
Clipboard.SetImage(Image.FromFile(_imagePath));
richTextBox1.Paste();
// Load and create the model
outToLog($"Loading modelfile '{_modelPath}' on the '{_deviceName}' device");
int ticks = Environment.TickCount;
_model = LearningModel.LoadFromFilePath(_modelPath);
ticks = Environment.TickCount - ticks;
outToLog($"model file loaded in { ticks } ticks");
// Create the evaluation session with the model and device
_session = new LearningModelSession(_model);
outToLog("Getting color management mode...");
ColorManagementMode colorManagementMode = GetColorManagementMode();
outToLog("Loading the image...");
ImageFeatureValue imageTensor = LoadImageFile(colorManagementMode);
// create a binding object from the session
outToLog("Binding...");
LearningModelBinding binding = new LearningModelBinding(_session);
binding.Bind(_model.InputFeatures.ElementAt(0).Name, imageTensor);
outToLog("Running the model...");
ticks = Environment.TickCount;
var results = _session.Evaluate(binding, "RunId");
ticks = Environment.TickCount - ticks;
outToLog($"model run took { ticks } ticks");
// retrieve results from evaluation
var resultTensor = results.Outputs[_model.OutputFeatures.ElementAt(0).Name] as TensorFloat;
var resultVector = resultTensor.GetAsVectorView();
PrintResults(resultVector);
Form form1 = new Form();
form1.Size = new Size(800, 800);
form1.Controls.Add(richTextBox1);
//form1.Show();
form1.ShowDialog();
}
public async Task <string> ObtenerIdentidadOnnX(VideoFrame videoFrame, LearningModelSession _session)
{
identidadEncontradaTexto = string.Empty;
if (videoFrame != null)
{
try
{
LearningModelBinding binding = new LearningModelBinding(_session);
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(videoFrame);
binding.Bind("data", imageTensor);
int ticks = Environment.TickCount;
// Process the frame with the model
var results = await _session.EvaluateAsync(binding, $"Run { ++_runCount } ");
ticks = Environment.TickCount - ticks;
var label = results.Outputs["classLabel"] as TensorString;
var resultVector = label.GetAsVectorView();
List <float> topProbabilities = new List <float>()
{
0.0f, 0.0f, 0.0f
};
List <int> topProbabilityLabelIndexes = new List <int>()
{
0, 0, 0
};
// SqueezeNet returns a list of 1000 options, with probabilities for each, loop through all
for (int i = 0; i < resultVector.Count(); i++)
{
// is it one of the top 3?
for (int j = 0; j < 3; j++)
{
identidadEncontradaTexto = resultVector[i].ToString();
//if (resultVector[i] > topProbabilities[j])
//{
// topProbabilityLabelIndexes[j] = i;
// topProbabilities[j] = resultVector[i];
// break;
//}
}
}
}
catch (Exception ex)
{
identidadEncontradaTexto = "";
}
}
return(identidadEncontradaTexto);
}
private void EvaluateInternal(LearningModelSession session, LearningModelBinding binding, bool wait = false)
{
if (IsCpu)
{
session.Evaluate(binding, "");
}
else
{
var results = session.EvaluateAsync(binding, "");
if (wait)
{
results.GetAwaiter().GetResult();
}
}
}
// usage: SqueezeNet [modelfile] [imagefile] [cpu|directx]
static int Main(string[] args)
{
if (!ParseArgs(args))
{
Console.WriteLine("Usage: [executable_name] [modelfile] [imagefile] [cpu|directx]");
return(-1);
}
// Load and create the model
Console.WriteLine($"Loading modelfile '{_modelPath}' on the '{_deviceName}' device");
int ticks = Environment.TickCount;
_model = LearningModel.LoadFromFilePath(_modelPath);
ticks = Environment.TickCount - ticks;
Console.WriteLine($"model file loaded in { ticks } ticks");
// Create the evaluation session with the model and device
_session = new LearningModelSession(_model, new LearningModelDevice(_deviceKind));
Console.WriteLine("Getting color management mode...");
ColorManagementMode colorManagementMode = GetColorManagementMode();
Console.WriteLine("Loading the image...");
ImageFeatureValue imageTensor = LoadImageFile(colorManagementMode);
// create a binding object from the session
Console.WriteLine("Binding...");
LearningModelBinding binding = new LearningModelBinding(_session);
binding.Bind(_model.InputFeatures.ElementAt(0).Name, imageTensor);
Console.WriteLine("Running the model...");
ticks = Environment.TickCount;
var results = _session.Evaluate(binding, "RunId");
ticks = Environment.TickCount - ticks;
Console.WriteLine($"model run took { ticks } ticks");
// retrieve results from evaluation
var resultTensor = results.Outputs[_model.OutputFeatures.ElementAt(0).Name] as TensorFloat;
var resultVector = resultTensor.GetAsVectorView();
PrintResults(resultVector);
return(0);
}
private bool disposedValue = false; // 要检测冗余调用
void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
// TODO: 释放托管状态(托管对象)。
}
// TODO: 释放未托管的资源(未托管的对象)并在以下内容中替代终结器。
// TODO: 将大型字段设置为 null。
model = null;
session = null;
binding = null;
disposedValue = true;
}
}
internal String Evaluate()
{
// input tensor shape is [1x4]
long[] shape = new long[2];
shape[0] = 1;
shape[1] = 4;
// set up the input tensor
float[] input_data = new float[4];
input_data[0] = _sepal_length;
input_data[1] = _sepal_width;
input_data[2] = _petal_length;
input_data[3] = _petal_width;
TensorFloat tensor_float = TensorFloat.CreateFromArray(shape, input_data);
// bind the tensor to "input"
var binding = new LearningModelBinding(_session);
binding.Bind("input", tensor_float);
// evaluate
var results = _session.Evaluate(binding, "");
// get the results
TensorFloat prediction = (TensorFloat)results.Outputs.First().Value;
var prediction_data = prediction.GetAsVectorView();
// find the highest predicted value
int max_index = 0;
float max_value = 0;
for (int i = 0; i < prediction_data.Count; i++)
{
var val = prediction_data.ElementAt(i);
if (val > max_value)
{
max_value = val;
max_index = i;
}
}
// return the label corresponding to the highest predicted value
return(_labels.ElementAt(max_index));
}
private LearningModelBinding Evaluate(LearningModelSession session, object input, object output, bool wait = false)
{
// Create the binding
var binding = new LearningModelBinding(session);
// Bind inputs and outputs
string inputName = session.Model.InputFeatures[0].Name;
string outputName = session.Model.OutputFeatures[0].Name;
binding.Bind(inputName, input);
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
binding.Bind(outputName, output, outputBindProperties);
// Evaluate
EvaluateInternal(session, binding, wait);
return(binding);
}
/// <summary>
/// Detect objects from the given image.
/// The input image must be 416x416.
/// </summary>
public async Task <IList <PredictionModel> > PredictImageAsync(VideoFrame image)
{
var imageWidth = image.SoftwareBitmap.PixelWidth;
var imageHeight = image.SoftwareBitmap.PixelHeight;
double ratio = Math.Sqrt((double)imageInputSize / (double)imageWidth / (double)imageHeight);
int targetWidth = 32 * (int)Math.Round(imageWidth * ratio / 32);
int targetHeight = 32 * (int)Math.Round(imageHeight * ratio / 32);
using (var resizedBitmap = await ResizeBitmap(image.SoftwareBitmap, targetWidth, targetHeight))
using (VideoFrame resizedVideoFrame = VideoFrame.CreateWithSoftwareBitmap(resizedBitmap))
{
var imageFeature = ImageFeatureValue.CreateFromVideoFrame(resizedVideoFrame);
var bindings = new LearningModelBinding(this.session);
bindings.Bind("input", imageFeature);
var result = await this.session.EvaluateAsync(bindings, "");
return(Postprocess(result.Outputs["output"] as TensorFloat));
}
}
private static float Evaluate(LearningModelSession session, List <VideoFrame> input)
{
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int j = 0; j < input.Count; j++)
{
if (navigatingAwayFromPage)
{
break;
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, input[j]);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return(totalDuration);
}
public override List <ONNXTensor> Run(IEnumerable <string> outputs, Dictionary <string, ONNXTensor> feedDict)
{
var binding = new LearningModelBinding(sess);
foreach (var item in feedDict)
{
object tensor;
if (!IsFP16)
{
tensor = TensorFloat.CreateFromArray(item.Value.Shape, item.Value.Buffer);
if (IsGPU)
{
//TODO: Move SoftwareTensor to DX12Tensor
tensor = MoveToGPU((TensorFloat)tensor);
}
}
else
{
tensor = TensorFloat16Bit.CreateFromArray(item.Value.Shape, item.Value.Buffer);
}
binding.Bind(item.Key, tensor);
}
var result = sess.Evaluate(binding, $"eval{++evalCount}");
var ret = new List <ONNXTensor>();
foreach (var item in outputs)
{
var tensor = result.Outputs[item] as TensorFloat;
var vector = tensor.GetAsVectorView().ToArray();
ret.Add(new ONNXTensor()
{
Buffer = vector, Shape = tensor.Shape.ToArray()
});
}
return(ret);
}
private static float EvaluateBatched(LearningModelSession session, List <VideoFrame> input, int batchSize)
{
int numBatches = (int)Math.Ceiling((Decimal)input.Count / batchSize);
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int i = 0; i < numBatches; i++)
{
if (navigatingAwayFromPage)
{
break;
}
int rangeStart = batchSize * i;
List <VideoFrame> batch;
// Add padding to the last batch if necessary
if (rangeStart + batchSize > input.Count)
{
int numInputsRemaining = input.Count - rangeStart;
int paddingAmount = batchSize - numInputsRemaining;
batch = input.GetRange(rangeStart, numInputsRemaining);
batch.AddRange(input.GetRange(0, paddingAmount));
}
else
{
batch = input.GetRange(rangeStart, batchSize);
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, batch);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return(totalDuration);
}
private static LearningModelEvaluationResult Evaluate(LearningModelSession session, object input)
{
// Create the binding
var binding = new LearningModelBinding(session);
// Create an empty output, that will keep the output resources on the GPU
// It will be chained into a the post processing on the GPU as well
var output = TensorFloat.Create();
// Bind inputs and outputs
// For squeezenet these evaluate to "data", and "squeezenet0_flatten0_reshape0"
string inputName = session.Model.InputFeatures[0].Name;
string outputName = session.Model.OutputFeatures[0].Name;
binding.Bind(inputName, input);
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
binding.Bind(outputName, output, outputBindProperties);
// Evaluate
return(session.Evaluate(binding, ""));
}
/// <summary>
/// Evaluate the VideoFrame passed in as arg
/// </summary>
/// <param name="inputFrame"></param>
/// <returns></returns>
private async Task EvaluateVideoFrameAsync(VideoFrame inputFrame)
{
if (inputFrame != null)
{
try
{
StatusBlock.Text = "Binding image...";
// create a binding object from the session
LearningModelBinding binding = new LearningModelBinding(_session);
// bind the input image
ImageFeatureValue imageTensor = ImageFeatureValue.CreateFromVideoFrame(inputFrame);
binding.Bind("data_0", imageTensor);
// temp: there is a bug where winml doesn't allow unbound outputs yet, prebind the output!
{
TensorFeatureDescriptor outputTensorDescription = _model.OutputFeatures.FirstOrDefault(
feature => feature.Name == "softmaxout_1"
) as TensorFeatureDescriptor;
TensorFloat outputTensor = TensorFloat.Create(outputTensorDescription.Shape);
binding.Bind("softmaxout_1", outputTensor);
}
StatusBlock.Text = "Running model...";
int ticks = Environment.TickCount;
// Process the frame with the model
var results = await _session.EvaluateAsync(binding, $"Run { ++_runCount } ");
ticks = Environment.TickCount - ticks;
// retrieve results from evaluation
var resultTensor = results.Outputs["softmaxout_1"] as TensorFloat;
var resultVector = resultTensor.GetAsVectorView();
// Find the top 3 probabilities
List <float> topProbabilities = new List <float>()
{
0.0f, 0.0f, 0.0f
};
List <int> topProbabilityLabelIndexes = new List <int>()
{
0, 0, 0
};
// SqueezeNet returns a list of 1000 options, with probabilities for each, loop through all
for (int i = 0; i < resultVector.Count(); i++)
{
// is it one of the top 3?
for (int j = 0; j < 3; j++)
{
if (resultVector[i] > topProbabilities[j])
{
topProbabilityLabelIndexes[j] = i;
topProbabilities[j] = resultVector[i];
break;
}
}
}
// Display the result
string message = $"Run took { ticks } ticks";
for (int i = 0; i < 3; i++)
{
message += $"\n\"{ _labels[topProbabilityLabelIndexes[i]]}\" with confidence of { topProbabilities[i]}";
}
StatusBlock.Text = message;
}
catch (Exception ex)
{
StatusBlock.Text = $"error: {ex.Message}";
}
ButtonRun.IsEnabled = true;
}
}
/// <summary>
/// 1) Bind input and output features
/// 2) Run evaluation of the model
/// 3) Report the result
/// </summary>
/// <param name="inputVideoFrame"></param>
/// <returns></returns>
private async Task EvaluateVideoFrameAsync(VideoFrame inputVideoFrame)
{
Debug.WriteLine("EvaluateVideoFrameAsync");
LearningModelSession session = null;
bool isReadyForEval = false;
bool showInitialImageAndProgress = true;
bool proceedWithEval = false;
_evaluationLock.Wait();
{
session = m_session;
isReadyForEval = _isReadyForEval;
_isReadyForEval = false;
showInitialImageAndProgress = _showInitialImageAndProgress;
proceedWithEval = _proceedWithEval;
}
_evaluationLock.Release();
if ((inputVideoFrame != null) &&
(inputVideoFrame.SoftwareBitmap != null || inputVideoFrame.Direct3DSurface != null) &&
isReadyForEval &&
(session != null) &&
proceedWithEval)
{
try
{
_perfStopwatch.Restart();
NotifyUser("Processing...", NotifyType.StatusMessage);
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
if (showInitialImageAndProgress)
{
UIProcessingProgressRing.IsActive = true;
UIProcessingProgressRing.Visibility = Visibility.Visible;
UIButtonSaveImage.IsEnabled = false;
}
});
// Crop the input image to communicate appropriately to the user what is being evaluated
_inputFrame = await ImageHelper.CenterCropImageAsync(inputVideoFrame, m_inWidth, m_inHeight);
_perfStopwatch.Stop();
Int64 cropTime = _perfStopwatch.ElapsedMilliseconds;
Debug.WriteLine($"Image handling: {cropTime}ms");
// Bind and Eval
if (_inputFrame != null)
{
_evaluationLock.Wait();
try
{
_perfStopwatch.Restart();
// create bindings for the input and output buffers
// ###### BUG 4794 - Reusing the same binding object currently fails to update output on 2nd+ eval call as of 07/17/2018
//if (m_binding == null)
{
m_binding = new LearningModelBinding(session);
ImageFeatureValue outputImageFeatureValue = ImageFeatureValue.CreateFromVideoFrame(_outputFrame);
m_binding.Bind(m_outName, outputImageFeatureValue);
}
ImageFeatureValue inputImageFeatureValue = ImageFeatureValue.CreateFromVideoFrame(_inputFrame);
m_binding.Bind(m_inName, inputImageFeatureValue);
Int64 bindTime = _perfStopwatch.ElapsedMilliseconds;
Debug.WriteLine($"Binding: {bindTime}ms");
// render the input frame
if (showInitialImageAndProgress)
{
await ImageHelper.RenderFrameAsync(_inputFrameRenderer, _inputFrame);
}
// Process the frame with the model
_perfStopwatch.Restart();
var results = m_session.Evaluate(m_binding, "test");
_perfStopwatch.Stop();
Int64 evalTime = _perfStopwatch.ElapsedMilliseconds;
Debug.WriteLine($"Eval: {evalTime}ms");
// Parse result
IReadOnlyDictionary <string, object> outputs = results.Outputs;
foreach (var output in outputs)
{
Debug.WriteLine($"{output.Key} : {output.Value} -> {output.Value.GetType()}");
}
// Display result
//ImageFeatureValue outputImage = (results.Outputs[m_outputTensorDescription.Name] as ImageFeatureValue);
//if(outputImage != null)
//{
// _outputFrame = outputImage.VideoFrame;
//}
await ImageHelper.RenderFrameAsync(_resultframeRenderer, _outputFrame);
}
catch (Exception ex)
{
NotifyUser(ex.Message, NotifyType.ErrorMessage);
Debug.WriteLine(ex.ToString());
}
finally
{
_evaluationLock.Release();
}
if (showInitialImageAndProgress)
{
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
UIProcessingProgressRing.IsActive = false;
UIProcessingProgressRing.Visibility = Visibility.Collapsed;
UIButtonSaveImage.IsEnabled = true;
});
}
NotifyUser("Done!", NotifyType.StatusMessage);
}
else
{
Debug.WriteLine("Skipped eval, null input frame");
}
}
catch (Exception ex)
{
NotifyUser(ex.Message, NotifyType.ErrorMessage);
Debug.WriteLine(ex.ToString());
}
_evaluationLock.Wait();
{
_isReadyForEval = true;
}
_evaluationLock.Release();
_perfStopwatch.Reset();
}
}
/// <summary>
/// Load the labels and model and initialize WinML
/// </summary>
/// <returns></returns>
private async Task LoadModelAsync(string modelFileName)
{
Debug.WriteLine("LoadModelAsync");
_evaluationLock.Wait();
{
m_binding = null;
m_model = null;
m_session = null;
_isReadyForEval = false;
try
{
// Start stopwatch
_perfStopwatch.Restart();
// Load Model
StorageFile modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/{modelFileName}.onnx"));
m_model = await LearningModel.LoadFromStorageFileAsync(modelFile);
// Stop stopwatch
_perfStopwatch.Stop();
// Setting preferred inference device given user's intent
m_inferenceDeviceSelected = _useGPU ? LearningModelDeviceKind.DirectXHighPerformance : LearningModelDeviceKind.Cpu;
m_session = new LearningModelSession(m_model, new LearningModelDevice(m_inferenceDeviceSelected));
// Debugging logic to see the input and output of ther model and retrieve dimensions of input/output variables
// ### DEBUG ###
foreach (var inputF in m_model.InputFeatures)
{
Debug.WriteLine($"input | kind:{inputF.Kind}, name:{inputF.Name}, type:{inputF.GetType()}");
int i = 0;
ImageFeatureDescriptor imgDesc = inputF as ImageFeatureDescriptor;
TensorFeatureDescriptor tfDesc = inputF as TensorFeatureDescriptor;
m_inWidth = (uint)(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width);
m_inHeight = (uint)(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height);
m_inName = inputF.Name;
Debug.WriteLine($"N: {(imgDesc == null ? tfDesc.Shape[0] : 1)}, " +
$"Channel: {(imgDesc == null ? tfDesc.Shape[1].ToString() : imgDesc.BitmapPixelFormat.ToString())}, " +
$"Height:{(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height)}, " +
$"Width: {(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width)}");
}
foreach (var outputF in m_model.OutputFeatures)
{
Debug.WriteLine($"output | kind:{outputF.Kind}, name:{outputF.Name}, type:{outputF.GetType()}");
int i = 0;
ImageFeatureDescriptor imgDesc = outputF as ImageFeatureDescriptor;
TensorFeatureDescriptor tfDesc = outputF as TensorFeatureDescriptor;
m_outWidth = (uint)(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width);
m_outHeight = (uint)(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height);
m_outName = outputF.Name;
Debug.WriteLine($"N: {(imgDesc == null ? tfDesc.Shape[0] : 1)}, " +
$"Channel: {(imgDesc == null ? tfDesc.Shape[1].ToString() : imgDesc.BitmapPixelFormat.ToString())}, " +
$"Height:{(imgDesc == null ? tfDesc.Shape[2] : imgDesc.Height)}, " +
$"Width: {(imgDesc == null ? tfDesc.Shape[3] : imgDesc.Width)}");
}
// ### END OF DEBUG ###
// Create output frame
_outputFrame?.Dispose();
_outputFrame = new VideoFrame(BitmapPixelFormat.Bgra8, (int)m_outWidth, (int)m_outHeight);
Debug.WriteLine($"Elapsed time: {_perfStopwatch.ElapsedMilliseconds} ms");
_isReadyForEval = true;
}
catch (Exception ex)
{
NotifyUser($"error: {ex.Message}", NotifyType.ErrorMessage);
Debug.WriteLine($"error: {ex.Message}");
}
}
_evaluationLock.Release();
}
