private async Task EvaluateImageAsync(string imagePath, string modelPath)
{
var selectedStorageFile = await StorageFile.GetFileFromPathAsync(imagePath);
SoftwareBitmap softwareBitmap;
using (var stream = await selectedStorageFile.OpenAsync(FileAccessMode.Read)) {
// Create the decoder from the stream
var decoder = await BitmapDecoder.CreateAsync(stream);
// Get the SoftwareBitmap representation of the file in BGRA8 format
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8,
BitmapAlphaMode.Premultiplied);
}
// Encapsulate the image within a VideoFrame to be bound and evaluated
var inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
if (_model == null)
{
var modelFile = await StorageFile.GetFileFromPathAsync(modelPath);
_model = new SqueezeNetModel {
LearningModel = await LearningModel.LoadFromStorageFileAsync(modelFile)
};
_model.Session = new LearningModelSession(_model.LearningModel,
new LearningModelDevice(LearningModelDeviceKind.Default));
_model.Binding = new LearningModelBinding(_model.Session);
}
if (_model == null)
{
return;
}
var input = new SqueezeNetInput {
Image = ImageFeatureValue.CreateFromVideoFrame(inputImage)
};
try {
var output = (SqueezeNetOutput)await _model.EvaluateAsync(input);
var(label, probability) = output.classLabelProbs.FirstOrDefault();
DispatchEvent(Result, probability + ", " + label);
}
catch (Exception ex) {
Trace(ex.Message, ex.StackTrace);
}
}
private async void DetectButton_Click(object sender, RoutedEventArgs e)
{
// Load model
var rootDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
var squeezeNetModel = SqueezeNetModel.CreateFromFilePath(Path.Combine(rootDir, "squeezenet1.0-9.onnx"));
// Load labels from JSON
var labels = new List <string>();
foreach (var kvp in JsonSerializer.Deserialize <Dictionary <string, string> >(File.ReadAllText(Path.Combine(rootDir, "Labels.json"))))
{
labels.Add(kvp.Value);
}
// Open image file
SqueezeNetOutput output;
using (var fileStream = File.OpenRead(filePath))
{
// Convert from FileStream to ImageFeatureValue
var decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(fileStream.AsRandomAccessStream());
using var softwareBitmap = await decoder.GetSoftwareBitmapAsync();
using var inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
var imageTensor = ImageFeatureValue.CreateFromVideoFrame(inputImage);
output = await squeezeNetModel.EvaluateAsync(new SqueezeNetInput
{
data_0 = imageTensor
});
}
// Get result, which is a list of floats with all the probabilities for all 1000 classes of SqueezeNet
var resultTensor = output.softmaxout_1;
var resultVector = resultTensor.GetAsVectorView();
// Order the 1000 results with their indexes to know which class is the highest ranked one
List <(int index, float p)> results = new List <(int, float)>();
for (int i = 0; i < resultVector.Count; i++)
{
results.Add((index: i, p: resultVector.ElementAt(i)));
}
results.Sort((a, b) => a.p switch
{
var p when p <b.p => 1,
var p when p> b.p => - 1,
_ => 0
});
async void Start()
{
try
{
// Get components
_tts = GetComponent <TextToSpeech>();
_user = GetComponent <UserInput>();
_user.Tapped += SayLastSeenObject;
// Load model
StatusBlock.text = $"Loading {SqueezeNetModel.ModelFileName} ...";
_dnnModel = new SqueezeNetModel();
await _dnnModel.LoadModelAsync(ShouldUseGpu);
StatusBlock.text = $"Loaded model. Starting camera...";
#if ENABLE_WINMD_SUPPORT
// Configure camera to return frames fitting the model input size
_mediaCapturer = new MediaCapturer();
await _mediaCapturer.StartCapturing(
_dnnModel.InputWidth,
_dnnModel.InputHeight);
StatusBlock.text = $"Camera started. Running!";
// Run processing loop in separate parallel Task
_isRunning = true;
await Task.Run(async() =>
{
while (_isRunning)
{
using (var videoFrame = _mediaCapturer.GetLatestFrame())
{
await EvaluateFrame(videoFrame);
}
}
});
#endif
}
catch (Exception ex)
{
StatusBlock.text = $"Error init: {ex.Message}";
Debug.LogError(ex);
}
}
async void Start()
{
try
{
// Get components
_tts = GetComponent <TextToSpeech>();
_user = GetComponent <UserInput>();
// Load model
StatusBlock.text = $"Loading {SqueezeNetModel.ModelFileName} ...";
_dnnModel = new SqueezeNetModel();
await _dnnModel.LoadModelAsync(ShouldUseGpu);
StatusBlock.text = $"Loaded model. Starting camera...";
#if ENABLE_WINMD_SUPPORT
// Configure camera to return frames fitting the model input size
try
{
_mediaCapturer = new MediaCapturer();
await _mediaCapturer.StartCapturing(
_dnnModel.InputWidth,
_dnnModel.InputHeight);
StatusBlock.text = $"Camera started. Running!";
}
catch (Exception ex)
{
StatusBlock.text = $"Failed to start camera: {ex.Message}. Using loaded/picked image.";
}
// Load fallback frame if there's no camera like when testing with the emulator
if (!_mediaCapturer.IsCapturing)
{
var loadedFrame = await _mediaCapturer.GetTestFrame();
}
// Run processing loop in separate parallel Task
_isRunning = true;
await Task.Run(async() =>
{
while (_isRunning)
{
if (_mediaCapturer.IsCapturing)
{
using (var videoFrame = _mediaCapturer.GetLatestFrame())
{
await EvaluateFrame(videoFrame);
}
}
// Use fallback if there's no camera like when testing with the emulator
else
{
var loadedFrame = await _mediaCapturer.GetTestFrame();
await EvaluateFrame(loadedFrame);
}
}
});
#endif
}
catch (Exception ex)
{
StatusBlock.text = $"Error init: {ex.Message}";
Debug.LogError(ex);
}
}
