private async Task EvaluateImageAsync(string imagePath, string modelPath) { var selectedStorageFile = await StorageFile.GetFileFromPathAsync(imagePath); SoftwareBitmap softwareBitmap; using (var stream = await selectedStorageFile.OpenAsync(FileAccessMode.Read)) { // Create the decoder from the stream var decoder = await BitmapDecoder.CreateAsync(stream); // Get the SoftwareBitmap representation of the file in BGRA8 format softwareBitmap = await decoder.GetSoftwareBitmapAsync(); softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); } // Encapsulate the image within a VideoFrame to be bound and evaluated var inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap); if (_model == null) { var modelFile = await StorageFile.GetFileFromPathAsync(modelPath); _model = new SqueezeNetModel { LearningModel = await LearningModel.LoadFromStorageFileAsync(modelFile) }; _model.Session = new LearningModelSession(_model.LearningModel, new LearningModelDevice(LearningModelDeviceKind.Default)); _model.Binding = new LearningModelBinding(_model.Session); } if (_model == null) { return; } var input = new SqueezeNetInput { Image = ImageFeatureValue.CreateFromVideoFrame(inputImage) }; try { var output = (SqueezeNetOutput)await _model.EvaluateAsync(input); var(label, probability) = output.classLabelProbs.FirstOrDefault(); DispatchEvent(Result, probability + ", " + label); } catch (Exception ex) { Trace(ex.Message, ex.StackTrace); } }
private async void DetectButton_Click(object sender, RoutedEventArgs e) { // Load model var rootDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); var squeezeNetModel = SqueezeNetModel.CreateFromFilePath(Path.Combine(rootDir, "squeezenet1.0-9.onnx")); // Load labels from JSON var labels = new List <string>(); foreach (var kvp in JsonSerializer.Deserialize <Dictionary <string, string> >(File.ReadAllText(Path.Combine(rootDir, "Labels.json")))) { labels.Add(kvp.Value); } // Open image file SqueezeNetOutput output; using (var fileStream = File.OpenRead(filePath)) { // Convert from FileStream to ImageFeatureValue var decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(fileStream.AsRandomAccessStream()); using var softwareBitmap = await decoder.GetSoftwareBitmapAsync(); using var inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap); var imageTensor = ImageFeatureValue.CreateFromVideoFrame(inputImage); output = await squeezeNetModel.EvaluateAsync(new SqueezeNetInput { data_0 = imageTensor }); } // Get result, which is a list of floats with all the probabilities for all 1000 classes of SqueezeNet var resultTensor = output.softmaxout_1; var resultVector = resultTensor.GetAsVectorView(); // Order the 1000 results with their indexes to know which class is the highest ranked one List <(int index, float p)> results = new List <(int, float)>(); for (int i = 0; i < resultVector.Count; i++) { results.Add((index: i, p: resultVector.ElementAt(i))); } results.Sort((a, b) => a.p switch { var p when p <b.p => 1, var p when p> b.p => - 1, _ => 0 });
async void Start() { try { // Get components _tts = GetComponent <TextToSpeech>(); _user = GetComponent <UserInput>(); _user.Tapped += SayLastSeenObject; // Load model StatusBlock.text = $"Loading {SqueezeNetModel.ModelFileName} ..."; _dnnModel = new SqueezeNetModel(); await _dnnModel.LoadModelAsync(ShouldUseGpu); StatusBlock.text = $"Loaded model. Starting camera..."; #if ENABLE_WINMD_SUPPORT // Configure camera to return frames fitting the model input size _mediaCapturer = new MediaCapturer(); await _mediaCapturer.StartCapturing( _dnnModel.InputWidth, _dnnModel.InputHeight); StatusBlock.text = $"Camera started. Running!"; // Run processing loop in separate parallel Task _isRunning = true; await Task.Run(async() => { while (_isRunning) { using (var videoFrame = _mediaCapturer.GetLatestFrame()) { await EvaluateFrame(videoFrame); } } }); #endif } catch (Exception ex) { StatusBlock.text = $"Error init: {ex.Message}"; Debug.LogError(ex); } }
async void Start() { try { // Get components _tts = GetComponent <TextToSpeech>(); _user = GetComponent <UserInput>(); // Load model StatusBlock.text = $"Loading {SqueezeNetModel.ModelFileName} ..."; _dnnModel = new SqueezeNetModel(); await _dnnModel.LoadModelAsync(ShouldUseGpu); StatusBlock.text = $"Loaded model. Starting camera..."; #if ENABLE_WINMD_SUPPORT // Configure camera to return frames fitting the model input size try { _mediaCapturer = new MediaCapturer(); await _mediaCapturer.StartCapturing( _dnnModel.InputWidth, _dnnModel.InputHeight); StatusBlock.text = $"Camera started. Running!"; } catch (Exception ex) { StatusBlock.text = $"Failed to start camera: {ex.Message}. Using loaded/picked image."; } // Load fallback frame if there's no camera like when testing with the emulator if (!_mediaCapturer.IsCapturing) { var loadedFrame = await _mediaCapturer.GetTestFrame(); } // Run processing loop in separate parallel Task _isRunning = true; await Task.Run(async() => { while (_isRunning) { if (_mediaCapturer.IsCapturing) { using (var videoFrame = _mediaCapturer.GetLatestFrame()) { await EvaluateFrame(videoFrame); } } // Use fallback if there's no camera like when testing with the emulator else { var loadedFrame = await _mediaCapturer.GetTestFrame(); await EvaluateFrame(loadedFrame); } } }); #endif } catch (Exception ex) { StatusBlock.text = $"Error init: {ex.Message}"; Debug.LogError(ex); } }