private async Task LoadAndEvaluateModelAsync(VideoFrame videoFrame) { var result = await _objectDetection.PredictImageAsync(videoFrame); var message = $"{DateTime.Now.ToLongTimeString()}{Environment.NewLine}============================={Environment.NewLine}"; PredictionModel chosenPredModel = null; foreach (var predictionModel in result) { var probability = (predictionModel.Probability * 100.0f); if (!(probability > 50)) { continue; } message += $" {predictionModel.TagName} {(predictionModel.Probability * 100.0f):#0.00}% {Environment.NewLine}"; chosenPredModel = predictionModel; } if (chosenPredModel != null) { Debug.WriteLine(message); await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => TextBlockResults.Text = $"{message}"); await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => ShowObjectDetectionBoxes(chosenPredModel)); await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, async() => await SteerRobotIfNeeded(chosenPredModel.TagName)); } }
public async Task <IList <PredictionModel> > EvaluateVideoFrameAsync(VideoFrame frame) { ModifyText(string.Empty); if (frame != null) { try { TimeRecorder.Restart(); var predictions = await Model.PredictImageAsync(frame).ConfigureAwait(false); TimeRecorder.Stop(); return(predictions); } catch (Exception ex) { var err_message = $"Error during prediction: {ex.Message}"; ModifyText(err_message); } } return(null); }
private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame) { try { var result = await _dnnModel.PredictImageAsync(videoFrame); var first = result.FirstOrDefault(); if (first != null && first.Probability > .5) { // Further process and surface results to UI on the UI thread UnityEngine.WSA.Application.InvokeOnAppThread(() => { // Measure distance between user's head and gaze ray hit point => distance to object var distMessage = string.Empty; if (_user.GazeHitDistance < 1) { distMessage = string.Format("{0:f0} {1}", _user.GazeHitDistance * 100, "centimeter"); } else { distMessage = string.Format("{0:f1} {1}", _user.GazeHitDistance, "meter"); } // Prepare strings for text and update labels var labelText = $"Found: {first.TagName}"; StatusBlock.text = labelText; }, false); } } catch (Exception ex) { //IsRunning = false; UnityEngine.WSA.Application.InvokeOnAppThread(() => { //StatusBlock.text = $"Error loop: {ex.Message}"; //Debug.LogError(ex); //Debug.LogError(videoFrame.Direct3DSurface == null ? "D3D null" : "D3D set"); //if (videoFrame.Direct3DSurface != null) //{ // Debug.LogError(videoFrame.Direct3DSurface.Description.Format); // Debug.LogError(videoFrame.Direct3DSurface.Description.Width); // Debug.LogError(videoFrame.Direct3DSurface.Description.Height); //} }, false); } }
public async Task AnalyseONNX(string imagePath) { #if UNITY_UWP try { if (!IsReady) { return; } Debug.Log("Analyzing..."); SoftwareBitmap softwareBitmap = null; using (var stream = new InMemoryRandomAccessStream()) using (var memStream = new InMemoryRandomAccessStream()) { imageBytes = GetImageAsByteArray(imagePath); await stream.WriteAsync(imageBytes.AsBuffer()); stream.Seek(0); var decoder = await BitmapDecoder.CreateAsync(stream); //コメント部分はなくてもいいらしい(入力が416x416画像のため縮尺を変更するロジック //BitmapEncoder encoder = await BitmapEncoder.CreateForTranscodingAsync(memStream, decoder); //encoder.BitmapTransform.ScaledWidth = 416; //encoder.BitmapTransform.ScaledHeight = 416; //await encoder.FlushAsync(); //memStream.Seek(0); //var decorder = await BitmapDecoder.CreateAsync(memStream); softwareBitmap = await decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Straight); } _predictionModels = await _objectDetection.PredictImageAsync(VideoFrame.CreateWithSoftwareBitmap(softwareBitmap)); } finally { // Stop the analysis process ImageCapture.Instance.ResetImageCapture(); } #endif }
public async Task <string> ProcessSoftwareBitmap(SoftwareBitmap bitmap, SoftwareBitmap originalBitmap, MainPage mainPage) { string ret = null; if (!Windows.Foundation.Metadata.ApiInformation.IsMethodPresent("Windows.Media.VideoFrame", "CreateWithSoftwareBitmap")) { return("hi\n"); } //Convert SoftwareBitmap into VideoFrame using (VideoFrame frame = VideoFrame.CreateWithSoftwareBitmap(bitmap)) { try { if (file == null) { file = await Windows.Storage.StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///AIModel/model.onnx")); await objectDetection.Init(file); } var output = await objectDetection.PredictImageAsync(frame); if (output != null) { mainPage.UpdateTextbox("output is not null.\n"); ret = await UpdateResult(output, mainPage, bitmap, originalBitmap); } else { mainPage.UpdateTextbox("No result.\n"); } } catch (Exception e) { string s = e.Message; mainPage.ShowMessagePopup(e.Message); } } return(ret); }
private async Task LoadAndEvaluateModelAsync(VideoFrame videoFrame) { //Reset the components if (prediction != null) { prediction = null; } var result = await _objectDetection.PredictImageAsync(videoFrame); //When recognition is successful if (result.Count != 0) { //성공 if (result[0].Probability > probabilityThreshold) { prediction = result[0]; ImageCapture.Instance.iState = 1; //Debug.Log("result : " + prediction.TagName + prediction.Probability); } //낮은 확률 else { ImageCapture.Instance.iState = -1; //Debug.Log("low probablility : " + result[0].TagName + result[0].Probability); } } //못찾음 else { ImageCapture.Instance.iState = -2; //Debug.Log("nothing"); } bFinalizeStart = true; }
private async void BdRecognize_Click(object sender, RoutedEventArgs e) { bdResult.Text = string.Empty; bdCanvas.Background = null; try { Uri uri = new Uri(bdUri.Text); var response = await new HttpClient().GetAsync(uri); var stream = (await response.Content.ReadAsStreamAsync()).AsRandomAccessStream(); var decoder = await BitmapDecoder.CreateAsync(stream); var softwareBitmap = await decoder.GetSoftwareBitmapAsync(); WriteableBitmap dispalySource = new WriteableBitmap(softwareBitmap.PixelWidth, softwareBitmap.PixelHeight); softwareBitmap.CopyToBuffer(dispalySource.PixelBuffer); var brush = new ImageBrush { ImageSource = dispalySource, AlignmentX = 0, AlignmentY = 0, Stretch = Stretch.None }; bdCanvas.Background = brush; bdCanvas.Children.Clear(); VideoFrame videoFrame = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap); var prediction = await od.PredictImageAsync(videoFrame); var acceptablePredictions = prediction.Where(p => p.Probability >= 0.5).ToList(); //Only show the ones with more than 50% probability DrawBoxes(acceptablePredictions, softwareBitmap); } catch (Exception ex) { MessageDialog a = new MessageDialog(String.Format("error: \n{0}", ex.ToString())); await a.ShowAsync(); } }
public async void StartPullCameraFrames() { stopwatchTake.Start(); // The position of the canvas also needs to be adjusted, as the size adjustment affects the centering of the control await Task.Run(async() => { await Task.Delay(3000); for (; ;) // Forever = While the app runs { await Task.Delay(100); try { if (!isPreviewing || mediaCapture == null) { continue; } //if (mediaCapture != null && mediaCapture.CameraStreamState != Windows.Media.Devices.CameraStreamState.Streaming) //{ // isPreviewing = false; // Debug.WriteLine(mediaCapture.CameraStreamState); //} var previewFrame = await mediaCapture.GetPreviewFrameAsync(videoFrame); previewFrame.SoftwareBitmap.CopyToBuffer(buffer); bitmap = SoftwareBitmap.CreateCopyFromBuffer(buffer, BitmapPixelFormat.Bgra8, frameWidth, frameHeight); var frameAI = VideoFrame.CreateWithSoftwareBitmap(bitmap); bitmap.CopyToBuffer(AIbuffer); var predicts = await objectDetection.PredictImageAsync(frameAI).ConfigureAwait(false); var loss = predicts.OrderBy(x => - (x.Probability)); // var good = loss.Where(x => { return (x.Probability > 0.5f && x.TagName == "No Mask"); }); if (loss.Count() > 0) { var first = loss.First(); RunOnMainThread(async() => { //await mediaCapture.VideoDeviceController.RegionsOfInterestControl.ClearRegionsAsync(); //// focus in the center of the screen //await mediaCapture.VideoDeviceController.RegionsOfInterestControl.SetRegionsAsync( // new[]{ // new RegionOfInterest() {Bounds = new Rect(first.BoundingBox.Left,first.BoundingBox.Top,0.02,0.02) } // }); }); } else { RunOnMainThread(async() => { //await mediaCapture.VideoDeviceController.RegionsOfInterestControl.ClearRegionsAsync(); //// focus in the center of the screen //await mediaCapture.VideoDeviceController.RegionsOfInterestControl.SetRegionsAsync( // new[]{ // new RegionOfInterest() {Bounds = new Rect(0.49f,0.49f,0.02,0.02) } // }); }); } // var lossStr = string.Join(", ", loss.Select(l => l.TagName + " " + (l.Probability * 100.0f).ToString("#0.00") + "%" + " Border:" + string.Format("x:{0},y:{1},w:{2},h:{3}", l.BoundingBox.Left, l.BoundingBox.Top, l.BoundingBox.Width, l.BoundingBox.Height))); //string message = $"Evaluation took {TimeRecorder.ElapsedMilliseconds}ms to execute, Predictions: {lossStr}."; //Debug.WriteLine(lossStr); RunOnMainThread(() => { ClearCanvas(); //DrawCanvas(0f, 0f, 1f, 1f); bool noMask = false; bool take = false; foreach (var pre in predicts) { if (pre.Probability >= 0.5f && pre.TagName == "No Mask") { noMask = true; if (AutoNoMaskCheckBox.IsChecked.Value) { take = true; } DrawCanvas(pre.BoundingBox.Left, pre.BoundingBox.Top, pre.BoundingBox.Width, pre.BoundingBox.Height, Colors.Red); } if (pre.Probability >= 0.5f && pre.TagName == "Mask") { if (AutoMaskCheckBox.IsChecked.Value) { take = true; } DrawCanvas(pre.BoundingBox.Left, pre.BoundingBox.Top, pre.BoundingBox.Width, pre.BoundingBox.Height, Colors.Green); } } if (noMask) { WarningImage.Visibility = Visibility.Visible; } else { WarningImage.Visibility = Visibility.Collapsed; } if (take && stopwatchTake.ElapsedMilliseconds >= 500) { System.Diagnostics.Debug.WriteLine("Auto Take Photo"); TakePhoto(AIbuffer.ToArray()); stopwatchTake.Restart(); } }); } catch (Exception ex) { System.Diagnostics.Debug.WriteLine("log: " + ex.StackTrace); return; } finally { } } }); }
public async Task <IList <IDictionary <string, float> > > processSoftwareBitmap(SoftwareBitmap bitmap) { SoftwareBitmap bitmapToProcess = SoftwareBitmap.Convert(bitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); uint cropSize = Convert.ToUInt32(bitmap.PixelHeight); uint sideDiff = Convert.ToUInt32(bitmap.PixelWidth - bitmap.PixelHeight); SoftwareBitmap refCropedImage = await cropImageWithPixel(bitmap, sideDiff, 0, cropSize, cropSize); bitmapToProcess = await resizeImgForDetect(refCropedImage); //Windows.Storage.StorageFile testFile = await folder.CreateFileAsync("test.jpg", Windows.Storage.CreationCollisionOption.ReplaceExisting); //using (IRandomAccessStream stream = await testFile.OpenAsync(FileAccessMode.ReadWrite)) //{ // Guid BitmapEncoderGuid = BitmapEncoder.JpegEncoderId; // BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoderGuid, stream); // encoder.SetSoftwareBitmap(refCropedImage); // await encoder.FlushAsync(); //} //Convert SoftwareBitmap into VideoFrame using (VideoFrame frame = VideoFrame.CreateWithSoftwareBitmap(bitmapToProcess)) { if (frame == null) { return(null); } try { modelInput input = new modelInput(); input.data = ImageFeatureValue.CreateFromVideoFrame(frame); if (input.data == null) { return(null); } IList <PredictionModel> result = await detection.PredictImageAsync(frame); //await saveDebugImage(refCropedImage, result); var reader = new QRCodeMultiReader(); for (int predictIdx = 0; predictIdx < result.Count; predictIdx++) { PredictionModel predictInfo = result[predictIdx]; SoftwareBitmap cropedImage = await cropImage(refCropedImage, Math.Max(predictInfo.BoundingBox.Left, 0), Math.Max(predictInfo.BoundingBox.Top, 0), predictInfo.BoundingBox.Width, predictInfo.BoundingBox.Height); //Windows.Storage.StorageFile testFile = await folder.CreateFileAsync("test" + predictIdx + ".jpg", Windows.Storage.CreationCollisionOption.ReplaceExisting); //using (IRandomAccessStream stream = await testFile.OpenAsync(FileAccessMode.ReadWrite)) //{ // Guid BitmapEncoderGuid = BitmapEncoder.JpegEncoderId; // BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoderGuid, stream); // encoder.SetSoftwareBitmap(cropedImage); // await encoder.FlushAsync(); //} var source = new SoftwareBitmapLuminanceSource(cropedImage); HybridBinarizer binarizer = new HybridBinarizer(source); var results = reader.decodeMultiple(new BinaryBitmap(binarizer)); var info = parseZXingResult(results); string locText = info["locText"]; string boxText = info["boxText"]; if (locText.Equals("") || !locText.StartsWith(zone)) { continue; } if (finalResult.ContainsKey(locText)) { if (finalResult[locText].Equals("")) { finalResult[locText] = boxText; } } else { finalResult.Add(locText, boxText); } } } catch (InvalidOperationException ie) { return(null); } catch (Exception e) { return(null); } finally { bitmap.Dispose(); } } return(null); }
private async Task <int> ScanFrame() { var frame = Drone.Instance.GetLiveFrame(); string resultText = ""; int returncode = 0; if (frame.Width == 0 || frame.Height == 0 || frame.Data == null || frame.Data.Length == 0) { return(-1); } var data = frame.Data; var height = frame.Height; var width = frame.Width; // resize image to (416,416) var mat = new Mat(height, width, MatType.CV_8UC4, data); var gray = new Mat(); Mat resizemat = new Mat(); resizemat = mat.Clone(); Cv2.CvtColor(mat, mat, ColorConversionCodes.RGBA2BGRA); Cv2.CvtColor(mat, gray, ColorConversionCodes.RGBA2GRAY); resizemat = resizemat.Resize(new Size(416, 416), 0, 0, InterpolationFlags.Linear); //Cv2.ImShow("hell0", mat); //Cv2.WaitKey(500); // Mat -> softwarebitmap var resizedLength = resizemat.Rows * resizemat.Cols * resizemat.ElemSize(); var buffer = new byte[resizedLength]; Marshal.Copy(resizemat.Data, buffer, 0, resizedLength); var bm = SoftwareBitmap.CreateCopyFromBuffer(buffer.AsBuffer(), BitmapPixelFormat.Rgba8, resizemat.Cols, resizemat.Rows); try { IList <PredictionModel> outputlist = await objectDetection.PredictImageAsync(VideoFrame.CreateWithSoftwareBitmap(bm)); foreach (var output in outputlist) { //chop origin image Mat chop = ChopOutData(gray, output.BoundingBox, height, width); //Cv2.ImShow("hello", chop); //Cv2.ImShow("Origin", gray); //Cv2.WaitKey(500); var chop_image_length = chop.Rows * chop.Cols * chop.ElemSize(); var chop_image_buffer = new byte[chop_image_length]; Marshal.Copy(chop.Data, chop_image_buffer, 0, chop_image_length); //var gray_length = gray.Rows * gray.Cols * gray.ElemSize(); //var gray_buffer = new byte[gray_length]; //Marshal.Copy(gray.Data, gray_buffer, 0, gray_length); var reader = new BarcodeReader(); reader.LicenseKeys = DynamsoftAppKey; //TextResult[] result = // br.DecodeBuffer(gray_buffer, gray.Cols, gray.Rows, gray.Cols * gray.ElemSize(), EnumImagePixelFormat.IPF_GrayScaled, ""); //chop TextResult[] result = reader.DecodeBuffer(chop_image_buffer, chop.Cols, chop.Rows, chop.Cols * chop.ElemSize(), EnumImagePixelFormat.IPF_GrayScaled, ""); int index = 0; switch (output.TagName) { case "Box": if (result.Length < 2) { returncode = -1; break; } string LocationTag = ""; List <string> CartonTag = new List <string>(); foreach (var pick in result) { if (LocationTagReg.Match(pick.BarcodeText).Success) { LocationTag = pick.BarcodeText; } else if (CartonTagReg.Match(pick.BarcodeText).Success) { CartonTag.Add(pick.BarcodeText); } } if (LocationTag == "" || CartonTag.Count < 1) { returncode = -2; break; } index = ResultLocation.IndexOf(LocationTag); if (index == -1) { ResultLocation.Add(LocationTag); ResultCarton.Add(CartonTag); } else { ResultCarton[index] = CartonTag; } break; case "Nobox": if (result.Length < 1 || !LocationTagReg.Match(result[0].BarcodeText).Success) { returncode = -2; break; } int i_dx = ResultLocation.IndexOf(result[0].BarcodeText); if (i_dx == -1) { ResultLocation.Add(result[0].BarcodeText); ResultCarton.Add(new List <string>()); } else { ResultCarton[i_dx] = new List <string>(); } break; } //UpdateDecodeText(); //lock (updateLock) } lock (updateLock) { int index = ResultLocation.Count; for (int i = 0; i < index; i++) { resultText += ResultLocation[i]; foreach (string carton in ResultCarton[i]) { resultText += "," + carton; } resultText += "\n"; } var qrcode = new QrcodeDetection(resultText, frame); if (null != QrcodeDetected) { QrcodeDetected.Invoke(qrcode); } //var results = new ResultLists(ResultLocation, ResultCarton); //if (null != QrcodeDetected) //{ // ResultLists.Invoke(qrcode); //} } } catch (Exception ex) { //lock (updateLock) //{ // var qrcode = new QrcodeDetection(ex.Message, frame); // if (null != QrcodeDetected) // { // QrcodeDetected.Invoke(qrcode); // } //} //Debug.WriteLine($"qrcode: {ex.ToString()}"); return(-1); } return(returncode); }
private async void TbRecognize_Click(object sender, RoutedEventArgs e) { tbResult.Text = string.Empty; imgPreview.Source = null; List <UIElement> removedItems = new List <UIElement>(); foreach (var item in gridMain.Children) { if (item is Border || (item is TextBlock && item != tbResult)) { removedItems.Add(item); } } foreach (var item in removedItems) { gridMain.Children.Remove(item); } try { Uri uri = new Uri(tbUrl.Text); var response = await new HttpClient().GetAsync(uri); var stream = (await response.Content.ReadAsStreamAsync()).AsRandomAccessStream(); var decoder = await BitmapDecoder.CreateAsync(stream); var softwareBitmap = await decoder.GetSoftwareBitmapAsync(); VideoFrame videoFrame = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap); //ImageFeatureValue imageFreatureValue = // ImageFeatureValue.CreateFromVideoFrame(videoFrame); var output = await model.PredictImageAsync(videoFrame); var stringResult = new StringBuilder(); var pbWidth = imgPreview.ActualWidth; var pbHeight = imgPreview.ActualHeight; Random rand = new Random(); foreach (var item in output) { if (item.Probability > 0.5) { int x = (int)Math.Round(item.BoundingBox.Left * pbWidth); int y = (int)Math.Round(item.BoundingBox.Top * pbHeight); int w = (int)Math.Round(item.BoundingBox.Width * pbWidth); int h = (int)Math.Round(item.BoundingBox.Height * pbHeight); stringResult.AppendLine(string.Format("{0}: {1}, [{2},{3},{4},{5}]", item.TagName, item.Probability, x, y, w, h)); Rectangle rect = new Rectangle(x, y, w, h); Color color = Color.FromKnownColor((KnownColor)rand.Next(1, 175)); Windows.UI.Color uiColor = Windows.UI.Color.FromArgb(color.A, color.R, color.G, color.B); Border b = new Border(); b.BorderBrush = new SolidColorBrush(uiColor); b.BorderThickness = new Thickness(2); b.Width = w; b.Height = h; b.Margin = new Thickness(x + imgPreview.Margin.Left, y + imgPreview.Margin.Top, 0, 0); b.HorizontalAlignment = HorizontalAlignment.Left; b.VerticalAlignment = VerticalAlignment.Top; gridMain.Children.Add(b); TextBlock tb = new TextBlock(); tb.Margin = new Thickness(b.Margin.Left + 2, b.Margin.Top + 2, 0, 0); tb.HorizontalAlignment = HorizontalAlignment.Left; tb.VerticalAlignment = VerticalAlignment.Top; tb.Text = item.TagName; tb.IsColorFontEnabled = true; tb.Foreground = b.BorderBrush; gridMain.Children.Add(tb); } } tbResult.Text = stringResult.ToString(); var sbs = new SoftwareBitmapSource(); var imagePreview = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); await sbs.SetBitmapAsync(imagePreview); imgPreview.Source = sbs; } catch (Exception ex) { MessageDialog a = new MessageDialog(String.Format("error: \n{0}", ex.ToString())); await a.ShowAsync(); } }