private async Task LoadAndEvaluateModelAsync(VideoFrame videoFrame)
{
var result = await _objectDetection.PredictImageAsync(videoFrame);
var message = $"{DateTime.Now.ToLongTimeString()}{Environment.NewLine}============================={Environment.NewLine}";
PredictionModel chosenPredModel = null;
foreach (var predictionModel in result)
{
var probability = (predictionModel.Probability * 100.0f);
if (!(probability > 50))
{
continue;
}
message += $" {predictionModel.TagName} {(predictionModel.Probability * 100.0f):#0.00}% {Environment.NewLine}";
chosenPredModel = predictionModel;
}
if (chosenPredModel != null)
{
Debug.WriteLine(message);
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => TextBlockResults.Text = $"{message}");
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => ShowObjectDetectionBoxes(chosenPredModel));
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, async() => await SteerRobotIfNeeded(chosenPredModel.TagName));
}
}
public async Task <IList <PredictionModel> > EvaluateVideoFrameAsync(VideoFrame frame)
{
ModifyText(string.Empty);
if (frame != null)
{
try
{
TimeRecorder.Restart();
var predictions = await Model.PredictImageAsync(frame).ConfigureAwait(false);
TimeRecorder.Stop();
return(predictions);
}
catch (Exception ex)
{
var err_message = $"Error during prediction: {ex.Message}";
ModifyText(err_message);
}
}
return(null);
}
private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame)
{
try
{
var result = await _dnnModel.PredictImageAsync(videoFrame);
var first = result.FirstOrDefault();
if (first != null && first.Probability > .5)
{
// Further process and surface results to UI on the UI thread
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
// Measure distance between user's head and gaze ray hit point => distance to object
var distMessage = string.Empty;
if (_user.GazeHitDistance < 1)
{
distMessage = string.Format("{0:f0} {1}", _user.GazeHitDistance * 100, "centimeter");
}
else
{
distMessage = string.Format("{0:f1} {1}", _user.GazeHitDistance, "meter");
}
// Prepare strings for text and update labels
var labelText = $"Found: {first.TagName}";
StatusBlock.text = labelText;
}, false);
}
}
catch (Exception ex)
{
//IsRunning = false;
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
//StatusBlock.text = $"Error loop: {ex.Message}";
//Debug.LogError(ex);
//Debug.LogError(videoFrame.Direct3DSurface == null ? "D3D null" : "D3D set");
//if (videoFrame.Direct3DSurface != null)
//{
// Debug.LogError(videoFrame.Direct3DSurface.Description.Format);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Width);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Height);
//}
}, false);
}
}
public async Task AnalyseONNX(string imagePath)
{
#if UNITY_UWP
try
{
if (!IsReady)
{
return;
}
Debug.Log("Analyzing...");
SoftwareBitmap softwareBitmap = null;
using (var stream = new InMemoryRandomAccessStream())
using (var memStream = new InMemoryRandomAccessStream())
{
imageBytes = GetImageAsByteArray(imagePath);
await stream.WriteAsync(imageBytes.AsBuffer());
stream.Seek(0);
var decoder = await BitmapDecoder.CreateAsync(stream);
//コメント部分はなくてもいいらしい（入力が416x416画像のため縮尺を変更するロジック
//BitmapEncoder encoder = await BitmapEncoder.CreateForTranscodingAsync(memStream, decoder);
//encoder.BitmapTransform.ScaledWidth = 416;
//encoder.BitmapTransform.ScaledHeight = 416;
//await encoder.FlushAsync();
//memStream.Seek(0);
//var decorder = await BitmapDecoder.CreateAsync(memStream);
softwareBitmap =
await decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Straight);
}
_predictionModels =
await _objectDetection.PredictImageAsync(VideoFrame.CreateWithSoftwareBitmap(softwareBitmap));
}
finally
{
// Stop the analysis process
ImageCapture.Instance.ResetImageCapture();
}
#endif
}
public async Task <string> ProcessSoftwareBitmap(SoftwareBitmap bitmap, SoftwareBitmap originalBitmap, MainPage mainPage)
{
string ret = null;
if (!Windows.Foundation.Metadata.ApiInformation.IsMethodPresent("Windows.Media.VideoFrame", "CreateWithSoftwareBitmap"))
{
return("hi\n");
}
//Convert SoftwareBitmap into VideoFrame
using (VideoFrame frame = VideoFrame.CreateWithSoftwareBitmap(bitmap))
{
try
{
if (file == null)
{
file = await Windows.Storage.StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///AIModel/model.onnx"));
await objectDetection.Init(file);
}
var output = await objectDetection.PredictImageAsync(frame);
if (output != null)
{
mainPage.UpdateTextbox("output is not null.\n");
ret = await UpdateResult(output, mainPage, bitmap, originalBitmap);
}
else
{
mainPage.UpdateTextbox("No result.\n");
}
}
catch (Exception e)
{
string s = e.Message;
mainPage.ShowMessagePopup(e.Message);
}
}
return(ret);
}
private async Task LoadAndEvaluateModelAsync(VideoFrame videoFrame)
{
//Reset the components
if (prediction != null)
{
prediction = null;
}
var result = await _objectDetection.PredictImageAsync(videoFrame);
//When recognition is successful
if (result.Count != 0)
{
//성공
if (result[0].Probability > probabilityThreshold)
{
prediction = result[0];
ImageCapture.Instance.iState = 1;
//Debug.Log("result : " + prediction.TagName + prediction.Probability);
}
//낮은 확률
else
{
ImageCapture.Instance.iState = -1;
//Debug.Log("low probablility : " + result[0].TagName + result[0].Probability);
}
}
//못찾음
else
{
ImageCapture.Instance.iState = -2;
//Debug.Log("nothing");
}
bFinalizeStart = true;
}
private async void BdRecognize_Click(object sender, RoutedEventArgs e)
{
bdResult.Text = string.Empty;
bdCanvas.Background = null;
try
{
Uri uri = new Uri(bdUri.Text);
var response = await new HttpClient().GetAsync(uri);
var stream = (await response.Content.ReadAsStreamAsync()).AsRandomAccessStream();
var decoder = await BitmapDecoder.CreateAsync(stream);
var softwareBitmap = await decoder.GetSoftwareBitmapAsync();
WriteableBitmap dispalySource = new WriteableBitmap(softwareBitmap.PixelWidth, softwareBitmap.PixelHeight);
softwareBitmap.CopyToBuffer(dispalySource.PixelBuffer);
var brush = new ImageBrush
{
ImageSource = dispalySource,
AlignmentX = 0,
AlignmentY = 0,
Stretch = Stretch.None
};
bdCanvas.Background = brush;
bdCanvas.Children.Clear();
VideoFrame videoFrame = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
var prediction = await od.PredictImageAsync(videoFrame);
var acceptablePredictions = prediction.Where(p => p.Probability >= 0.5).ToList(); //Only show the ones with more than 50% probability
DrawBoxes(acceptablePredictions, softwareBitmap);
}
catch (Exception ex)
{
MessageDialog a = new MessageDialog(String.Format("error: \n{0}", ex.ToString()));
await a.ShowAsync();
}
}
public async void StartPullCameraFrames()
{
stopwatchTake.Start();
// The position of the canvas also needs to be adjusted, as the size adjustment affects the centering of the control
await Task.Run(async() =>
{
await Task.Delay(3000);
for (; ;) // Forever = While the app runs
{
await Task.Delay(100);
try
{
if (!isPreviewing || mediaCapture == null)
{
continue;
}
//if (mediaCapture != null && mediaCapture.CameraStreamState != Windows.Media.Devices.CameraStreamState.Streaming)
//{
// isPreviewing = false;
// Debug.WriteLine(mediaCapture.CameraStreamState);
//}
var previewFrame = await mediaCapture.GetPreviewFrameAsync(videoFrame);
previewFrame.SoftwareBitmap.CopyToBuffer(buffer);
bitmap = SoftwareBitmap.CreateCopyFromBuffer(buffer, BitmapPixelFormat.Bgra8, frameWidth, frameHeight);
var frameAI = VideoFrame.CreateWithSoftwareBitmap(bitmap);
bitmap.CopyToBuffer(AIbuffer);
var predicts = await objectDetection.PredictImageAsync(frameAI).ConfigureAwait(false);
var loss = predicts.OrderBy(x => - (x.Probability));
// var good = loss.Where(x => { return (x.Probability > 0.5f && x.TagName == "No Mask"); });
if (loss.Count() > 0)
{
var first = loss.First();
RunOnMainThread(async() =>
{
//await mediaCapture.VideoDeviceController.RegionsOfInterestControl.ClearRegionsAsync();
//// focus in the center of the screen
//await mediaCapture.VideoDeviceController.RegionsOfInterestControl.SetRegionsAsync(
// new[]{
// new RegionOfInterest() {Bounds = new Rect(first.BoundingBox.Left,first.BoundingBox.Top,0.02,0.02) }
// });
});
}
else
{
RunOnMainThread(async() =>
{
//await mediaCapture.VideoDeviceController.RegionsOfInterestControl.ClearRegionsAsync();
//// focus in the center of the screen
//await mediaCapture.VideoDeviceController.RegionsOfInterestControl.SetRegionsAsync(
// new[]{
// new RegionOfInterest() {Bounds = new Rect(0.49f,0.49f,0.02,0.02) }
// });
});
}
// var lossStr = string.Join(", ", loss.Select(l => l.TagName + " " + (l.Probability * 100.0f).ToString("#0.00") + "%" + " Border:" + string.Format("x:{0},y:{1},w:{2},h:{3}", l.BoundingBox.Left, l.BoundingBox.Top, l.BoundingBox.Width, l.BoundingBox.Height)));
//string message = $"Evaluation took {TimeRecorder.ElapsedMilliseconds}ms to execute, Predictions: {lossStr}.";
//Debug.WriteLine(lossStr);
RunOnMainThread(() =>
{
ClearCanvas();
//DrawCanvas(0f, 0f, 1f, 1f);
bool noMask = false;
bool take = false;
foreach (var pre in predicts)
{
if (pre.Probability >= 0.5f && pre.TagName == "No Mask")
{
noMask = true;
if (AutoNoMaskCheckBox.IsChecked.Value)
{
take = true;
}
DrawCanvas(pre.BoundingBox.Left, pre.BoundingBox.Top, pre.BoundingBox.Width, pre.BoundingBox.Height, Colors.Red);
}
if (pre.Probability >= 0.5f && pre.TagName == "Mask")
{
if (AutoMaskCheckBox.IsChecked.Value)
{
take = true;
}
DrawCanvas(pre.BoundingBox.Left, pre.BoundingBox.Top, pre.BoundingBox.Width, pre.BoundingBox.Height, Colors.Green);
}
}
if (noMask)
{
WarningImage.Visibility = Visibility.Visible;
}
else
{
WarningImage.Visibility = Visibility.Collapsed;
}
if (take && stopwatchTake.ElapsedMilliseconds >= 500)
{
System.Diagnostics.Debug.WriteLine("Auto Take Photo");
TakePhoto(AIbuffer.ToArray());
stopwatchTake.Restart();
}
});
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine("log: " + ex.StackTrace);
return;
}
finally
{
}
}
});
}
public async Task <IList <IDictionary <string, float> > > processSoftwareBitmap(SoftwareBitmap bitmap)
{
SoftwareBitmap bitmapToProcess = SoftwareBitmap.Convert(bitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
uint cropSize = Convert.ToUInt32(bitmap.PixelHeight);
uint sideDiff = Convert.ToUInt32(bitmap.PixelWidth - bitmap.PixelHeight);
SoftwareBitmap refCropedImage = await cropImageWithPixel(bitmap,
sideDiff,
0,
cropSize,
cropSize);
bitmapToProcess = await resizeImgForDetect(refCropedImage);
//Windows.Storage.StorageFile testFile = await folder.CreateFileAsync("test.jpg", Windows.Storage.CreationCollisionOption.ReplaceExisting);
//using (IRandomAccessStream stream = await testFile.OpenAsync(FileAccessMode.ReadWrite))
//{
// Guid BitmapEncoderGuid = BitmapEncoder.JpegEncoderId;
// BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoderGuid, stream);
// encoder.SetSoftwareBitmap(refCropedImage);
// await encoder.FlushAsync();
//}
//Convert SoftwareBitmap into VideoFrame
using (VideoFrame frame = VideoFrame.CreateWithSoftwareBitmap(bitmapToProcess))
{
if (frame == null)
{
return(null);
}
try
{
modelInput input = new modelInput();
input.data = ImageFeatureValue.CreateFromVideoFrame(frame);
if (input.data == null)
{
return(null);
}
IList <PredictionModel> result = await detection.PredictImageAsync(frame);
//await saveDebugImage(refCropedImage, result);
var reader = new QRCodeMultiReader();
for (int predictIdx = 0; predictIdx < result.Count; predictIdx++)
{
PredictionModel predictInfo = result[predictIdx];
SoftwareBitmap cropedImage = await cropImage(refCropedImage,
Math.Max(predictInfo.BoundingBox.Left, 0),
Math.Max(predictInfo.BoundingBox.Top, 0),
predictInfo.BoundingBox.Width,
predictInfo.BoundingBox.Height);
//Windows.Storage.StorageFile testFile = await folder.CreateFileAsync("test" + predictIdx + ".jpg", Windows.Storage.CreationCollisionOption.ReplaceExisting);
//using (IRandomAccessStream stream = await testFile.OpenAsync(FileAccessMode.ReadWrite))
//{
// Guid BitmapEncoderGuid = BitmapEncoder.JpegEncoderId;
// BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoderGuid, stream);
// encoder.SetSoftwareBitmap(cropedImage);
// await encoder.FlushAsync();
//}
var source = new SoftwareBitmapLuminanceSource(cropedImage);
HybridBinarizer binarizer = new HybridBinarizer(source);
var results = reader.decodeMultiple(new BinaryBitmap(binarizer));
var info = parseZXingResult(results);
string locText = info["locText"];
string boxText = info["boxText"];
if (locText.Equals("") || !locText.StartsWith(zone))
{
continue;
}
if (finalResult.ContainsKey(locText))
{
if (finalResult[locText].Equals(""))
{
finalResult[locText] = boxText;
}
}
else
{
finalResult.Add(locText, boxText);
}
}
}
catch (InvalidOperationException ie)
{
return(null);
}
catch (Exception e)
{
return(null);
}
finally
{
bitmap.Dispose();
}
}
return(null);
}
private async Task <int> ScanFrame()
{
var frame = Drone.Instance.GetLiveFrame();
string resultText = "";
int returncode = 0;
if (frame.Width == 0 || frame.Height == 0 || frame.Data == null || frame.Data.Length == 0)
{
return(-1);
}
var data = frame.Data;
var height = frame.Height;
var width = frame.Width;
// resize image to (416,416)
var mat = new Mat(height, width, MatType.CV_8UC4, data);
var gray = new Mat();
Mat resizemat = new Mat();
resizemat = mat.Clone();
Cv2.CvtColor(mat, mat, ColorConversionCodes.RGBA2BGRA);
Cv2.CvtColor(mat, gray, ColorConversionCodes.RGBA2GRAY);
resizemat = resizemat.Resize(new Size(416, 416), 0, 0, InterpolationFlags.Linear);
//Cv2.ImShow("hell0", mat);
//Cv2.WaitKey(500);
// Mat -> softwarebitmap
var resizedLength = resizemat.Rows * resizemat.Cols * resizemat.ElemSize();
var buffer = new byte[resizedLength];
Marshal.Copy(resizemat.Data, buffer, 0, resizedLength);
var bm = SoftwareBitmap.CreateCopyFromBuffer(buffer.AsBuffer(), BitmapPixelFormat.Rgba8, resizemat.Cols, resizemat.Rows);
try
{
IList <PredictionModel> outputlist = await objectDetection.PredictImageAsync(VideoFrame.CreateWithSoftwareBitmap(bm));
foreach (var output in outputlist)
{
//chop origin image
Mat chop = ChopOutData(gray, output.BoundingBox, height, width);
//Cv2.ImShow("hello", chop);
//Cv2.ImShow("Origin", gray);
//Cv2.WaitKey(500);
var chop_image_length = chop.Rows * chop.Cols * chop.ElemSize();
var chop_image_buffer = new byte[chop_image_length];
Marshal.Copy(chop.Data, chop_image_buffer, 0, chop_image_length);
//var gray_length = gray.Rows * gray.Cols * gray.ElemSize();
//var gray_buffer = new byte[gray_length];
//Marshal.Copy(gray.Data, gray_buffer, 0, gray_length);
var reader = new BarcodeReader();
reader.LicenseKeys = DynamsoftAppKey;
//TextResult[] result =
// br.DecodeBuffer(gray_buffer, gray.Cols, gray.Rows, gray.Cols * gray.ElemSize(), EnumImagePixelFormat.IPF_GrayScaled, "");
//chop
TextResult[] result =
reader.DecodeBuffer(chop_image_buffer, chop.Cols, chop.Rows, chop.Cols * chop.ElemSize(), EnumImagePixelFormat.IPF_GrayScaled, "");
int index = 0;
switch (output.TagName)
{
case "Box":
if (result.Length < 2)
{
returncode = -1;
break;
}
string LocationTag = "";
List <string> CartonTag = new List <string>();
foreach (var pick in result)
{
if (LocationTagReg.Match(pick.BarcodeText).Success)
{
LocationTag = pick.BarcodeText;
}
else if (CartonTagReg.Match(pick.BarcodeText).Success)
{
CartonTag.Add(pick.BarcodeText);
}
}
if (LocationTag == "" || CartonTag.Count < 1)
{
returncode = -2;
break;
}
index = ResultLocation.IndexOf(LocationTag);
if (index == -1)
{
ResultLocation.Add(LocationTag);
ResultCarton.Add(CartonTag);
}
else
{
ResultCarton[index] = CartonTag;
}
break;
case "Nobox":
if (result.Length < 1 || !LocationTagReg.Match(result[0].BarcodeText).Success)
{
returncode = -2;
break;
}
int i_dx = ResultLocation.IndexOf(result[0].BarcodeText);
if (i_dx == -1)
{
ResultLocation.Add(result[0].BarcodeText);
ResultCarton.Add(new List <string>());
}
else
{
ResultCarton[i_dx] = new List <string>();
}
break;
}
//UpdateDecodeText();
//lock (updateLock)
}
lock (updateLock)
{
int index = ResultLocation.Count;
for (int i = 0; i < index; i++)
{
resultText += ResultLocation[i];
foreach (string carton in ResultCarton[i])
{
resultText += "," + carton;
}
resultText += "\n";
}
var qrcode = new QrcodeDetection(resultText, frame);
if (null != QrcodeDetected)
{
QrcodeDetected.Invoke(qrcode);
}
//var results = new ResultLists(ResultLocation, ResultCarton);
//if (null != QrcodeDetected)
//{
// ResultLists.Invoke(qrcode);
//}
}
}
catch (Exception ex)
{
//lock (updateLock)
//{
// var qrcode = new QrcodeDetection(ex.Message, frame);
// if (null != QrcodeDetected)
// {
// QrcodeDetected.Invoke(qrcode);
// }
//}
//Debug.WriteLine($"qrcode: {ex.ToString()}");
return(-1);
}
return(returncode);
}
private async void TbRecognize_Click(object sender, RoutedEventArgs e)
{
tbResult.Text = string.Empty;
imgPreview.Source = null;
List <UIElement> removedItems = new List <UIElement>();
foreach (var item in gridMain.Children)
{
if (item is Border || (item is TextBlock && item != tbResult))
{
removedItems.Add(item);
}
}
foreach (var item in removedItems)
{
gridMain.Children.Remove(item);
}
try
{
Uri uri = new Uri(tbUrl.Text);
var response = await new HttpClient().GetAsync(uri);
var stream = (await response.Content.ReadAsStreamAsync()).AsRandomAccessStream();
var decoder = await BitmapDecoder.CreateAsync(stream);
var softwareBitmap = await decoder.GetSoftwareBitmapAsync();
VideoFrame videoFrame = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
//ImageFeatureValue imageFreatureValue =
// ImageFeatureValue.CreateFromVideoFrame(videoFrame);
var output = await model.PredictImageAsync(videoFrame);
var stringResult = new StringBuilder();
var pbWidth = imgPreview.ActualWidth;
var pbHeight = imgPreview.ActualHeight;
Random rand = new Random();
foreach (var item in output)
{
if (item.Probability > 0.5)
{
int x = (int)Math.Round(item.BoundingBox.Left * pbWidth);
int y = (int)Math.Round(item.BoundingBox.Top * pbHeight);
int w = (int)Math.Round(item.BoundingBox.Width * pbWidth);
int h = (int)Math.Round(item.BoundingBox.Height * pbHeight);
stringResult.AppendLine(string.Format("{0}: {1}, [{2},{3},{4},{5}]",
item.TagName, item.Probability, x, y, w, h));
Rectangle rect = new Rectangle(x, y, w, h);
Color color = Color.FromKnownColor((KnownColor)rand.Next(1, 175));
Windows.UI.Color uiColor = Windows.UI.Color.FromArgb(color.A, color.R, color.G, color.B);
Border b = new Border();
b.BorderBrush = new SolidColorBrush(uiColor);
b.BorderThickness = new Thickness(2);
b.Width = w;
b.Height = h;
b.Margin = new Thickness(x + imgPreview.Margin.Left, y + imgPreview.Margin.Top, 0, 0);
b.HorizontalAlignment = HorizontalAlignment.Left;
b.VerticalAlignment = VerticalAlignment.Top;
gridMain.Children.Add(b);
TextBlock tb = new TextBlock();
tb.Margin = new Thickness(b.Margin.Left + 2, b.Margin.Top + 2, 0, 0);
tb.HorizontalAlignment = HorizontalAlignment.Left;
tb.VerticalAlignment = VerticalAlignment.Top;
tb.Text = item.TagName;
tb.IsColorFontEnabled = true;
tb.Foreground = b.BorderBrush;
gridMain.Children.Add(tb);
}
}
tbResult.Text = stringResult.ToString();
var sbs = new SoftwareBitmapSource();
var imagePreview = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8,
BitmapAlphaMode.Premultiplied);
await sbs.SetBitmapAsync(imagePreview);
imgPreview.Source = sbs;
}
catch (Exception ex)
{
MessageDialog a = new MessageDialog(String.Format("error: \n{0}", ex.ToString()));
await a.ShowAsync();
}
}
