private void DrawToTexture(Texture texture, Annotation[] annotations, Texture2D results)
{
if (texture is Texture2D)
{
Texture2D t2d = texture as Texture2D;
_drawableTexture.SetPixels32(t2d.GetPixels32());
}
else
{
Texture2D tmp = new Texture2D(texture.width, texture.height, GraphicsFormat.R8G8B8A8_SRGB, texture.mipmapCount, TextureCreationFlags.None);
Graphics.CopyTexture(texture, tmp);
_drawableTexture.SetPixels32(tmp.GetPixels32());
}
foreach (Annotation annotation in annotations)
{
float left = annotation.Rectangle[0] * _drawableTexture.width;
float top = annotation.Rectangle[1] * _drawableTexture.height;
float right = annotation.Rectangle[2] * _drawableTexture.width;
float bottom = annotation.Rectangle[3] * _drawableTexture.height;
Rect scaledLocation = new Rect(left, top, right - left, bottom - top);
scaledLocation.y = texture.height - scaledLocation.y;
scaledLocation.height = -scaledLocation.height;
NativeImageIO.DrawRect(_drawableTexture, scaledLocation, Color.red);
}
results.Apply();
}
public RecognitionResult[] Recognize(
Texture2D texture2D,
int inputHeight = 299,
int inputWidth = 299,
float inputMean = 0.0f,
float scale = 1.0f / 255.0f,
bool flipUpsideDown = true,
bool swapBR = false)
{
NativeImageIO.ReadTensorFromTexture2D <float>(
texture2D,
_inputTensor.DataPointer,
inputHeight,
inputWidth,
inputMean,
scale,
flipUpsideDown,
swapBR);
_interpreter.Invoke();
float[] probability = _outputTensor.GetData(false) as float[];
if (probability == null)
{
return(null);
}
return(SortResults(probability));
}
private void ReadImageFileToTensor(String imageFile)
{
int height = _inputTensor.Dims[1];
int width = _inputTensor.Dims[2];
NativeImageIO.ReadImageFileToTensor <byte>(imageFile, _inputTensor.DataPointer, height, width, 0.0f, 1.0f);
}
private void onDownloadCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
{
Stopwatch watch = Stopwatch.StartNew();
var result = _mobilenet.Recognize(_imageFiles[0], 0.5f);
watch.Stop();
NativeImageIO.Annotation[] annotations = new NativeImageIO.Annotation[result.Length];
for (int i = 0; i < result.Length; i++)
{
NativeImageIO.Annotation annotation = new NativeImageIO.Annotation();
annotation.Rectangle = result[i].Rectangle;
annotation.Label = String.Format("{0}:({1:0.00}%)", result[i].Label, result[i].Score * 100);
annotations[i] = annotation;
}
NativeImageIO.JpegData jpeg = NativeImageIO.ImageFileToJpeg(_imageFiles[0], annotations);
//NativeImageIO.JpegData jpeg = NativeImageIO.ImageFileToJpeg(_imageFiles[0]);
//String names = String.Join(";", Array.ConvertAll(result, r => r.Label));
SetImage(jpeg.Raw, jpeg.Width, jpeg.Height);
String resStr = String.Format("Detected {1} objects in {0} milliseconds.", watch.ElapsedMilliseconds, result.Length);
SetMessage(resStr);
}
/*
* public static Tensor ReadTensorFromTexture2D_V0(
* Texture2D texture, int inputHeight = -1, int inputWidth = -1,
* float inputMean = 0.0f, float scale = 1.0f, bool flipUpsideDown = false)
* {
* Color32[] colors;
* Tensor t;
* int width, height;
* if (inputHeight > 0 || inputWidth > 0)
* {
* Texture2D small = Resize(texture, inputWidth, inputHeight);
* colors = small.GetPixels32();
* width = inputWidth;
* height = inputHeight;
* }
* else
* {
* width = texture.width;
* height = texture.height;
* colors = texture.GetPixels32();
* }
* t = new Tensor(DataType.Float, new int[] { 1, height, width, 3 });
*
* float[] floatValues = new float[colors.Length*3];
*
* if (flipUpsideDown)
* {
* for (int i = 0; i < height; i++)
* for (int j = 0; j < width; j++)
* {
* Color32 val = colors[(height - i - 1) * width + j];
* int idx = i * width + j;
* floatValues[idx * 3 + 0] = (val.r - inputMean) * scale;
* floatValues[idx * 3 + 1] = (val.g - inputMean) * scale;
* floatValues[idx * 3 + 2] = (val.b - inputMean) * scale;
* }
* }
* else
* {
* for (int i = 0; i < colors.Length; ++i)
* {
* Color32 val = colors[i];
* floatValues[i * 3 + 0] = (val.r - inputMean) * scale;
* floatValues[i * 3 + 1] = (val.g - inputMean) * scale;
* floatValues[i * 3 + 2] = (val.b - inputMean) * scale;
* }
* }
*
*
* System.Runtime.InteropServices.Marshal.Copy(floatValues, 0, t.DataPointer, floatValues.Length);
*
* return t;
* }*/
#else
public static byte[] TensorToJpeg(Tensor image, float scale = 1.0f, float mean = 0.0f, Status status = null)
{
#if __ANDROID__
byte[] rawPixel = TensorToPixel(image, scale, mean, 4);
int[] dim = image.Dim;
return(NativeImageIO.PixelToJpeg(rawPixel, dim[2], dim[1], 4));
#elif __IOS__
if (mean != 0.0)
{
throw new NotImplemenetedException("Not able to accept mean values on this platform");
}
byte[] rawPixel = TensorToPixel(image, scale, mean, 3);
int[] dim = image.Dim;
return(NativeImageIO.PixelToJpeg(rawPixel, dim[2], dim[1], 3));
#elif __UNIFIED__ //Mac OSX
byte[] rawPixel = TensorToPixel(image, scale, mean, 4);
int[] dim = image.Dim;
return(NativeImageIO.PixelToJpeg(rawPixel, dim[2], dim[1], 4));
#else
//if (mean != 0.0)
// throw new NotImplemenetedException("Not able to accept mean values on this platform");
//byte[] rawPixel = TensorToPixel(image, scale, mean, 3);
//int[] dim = image.Dim;
//return NativeImageIO.PixelToJpeg(rawPixel, dim[2], dim[1], 3);
return(EncodeJpeg(image, 1.0f, 0.0f));
#endif
}
private async void OnButtonClicked(Object sender, EventArgs args)
{
SetMessage("Please wait while the Coco SSD Mobilenet Model is being downloaded...");
#if !DEBUG
try
#endif
{
await _mobilenet.Init();
}
#if !DEBUG
catch (Exception e)
{
String msg = e.Message.Replace(System.Environment.NewLine, " ");
SetMessage(msg);
}
#endif
if (this.TopButton.Text.Equals("Stop"))
{
// Stop camera
#if __IOS__ || __MACOS__
this.StopCaptureSession();
#endif
this.TopButton.Text = "Perform Object Detection";
}
else
{
String[] imageFiles = await LoadImages(new string[] { "dog416.png" });
String imageFileName = imageFiles[0];
if (imageFileName.Equals("Camera Stream"))
{
#if __MACOS__ || __IOS__
SetMessage(String.Format("Model trained to recognize the following objects: {0}", String.Join("; ", _mobilenet.Labels)));
this.TopButton.Text = "Stop";
CheckVideoPermissionAndStart();
#else
#endif
}
else
{
Stopwatch watch = Stopwatch.StartNew();
var result = _mobilenet.Recognize(imageFileName, 0.5f);
watch.Stop();
Annotation[] annotations = GetAnnotations(result);
JpegData jpeg = NativeImageIO.ImageFileToJpeg(imageFileName, annotations);
//NativeImageIO.JpegData jpeg = NativeImageIO.ImageFileToJpeg(_imageFiles[0]);
//String names = String.Join(";", Array.ConvertAll(result, r => r.Label));
SetImage(jpeg.Raw, jpeg.Width, jpeg.Height);
String resStr = String.Format("Detected {1} objects in {0} milliseconds.", watch.ElapsedMilliseconds, result.Length);
SetMessage(resStr);
}
}
}
public float[] Recognize(String imageFile)
{
NativeImageIO.ReadImageFileToTensor <float>(imageFile, _inputTensor.DataPointer, 224, 224, 128.0f, 1.0f / 128.0f);
_interpreter.Invoke();
float[] probability = _outputTensor.Data as float[];
return(probability);
}
/// <summary>
/// Perform Coco Ssd Mobilenet detection
/// </summary>
/// <param name="image">The image where we will ran the network through</param>
/// <param name="scoreThreshold">If non-positive, will return all results. If positive, we will only return results with score larger than this value</param>
/// <returns>The result of the detection.</returns>
public RecognitionResult[] Recognize(NSImage image, float scoreThreshold = 0.0f)
{
int height = _inputTensor.Dims[1];
int width = _inputTensor.Dims[2];
NativeImageIO.ReadImageToTensor <byte>(image, _inputTensor.DataPointer, height, width, 0.0f, 1.0f);
_interpreter.Invoke();
return(GetResults(scoreThreshold));
}
//private int _counter = 0;
private void OutputRecorder_BufferReceived(object sender, OutputRecorder.BufferReceivedEventArgs e)
{
try
{
//_counter++;
#if __IOS__
UIImage image = e.Buffer.ToUIImage();
CocoSsdMobilenet.RecognitionResult[] result = _mobilenet.Recognize(image, 0.5f);
Annotation[] annotations = GetAnnotations(result);
UIImage annotatedImage = NativeImageIO.DrawAnnotations(image, annotations);
image.Dispose();
#else
NSImage image = e.Buffer.ToNSImage();
CocoSsdMobilenet.RecognitionResult[] result = _mobilenet.Recognize(image, 0.5f);
Annotation[] annotations = GetAnnotations(result);
#endif
//Debug.WriteLine(image == null ? "null image" : String.Format(">image {0} x {1}", image.Size.Width, image.Size.Height));
// Do something with the image, we just stuff it in our main view.
Xamarin.Forms.Device.BeginInvokeOnMainThread(delegate
{
//Debug.WriteLine(image == null ? "null image" : String.Format(">>image {0} x {1}", image.Size.Width, image.Size.Height));
#if __IOS__
//if (UIImageView.Frame.Size != annotatedImage.Size)
// UIImageView.Frame = new CGRect(CGPoint.Empty, annotatedImage.Size);
//SetMessage( String.Format("{0} image", _counter) );
//UIImage oldImage = UIImageView.Image;
//UIImageView.Image = annotatedImage;
//if (oldImage != null)
// oldImage.Dispose();
SetImage(annotatedImage);
#else
NativeImageIO.DrawAnnotations(image, annotations);
//SetMessage(String.Format("{0} image", _counter));
SetImage(image);
#endif
});
//
// Although this looks innocent "Oh, he is just optimizing this case away"
// this is incredibly important to call on this callback, because the AVFoundation
// has a fixed number of buffers and if it runs out of free buffers, it will stop
// delivering frames.
//
//Console.WriteLine(String.Format("Frame at: {0}", DateTime.Now));
}
catch (Exception ex)
{
Console.WriteLine(e);
SetMessage(ex.Message);
}
}
/// <summary>
/// Load the file, ran it through the mobile net graph and return the recognition results
/// </summary>
/// <param name="imageFile">The image to be loaded</param>
/// <param name="width">The width of the input tensor</param>
/// <param name="height">The height of the input tensor</param>
/// <param name="mean">The mean to be subtracted when converting the image to input tensor</param>
/// <param name="scale">The scale to be multiplied when converting the image to input tensor</param>
/// <param name="flipUpsideDown">If true, the image will be flipped upside down when it is coverted to input tensor</param>
/// <param name="swapBR">If true, the blue and red channel will be swapped when converting the image to input tensor </param>
/// <returns>The recognition result sorted by probability</returns>
public RecognitionResult[] Recognize(String imageFile, int width = 299, int height = 299, float mean = 0.0f, float scale = 1.0f / 255.0f, bool flipUpsideDown = false, bool swapBR = true)
{
NativeImageIO.ReadImageFileToTensor <float>(imageFile, _inputTensor.DataPointer, height, width, mean, scale, flipUpsideDown, swapBR);
_interpreter.Invoke();
float[] probability = _outputTensor.Data as float[];
if (probability == null)
{
return(null);
}
return(SortResults(probability));
}
/// <summary>
/// Load the file, ran it through the mobile net graph and return the recognition results
/// </summary>
/// <param name="imageFile">The image to be loaded</param>
/// <returns>The recognition result sorted by probability</returns>
public RecognitionResult[] Recognize(String imageFile)
{
NativeImageIO.ReadImageFileToTensor <float>(imageFile, _inputTensor.DataPointer, 224, 224, 128.0f, 1.0f / 128.0f);
_interpreter.Invoke();
float[] probability = _outputTensor.Data as float[];
if (probability == null)
{
return(null);
}
return(SortResults(probability));
}
/// <summary>
/// Perform Coco Ssd Mobilenet detection
/// </summary>
/// <param name="imageFile">The image file where we will ran the network through</param>
/// <param name="scoreThreshold">If non-positive, will return all results. If positive, we will only return results with score larger than this value</param>
/// <returns>The result of the detection.</returns>
public RecognitionResult[] Recognize(String imageFile, float scoreThreshold = 0.0f)
{
//NativeImageIO.ReadImageFileToTensor<byte>(imageFile, _inputTensor.DataPointer, _inputTensor.Dims[1], _inputTensor.Dims[2], _inputTensor.QuantizationParams.ZeroPoint, _inputTensor.QuantizationParams.Scale);
int height = _inputTensor.Dims[1];
int width = _inputTensor.Dims[2];
NativeImageIO.ReadImageFileToTensor <byte>(imageFile, _inputTensor.DataPointer, height, width, 0.0f, 1.0f);
_interpreter.Invoke();
float[,,] outputLocations = _interpreter.Outputs[0].JaggedData as float[, , ];
float[] classes = _interpreter.Outputs[1].Data as float[];
float[] scores = _interpreter.Outputs[2].Data as float[];
int numDetections = (int)Math.Round((_interpreter.Outputs[3].Data as float[])[0]);
// SSD Mobilenet V1 Model assumes class 0 is background class
// in label file and class labels start from 1 to number_of_classes+1,
// while outputClasses correspond to class index from 0 to number_of_classes
List <RecognitionResult> results = new List <RecognitionResult>();
int labelOffset = 1;
for (int i = 0; i < numDetections; i++)
{
if (classes[i] == 0) //background class
{
continue;
}
if (scoreThreshold > 0.0f && scores[i] < scoreThreshold)
{
continue;
}
RecognitionResult r = new RecognitionResult();
r.Class = (int)Math.Round(classes[i]);
r.Label = _labels[r.Class + labelOffset];
r.Score = scores[i];
float x0 = outputLocations[0, i, 1];
float y0 = outputLocations[0, i, 0];
float x1 = outputLocations[0, i, 3];
float y1 = outputLocations[0, i, 2];
r.Rectangle = new float[] { x0, y0, x1, y1 };
results.Add(r);
}
return(results.ToArray());
}
private void onDownloadCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
{
if (e != null && e.Error != null)
{
SetMessage(e.Error.Message);
return;
}
String _imageFileName = _imageFiles[0];
if (_imageFileName.Equals("Camera Stream"))
{
#if __MACOS__ || __IOS__
SetMessage(String.Format("Model trained to recognize the following objects: {0}", String.Join("; ", _mobilenet.Labels)));
this.TopButton.Text = "Stop";
CheckVideoPermissionAndStart();
#else
#endif
}
else
{
Stopwatch watch = Stopwatch.StartNew();
var result = _mobilenet.Recognize(_imageFiles[0], 0.5f);
watch.Stop();
Annotation[] annotations = GetAnnotations(result);
/*
* Annotation[] annotations = new Annotation[result.Length];
* for (int i = 0; i < result.Length; i++)
* {
* Annotation annotation = new Annotation();
* annotation.Rectangle = result[i].Rectangle;
* annotation.Label = String.Format("{0}:({1:0.00}%)", result[i].Label, result[i].Score * 100);
* annotations[i] = annotation;
* }*/
JpegData jpeg = NativeImageIO.ImageFileToJpeg(_imageFiles[0], annotations);
//NativeImageIO.JpegData jpeg = NativeImageIO.ImageFileToJpeg(_imageFiles[0]);
//String names = String.Join(";", Array.ConvertAll(result, r => r.Label));
SetImage(jpeg.Raw, jpeg.Width, jpeg.Height);
String resStr = String.Format("Detected {1} objects in {0} milliseconds.", watch.ElapsedMilliseconds, result.Length);
SetMessage(resStr);
}
}
public RecognitionResult[] Recognize(Texture texture, bool flipUpsideDown = true, bool swapBR = false, float scoreThreshold = 0.0f)
{
int height = _inputTensor.Dims[1];
int width = _inputTensor.Dims[2];
NativeImageIO.ReadTensorFromTexture <byte>(
texture,
_inputTensor.DataPointer,
height, width, 0.0f, 1.0f,
flipUpsideDown,
swapBR);
_interpreter.Invoke();
return(GetResults(scoreThreshold));
}
/*
* public static Tensor ReadTensorFromTexture2D_V0(
* Texture2D texture, int inputHeight = -1, int inputWidth = -1,
* float inputMean = 0.0f, float scale = 1.0f, bool flipUpsideDown = false)
* {
* Color32[] colors;
* Tensor t;
* int width, height;
* if (inputHeight > 0 || inputWidth > 0)
* {
* Texture2D small = Resize(texture, inputWidth, inputHeight);
* colors = small.GetPixels32();
* width = inputWidth;
* height = inputHeight;
* }
* else
* {
* width = texture.width;
* height = texture.height;
* colors = texture.GetPixels32();
* }
* t = new Tensor(DataType.Float, new int[] { 1, height, width, 3 });
*
* float[] floatValues = new float[colors.Length*3];
*
* if (flipUpsideDown)
* {
* for (int i = 0; i < height; i++)
* for (int j = 0; j < width; j++)
* {
* Color32 val = colors[(height - i - 1) * width + j];
* int idx = i * width + j;
* floatValues[idx * 3 + 0] = (val.r - inputMean) * scale;
* floatValues[idx * 3 + 1] = (val.g - inputMean) * scale;
* floatValues[idx * 3 + 2] = (val.b - inputMean) * scale;
* }
* }
* else
* {
* for (int i = 0; i < colors.Length; ++i)
* {
* Color32 val = colors[i];
* floatValues[i * 3 + 0] = (val.r - inputMean) * scale;
* floatValues[i * 3 + 1] = (val.g - inputMean) * scale;
* floatValues[i * 3 + 2] = (val.b - inputMean) * scale;
* }
* }
*
*
* System.Runtime.InteropServices.Marshal.Copy(floatValues, 0, t.DataPointer, floatValues.Length);
*
* return t;
* }*/
#else
public static byte[] TensorToJpeg(Tensor stylizedImage, float scale = 1.0f, Status status = null)
{
#if __ANDROID__
byte[] rawPixel = TensorToPixel(stylizedImage, scale, 4);
int[] dim = stylizedImage.Dim;
return(NativeImageIO.PixelToJpeg(rawPixel, dim[2], dim[1], 4));
#elif __IOS__
byte[] rawPixel = TensorToPixel(stylizedImage, scale, 3);
int[] dim = stylizedImage.Dim;
return(NativeImageIO.PixelToJpeg(rawPixel, dim[2], dim[1], 3));
#elif __UNIFIED__ //Mac OSX
byte[] rawPixel = TensorToPixel(stylizedImage, scale, 4);
int[] dim = stylizedImage.Dim;
return(NativeImageIO.PixelToJpeg(rawPixel, dim[2], dim[1], 4));
#else
return(null);
#endif
}
public RecognitionResult[] Recognize(Texture2D texture2D, bool flipUpsideDown = true, bool swapBR = false)
{
NativeImageIO.ReadTensorFromTexture2D <float>(
texture2D,
_inputTensor.DataPointer,
224, 224, 128.0f, 1.0f / 128.0f,
flipUpsideDown,
swapBR);
_interpreter.Invoke();
float[] probability = _outputTensor.GetData(false) as float[];
if (probability == null)
{
return(null);
}
return(SortResults(probability));
}
private void OnDownloadComplete(object sender, AsyncCompletedEventArgs e)
{
try
{
using (var imageTensor = ImageIO.ReadTensorFromImageFile <float>(_fileName, 224, 224, 128.0f, 1.0f / 128.0f))
{
var detectionResult = _graph.Detect(imageTensor);
var detectionAnnotations = MultiboxGraph.FilterResults(detectionResult, 0.1f);
var detectionImage = NativeImageIO.ImageFileToJpeg(_fileName, detectionAnnotations);
var typeConverter = TypeDescriptor.GetConverter(typeof(Bitmap));
var detectionBmpImage = (Bitmap)typeConverter.ConvertFrom(detectionImage.Raw);
ImageSource = BitmapConverter.ConvertBitmap(detectionBmpImage);
}
IsModalVisible = false;
}
catch (Exception ex)
{
_exceptionLogDataAccess.LogException(ex.ToString());
IsModalVisible = false;
}
}
private static void DrawToTexture(Texture texture, Annotation[] annotations, Texture2D results)
{
Debug.Assert(texture.width == results.width && texture.height == results.height, "Input texture and output texture must have the same width & height.");
Color32[] pixels;
if (texture is Texture2D)
{
Texture2D t2d = texture as Texture2D;
pixels = t2d.GetPixels32();
}
else if (texture is WebCamTexture)
{
WebCamTexture wct = texture as WebCamTexture;
pixels = wct.GetPixels32();
}
else
{
Texture2D tmp = new Texture2D(texture.width, texture.height, GraphicsFormat.R8G8B8A8_SRGB, texture.mipmapCount, TextureCreationFlags.None);
Graphics.CopyTexture(texture, tmp);
pixels = tmp.GetPixels32();
Destroy(tmp);
}
foreach (Annotation annotation in annotations)
{
float left = annotation.Rectangle[0] * texture.width;
float top = annotation.Rectangle[1] * texture.height;
float right = annotation.Rectangle[2] * texture.width;
float bottom = annotation.Rectangle[3] * texture.height;
Rect scaledLocation = new Rect(left, top, right - left, bottom - top);
scaledLocation.y = texture.height - scaledLocation.y;
scaledLocation.height = -scaledLocation.height;
NativeImageIO.DrawRect(pixels, texture.width, texture.height, scaledLocation, Color.red);
}
results.SetPixels32(pixels);
results.Apply();
}
private static Tensor NativeReadTensorFromImageFile <T>(
String fileName,
int inputHeight = -1,
int inputWidth = -1,
float inputMean = 0.0f,
float scale = 1.0f,
bool flipUpSideDown = false,
bool swapBR = false,
Status status = null) where T : struct
{
//Use native Image handler to import the file
Tensor t;
if (typeof(T) == typeof(float))
{
t = new Tensor(DataType.Float, new int[] { 1, (int)inputHeight, (int)inputWidth, 3 });
}
else if (typeof(T) == typeof(byte))
{
t = new Tensor(DataType.Uint8, new int[] { 1, (int)inputHeight, (int)inputWidth, 3 });
}
else
{
throw new Exception(String.Format("Conversion to tensor of type {0} is not implemented", typeof(T)));
}
NativeImageIO.ReadImageFileToTensor <T>(
fileName,
t.DataPointer,
inputHeight,
inputWidth,
inputMean,
scale,
flipUpSideDown,
!swapBR //No swapping BR in tensorflow is the equivalent of swapping BR in Bitmap
);
return(t);
}
public void TestNativeImageIO()
{
int inputHeight = 299;
int inputWidth = 299;
String file = "surfers.jpg";
Tensor t0 = new Tensor(DataType.Uint8, new int[] { 1, (int)inputHeight, (int)inputWidth, 3 });
NativeImageIO.ReadImageFileToTensor <float>(file, t0.DataPointer, inputHeight, inputWidth, 0.0f,
1.0f / 255.0f);
Tensor t1 = ImageIO.ReadTensorFromImageFile <float>(file);
float maxDiff = 0.0f;
var ta0 = t0.GetData(false) as float[];
var ta1 = t1.GetData(false) as float[];
for (int i = 0; i < ta0.Length; i++)
{
float diff = Math.Abs(ta0[i] - ta1[i]);
if (diff < maxDiff)
{
maxDiff = diff;
}
}
}
public static void DrawResults(Texture2D image, MultiboxGraph.Result[] results, float scoreThreshold, bool flipUpSideDown = false)
{
Annotation[] annotations = FilterResults(results, scoreThreshold);
Color color = new Color(1.0f, 0, 0);//Set color to red
for (int i = 0; i < annotations.Length; i++)
{
Rect[] rects = ScaleLocation(annotations[i].Rectangle, image.width, image.height, flipUpSideDown);
foreach (Rect r in rects)
{
NativeImageIO.DrawRect(image, r, color);
}
}
image.Apply();
//GUI.color = Color.white;//Reset color to white
/*
* Android.Graphics.Paint p = new Android.Graphics.Paint();
* p.SetStyle(Paint.Style.Stroke);
* p.AntiAlias = true;
* p.Color = Android.Graphics.Color.Red;
* Canvas c = new Canvas(bmp);
*
*
* for (int i = 0; i < result.Scores.Length; i++)
* {
* if (result.Scores[i] > scoreThreshold)
* {
* Rectangle rect = locations[result.Indices[i]];
* Android.Graphics.Rect r = new Rect(rect.Left, rect.Top, rect.Right, rect.Bottom);
* c.DrawRect(r, p);
* }
* }*/
}
//private Inception _inceptionGraph = null;
private void RecognizeAndUpdateText(Texture2D texture)
{
int[] input = _interpreter.GetInput();
int[] output = _interpreter.GetOutput();
Tensor inputTensor = _interpreter.GetTensor(input[0]);
Tensor outputTensor = _interpreter.GetTensor(output[0]);
NativeImageIO.ReadImageFileToTensor(_image[0], inputTensor.DataPointer, 224, 224, 128.0f, 1.0f / 128.0f);
Stopwatch watch = Stopwatch.StartNew();
_interpreter.Invoke();
watch.Stop();
float[] probability = outputTensor.GetData() as float[];
String resStr = String.Empty;
if (probability != null)
{
float maxVal = 0;
int maxIdx = 0;
for (int i = 0; i < probability.Length; i++)
{
if (probability[i] > maxVal)
{
maxVal = probability[i];
maxIdx = i;
}
}
resStr = String.Format("Object is {0} with {1}% probability. Recognition completed in {2} milliseconds.", _labels[maxIdx], maxVal * 100, watch.ElapsedMilliseconds);
}
//SetImage(_image[0]);
_displayMessage = resStr;
}
private void onDownloadCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
{
String localFileName = _downloadManager.Files[0].LocalFile;
if (_labels == null)
{
_labels = File.ReadAllLines(_downloadManager.Files[1].LocalFile);
}
System.Diagnostics.Debug.Assert(File.Exists(localFileName), "File doesn't exist");
FileInfo file = new FileInfo(localFileName);
if (_model == null)
{
_model = new FlatBufferModel(localFileName);
if (!_model.CheckModelIdentifier())
{
throw new Exception("Model indentifier check failed");
}
}
if (_resolver == null)
{
_resolver = new BuildinOpResolver();
}
if (_interpreter == null)
{
_interpreter = new Interpreter(_model, _resolver);
Status allocateTensorStatus = _interpreter.AllocateTensors();
if (allocateTensorStatus == Status.Error)
{
throw new Exception("Failed to allocate tensor");
}
}
int[] input = _interpreter.GetInput();
int[] output = _interpreter.GetOutput();
Tensor inputTensor = _interpreter.GetTensor(input[0]);
Tensor outputTensor = _interpreter.GetTensor(output[0]);
NativeImageIO.ReadImageFileToTensor(_image[0], inputTensor.DataPointer, 224, 224, 128.0f, 1.0f / 128.0f);
Stopwatch watch = Stopwatch.StartNew();
_interpreter.Invoke();
watch.Stop();
float[] probability = outputTensor.GetData() as float[];
String resStr = String.Empty;
if (probability != null)
{
float maxVal = 0;
int maxIdx = 0;
for (int i = 0; i < probability.Length; i++)
{
if (probability[i] > maxVal)
{
maxVal = probability[i];
maxIdx = i;
}
}
resStr = String.Format("Object is {0} with {1}% probability. Recognition completed in {2} milliseconds.", _labels[maxIdx], maxVal * 100, watch.ElapsedMilliseconds);
}
SetImage(_image[0]);
SetMessage(resStr);
}
public static Tensor ReadTensorFromImageFile(String fileName, int inputHeight = -1, int inputWidth = -1, float inputMean = 0.0f, float scale = 1.0f, bool flipUpsideDown = false)
{
Texture2D texture = NativeImageIO.ReadTexture2DFromFile(fileName);
return(ReadTensorFromTexture2D(texture, inputHeight, inputWidth, inputMean, scale, flipUpsideDown));
}
public override void DidOutputSampleBuffer(AVCaptureOutput captureOutput, CMSampleBuffer sampleBuffer, AVCaptureConnection connection)
{
try
{
_counter++;
#if __IOS__
UIImage image = ImageFromSampleBuffer(sampleBuffer);
#else
NSImage image = ImageFromSampleBuffer(sampleBuffer);
#endif
CocoSsdMobilenet.RecognitionResult[] result = _mobilenet.Recognize(image, 0.5f);
Annotation[] annotations = GetAnnotations(result);
#if __IOS__
UIImage annotatedImage = NativeImageIO.DrawAnnotations(image, annotations);
image.Dispose();
#endif
//Debug.WriteLine(image == null ? "null image" : String.Format(">image {0} x {1}", image.Size.Width, image.Size.Height));
// Do something with the image, we just stuff it in our main view.
BeginInvokeOnMainThread(delegate
{
//Debug.WriteLine(image == null ? "null image" : String.Format(">>image {0} x {1}", image.Size.Width, image.Size.Height));
#if __MACOS__
NativeImageIO.DrawAnnotations(image, annotations);
#endif
if (_imageView.Frame.Size != image.Size)
{
_imageView.Frame = new CGRect(CGPoint.Empty, image.Size);
}
_label.Text = String.Format("{0} image", _counter);
#if __IOS__
UIImage oldImage = _imageView.Image;
_imageView.Image = annotatedImage;
#else
NSImage oldImage = _imageView.Image;
//Debug.WriteLine(image == null ? "null image" : String.Format(">>>image {0} x {1}", image.Size.Width, image.Size.Height));
_imageView.Image = image;
#endif
if (oldImage != null)
{
oldImage.Dispose();
}
});
//
// Although this looks innocent "Oh, he is just optimizing this case away"
// this is incredibly important to call on this callback, because the AVFoundation
// has a fixed number of buffers and if it runs out of free buffers, it will stop
// delivering frames.
//
sampleBuffer.Dispose();
//Console.WriteLine(String.Format("Frame at: {0}", DateTime.Now));
}
catch (Exception e)
{
Console.WriteLine(e);
BeginInvokeOnMainThread(delegate
{
_label.Text = String.Format("{0} image", e.Message);
});
}
}
/// <summary>
/// Read image files, covert the data and save it to the native pointer
/// </summary>
/// <typeparam name="T">The type of the data to covert the image pixel values to. e.g. "float" or "byte"</typeparam>
/// <param name="fileNames">The name of the image files</param>
/// <param name="inputHeight">The height of the image, must match the height requirement for the tensor</param>
/// <param name="inputWidth">The width of the image, must match the width requirement for the tensor</param>
/// <param name="inputMean">The mean value, it will be subtracted from the input image pixel values</param>
/// <param name="scale">The scale, after mean is subtracted, the scale will be used to multiply the pixel values</param>
/// <param name="flipUpSideDown">If true, the image needs to be flipped up side down</param>
/// <param name="swapBR">If true, will flip the Blue channel with the Red. e.g. If false, the tensor's color channel order will be RGB. If true, the tensor's color channle order will be BGR </param>
/// <param name="status">Tensorflow status</param>
/// <returns>The tensor that contains all the image files</returns>
private static Tensor NativeReadTensorFromImageFiles <T>(
String[] fileNames,
int inputHeight = -1,
int inputWidth = -1,
float inputMean = 0.0f,
float scale = 1.0f,
bool flipUpSideDown = false,
bool swapBR = false,
Status status = null) where T : struct
{
if (fileNames.Length == 0)
{
throw new ArgumentException("Intput file names do not contain any files");
}
String fileName = fileNames[0];
if (!File.Exists(fileName))
{
throw new FileNotFoundException(String.Format("File {0} do not exist.", fileName));
}
IntPtr dataPtr;
int step;
Tensor t;
#if __ANDROID__
using (BitmapFactory.Options options = new BitmapFactory.Options())
{
options.InPreferredConfig = Android.Graphics.Bitmap.Config.Argb8888; //Prefer ARGB8888 format
using (Android.Graphics.Bitmap bmp0 = Android.Graphics.BitmapFactory.DecodeFile(fileName, options))
{
if (inputHeight <= 0)
{
inputHeight = bmp0.Height;
}
if (inputWidth <= 0)
{
inputWidth = bmp0.Width;
}
if (typeof(T) == typeof(float))
{
t = new Tensor(DataType.Float,
new int[] { fileNames.Length, (int)inputHeight, (int)inputWidth, 3 });
}
else if (typeof(T) == typeof(byte))
{
t = new Tensor(DataType.Uint8,
new int[] { fileNames.Length, (int)inputHeight, (int)inputWidth, 3 });
}
else
{
throw new Exception(String.Format("Conversion to tensor of type {0} is not implemented",
typeof(T)));
}
dataPtr = t.DataPointer;
step = NativeImageIO.ReadBitmapToTensor <T>(
bmp0,
dataPtr,
inputHeight,
inputWidth,
inputMean,
scale,
flipUpSideDown,
swapBR);
dataPtr = new IntPtr(dataPtr.ToInt64() + step);
}
for (int i = 1; i < fileNames.Length; i++)
{
fileName = fileNames[i];
if (!File.Exists(fileName))
{
throw new FileNotFoundException(String.Format("File {0} do not exist.", fileName));
}
//Read the file using Bitmap class
using (Android.Graphics.Bitmap bmp = Android.Graphics.BitmapFactory.DecodeFile(fileName, options))
{
step = NativeImageIO.ReadBitmapToTensor <T>(
bmp,
dataPtr,
inputHeight,
inputWidth,
inputMean,
scale,
flipUpSideDown,
swapBR);
dataPtr = new IntPtr(dataPtr.ToInt64() + step);
}
}
}
#elif __MACOS__
using (NSImage image0 = new NSImage(fileName))
{
if (inputHeight <= 0)
{
inputHeight = (int)image0.Size.Height;
}
if (inputWidth <= 0)
{
inputWidth = (int)image0.Size.Width;
}
if (typeof(T) == typeof(float))
{
t = new Tensor(DataType.Float,
new int[] { fileNames.Length, (int)inputHeight, (int)inputWidth, 3 });
}
else if (typeof(T) == typeof(byte))
{
t = new Tensor(DataType.Uint8,
new int[] { fileNames.Length, (int)inputHeight, (int)inputWidth, 3 });
}
else
{
throw new Exception(String.Format("Conversion to tensor of type {0} is not implemented",
typeof(T)));
}
dataPtr = t.DataPointer;
step = NativeImageIO.ReadImageToTensor <T>(
image0, dataPtr, inputHeight, inputWidth, inputMean, scale,
flipUpSideDown, swapBR);
dataPtr = new IntPtr(dataPtr.ToInt64() + step);
}
for (int i = 1; i < fileNames.Length; i++)
{
fileName = fileNames[i];
if (!File.Exists(fileName))
{
throw new FileNotFoundException(String.Format("File {0} do not exist.", fileName));
}
//Read the file using Bitmap class
using (NSImage image = new NSImage(fileName))
{
step = NativeImageIO.ReadImageToTensor <T>(image, dataPtr, inputHeight, inputWidth, inputMean, scale,
flipUpSideDown, swapBR);
dataPtr = new IntPtr(dataPtr.ToInt64() + step);
}
}
#else
using (System.Drawing.Bitmap bmp0 = new System.Drawing.Bitmap(fileName))
{
if (inputHeight <= 0)
{
inputHeight = bmp0.Height;
}
if (inputWidth <= 0)
{
inputWidth = bmp0.Width;
}
if (typeof(T) == typeof(float))
{
t = new Tensor(DataType.Float,
new int[] { fileNames.Length, (int)inputHeight, (int)inputWidth, 3 });
}
else if (typeof(T) == typeof(byte))
{
t = new Tensor(DataType.Uint8,
new int[] { fileNames.Length, (int)inputHeight, (int)inputWidth, 3 });
}
else
{
throw new Exception(String.Format("Conversion to tensor of type {0} is not implemented",
typeof(T)));
}
dataPtr = t.DataPointer;
step = NativeImageIO.ReadBitmapToTensor <T>(bmp0, dataPtr, inputHeight, inputWidth, inputMean, scale,
flipUpSideDown, swapBR);
dataPtr = new IntPtr(dataPtr.ToInt64() + step);
}
for (int i = 1; i < fileNames.Length; i++)
{
fileName = fileNames[i];
if (!File.Exists(fileName))
{
throw new FileNotFoundException(String.Format("File {0} do not exist.", fileName));
}
//Read the file using Bitmap class
using (System.Drawing.Bitmap bmp = new System.Drawing.Bitmap(fileName))
{
step = NativeImageIO.ReadBitmapToTensor <T>(bmp, dataPtr, inputHeight, inputWidth, inputMean, scale,
flipUpSideDown, swapBR);
dataPtr = new IntPtr(dataPtr.ToInt64() + step);
}
}
#endif
return(t);
}
private void RecognizeAndUpdateText(Texture2D texture)
{
if (_mobilenet == null)
{
_displayMessage = "Waiting for mobile net model to be loaded...";
return;
}
Stopwatch watch = Stopwatch.StartNew();
CocoSsdMobilenet.RecognitionResult[] results = _mobilenet.Recognize(texture, true, false, 0.5f);
watch.Stop();
if (drawableTexture == null || drawableTexture.width != texture.width ||
drawableTexture.height != texture.height)
{
drawableTexture = new Texture2D(texture.width, texture.height, TextureFormat.ARGB32, false);
}
drawableTexture.SetPixels(texture.GetPixels());
Annotation[] annotations = new Annotation[results.Length];
for (int i = 0; i < results.Length; i++)
{
Annotation annotation = new Annotation();
annotation.Rectangle = results[i].Rectangle;
annotation.Label = String.Format("{0}:({1:0.00}%)", results[i].Label, results[i].Score * 100);
annotations[i] = annotation;
}
String objectNames = String.Empty;
foreach (Annotation annotation in annotations)
{
float left = annotation.Rectangle[0] * drawableTexture.width;
float top = annotation.Rectangle[1] * drawableTexture.height;
float right = annotation.Rectangle[2] * drawableTexture.width;
float bottom = annotation.Rectangle[3] * drawableTexture.height;
Rect scaledLocation = new Rect(left, top, right - left, bottom - top);
scaledLocation.y = texture.height - scaledLocation.y;
scaledLocation.height = -scaledLocation.height;
NativeImageIO.DrawRect(drawableTexture, scaledLocation, Color.red);
objectNames = objectNames + annotation.Label + ";";
}
drawableTexture.Apply();
//MultiboxGraph.DrawResults(drawableTexture, results, 0.2f, true);
if (!String.IsNullOrEmpty(objectNames))
{
objectNames = String.Format("({0})", objectNames);
}
String resStr = String.Empty;
if (results != null)
{
resStr = String.Format("{0} objects detected{1}. Recognition completed in {2} milliseconds.", annotations.Length, objectNames, watch.ElapsedMilliseconds);
//resStr = String.Format("Object is {0} with {1}% probability. Recognition completed in {2} milliseconds.", results[0].Label, results[0].Probability * 100, watch.ElapsedMilliseconds);
}
_displayMessage = resStr;
}
public static Tensor ReadTensorFromImageFile(String fileName, int inputHeight = -1, int inputWidth = -1, float inputMean = 0.0f, float scale = 1.0f, Status status = null)
{
#if __ANDROID__
Android.Graphics.Bitmap bmp = BitmapFactory.DecodeFile(fileName);
if (inputHeight > 0 || inputWidth > 0)
{
Bitmap resized = Bitmap.CreateScaledBitmap(bmp, inputWidth, inputHeight, false);
bmp.Dispose();
bmp = resized;
}
int[] intValues = new int[bmp.Width * bmp.Height];
float[] floatValues = new float[bmp.Width * bmp.Height * 3];
bmp.GetPixels(intValues, 0, bmp.Width, 0, 0, bmp.Width, bmp.Height);
for (int i = 0; i < intValues.Length; ++i)
{
int val = intValues[i];
floatValues[i * 3 + 0] = (((val >> 16) & 0xFF) - inputMean) * scale;
floatValues[i * 3 + 1] = (((val >> 8) & 0xFF) - inputMean) * scale;
floatValues[i * 3 + 2] = ((val & 0xFF) - inputMean) * scale;
}
Tensor t = new Tensor(DataType.Float, new int[] { 1, bmp.Height, bmp.Width, 3 });
System.Runtime.InteropServices.Marshal.Copy(floatValues, 0, t.DataPointer, floatValues.Length);
return(t);
#elif __IOS__
UIImage image = new UIImage(fileName);
if (inputHeight > 0 || inputWidth > 0)
{
UIImage resized = image.Scale(new CGSize(inputWidth, inputHeight));
image.Dispose();
image = resized;
}
int[] intValues = new int[(int)(image.Size.Width * image.Size.Height)];
float[] floatValues = new float[(int)(image.Size.Width * image.Size.Height * 3)];
System.Runtime.InteropServices.GCHandle handle = System.Runtime.InteropServices.GCHandle.Alloc(intValues, System.Runtime.InteropServices.GCHandleType.Pinned);
using (CGImage cgimage = image.CGImage)
using (CGColorSpace cspace = CGColorSpace.CreateDeviceRGB())
using (CGBitmapContext context = new CGBitmapContext(
handle.AddrOfPinnedObject(),
(nint)image.Size.Width,
(nint)image.Size.Height,
8,
(nint)image.Size.Width * 4,
cspace,
CGImageAlphaInfo.PremultipliedLast
))
{
context.DrawImage(new CGRect(new CGPoint(), image.Size), cgimage);
}
handle.Free();
for (int i = 0; i < intValues.Length; ++i)
{
int val = intValues[i];
floatValues[i * 3 + 0] = (((val >> 16) & 0xFF) - inputMean) * scale;
floatValues[i * 3 + 1] = (((val >> 8) & 0xFF) - inputMean) * scale;
floatValues[i * 3 + 2] = ((val & 0xFF) - inputMean) * scale;
}
Tensor t = new Tensor(DataType.Float, new int[] { 1, (int)image.Size.Height, (int)image.Size.Width, 3 });
System.Runtime.InteropServices.Marshal.Copy(floatValues, 0, t.DataPointer, floatValues.Length);
return(t);
#else
if (Emgu.TF.Util.Platform.OperationSystem == OS.Windows)
{
Tensor t = new Tensor(DataType.Float, new int[] { 1, (int)inputHeight, (int)inputWidth, 3 });
NativeImageIO.ReadImageFileToTensor(fileName, t.DataPointer, inputHeight, inputWidth, inputMean, scale);
return(t);
}
else
{
using (StatusChecker checker = new StatusChecker(status))
{
var graph = new Graph();
Operation input = graph.Placeholder(DataType.String);
Operation jpegDecoder = graph.DecodeJpeg(input, 3); //dimension 3
Operation floatCaster = graph.Cast(jpegDecoder, DstT: DataType.Float); //cast to float
Tensor axis = new Tensor(0);
Operation axisOp = graph.Const(axis, axis.Type, opName: "axis");
Operation dimsExpander = graph.ExpandDims(floatCaster, axisOp); //turn it to dimension [1,3]
Operation resized;
bool resizeRequired = (inputHeight > 0) && (inputWidth > 0);
if (resizeRequired)
{
Tensor size = new Tensor(new int[] { inputHeight, inputWidth }); // new size;
Operation sizeOp = graph.Const(size, size.Type, opName: "size");
resized = graph.ResizeBilinear(dimsExpander, sizeOp); //resize image
}
else
{
resized = dimsExpander;
}
Tensor mean = new Tensor(inputMean);
Operation meanOp = graph.Const(mean, mean.Type, opName: "mean");
Operation substracted = graph.Sub(resized, meanOp);
Tensor scaleTensor = new Tensor(scale);
Operation scaleOp = graph.Const(scaleTensor, scaleTensor.Type, opName: "scale");
Operation scaled = graph.Mul(substracted, scaleOp);
Session session = new Session(graph);
Tensor imageTensor = Tensor.FromString(File.ReadAllBytes(fileName), status);
Tensor[] imageResults = session.Run(new Output[] { input }, new Tensor[] { imageTensor },
new Output[] { scaled });
return(imageResults[0]);
}
}
#endif
}
