/// <summary>
/// Given the left and right image, computer the disparity map and the 3D point cloud.
/// </summary>
/// <param name="left">The left image</param>
/// <param name="right">The right image</param>
/// <param name="outputDisparityMap">The left disparity map</param>
/// <param name="points">The 3D point cloud within a [-0.5, 0.5] cube</param>
private static void Computer3DPointsFromStereoPair(IInputArray left, IInputArray right, Mat outputDisparityMap, Mat points)
{
Size size;
using (InputArray ia = left.GetInputArray())
size = ia.GetSize();
using (StereoBM stereoSolver = new StereoBM())
{
stereoSolver.Compute(left, right, outputDisparityMap);
float scale = Math.Max(size.Width, size.Height);
//Construct a simple Q matrix, if you have a matrix from cvStereoRectify, you should use that instead
using (Matrix <double> q = new Matrix <double>(
new double[, ]
{
{ 1.0, 0.0, 0.0, -size.Width / 2 }, //shift the x origin to image center
{ 0.0, -1.0, 0.0, size.Height / 2 }, //shift the y origin to image center and flip it upside down
{ 0.0, 0.0, -1.0, 0.0 }, //Multiply the z value by -1.0,
{ 0.0, 0.0, 0.0, scale }
})) //scale the object's coordinate to within a [-0.5, 0.5] cube
{
CvInvoke.ReprojectImageTo3D(outputDisparityMap, points, q, false, DepthType.Cv32F);
}
//points = PointCollection.ReprojectImageTo3D(outputDisparityMap, q);
}
}
public void SetImage(IInputArray image)
{
if (image == null)
{
Xamarin.Forms.Device.BeginInvokeOnMainThread(
() => { this.DisplayImage.Source = null; });
return;
}
using (VectorOfByte vb = new VectorOfByte())
{
CvInvoke.Imencode(".jpg", image, vb);
byte[] rawData = vb.ToArray();
Xamarin.Forms.Device.BeginInvokeOnMainThread(
() =>
{
this.DisplayImage.Source = ImageSource.FromStream(() => new MemoryStream(rawData));
#if __MACOS__
using (InputArray iaImage = image.GetInputArray())
this.DisplayImage.HeightRequest = iaImage.GetSize().Height;
#elif __IOS__
//the following is needed for iOS due to the fact that
//Xamarin Forms' Image object do not seems to refresh after we set the source.
//Forcing the focus seems to force a rendering update.
this.DisplayImage.Focus();
#endif
});
}
}
/// <summary>
/// Given the left and right image, computer the disparity map and the 3D point cloud.
/// </summary>
/// <param name="left">The left image</param>
/// <param name="right">The right image</param>
/// <param name="outputDisparityMap">The left disparity map</param>
/// <param name="points">The 3D point cloud within a [-0.5, 0.5] cube</param>
private static void Computer3DPointsFromStereoPair(IInputArray left, IInputArray right, Mat outputDisparityMap, Mat points, bool handleMissingValues = true)
{
System.Drawing.Size size;
using (InputArray ia = left.GetInputArray())
size = ia.GetSize();
using (StereoBM leftMatcher = new StereoBM())
using (RightMatcher rightMatcher = new RightMatcher(leftMatcher))
using (Mat leftDisparity = new Mat())
using (Mat rightDisparity = new Mat())
using (DisparityWLSFilter wls = new DisparityWLSFilter(leftMatcher))
{
leftMatcher.Compute(left, right, leftDisparity);
rightMatcher.Compute(right, left, rightDisparity);
wls.Filter(leftDisparity, left, outputDisparityMap, rightDisparity, rightView: right);
float scale = Math.Max(size.Width, size.Height);
//Construct a simple Q matrix, if you have a matrix from cvStereoRectify, you should use that instead
using (Matrix <double> q = new Matrix <double>(
new double[, ]
{
{ 1.0, 0.0, 0.0, -size.Width / 2 }, //shift the x origin to image center
{ 0.0, -1.0, 0.0, size.Height / 2 }, //shift the y origin to image center and flip it upside down
{ 0.0, 0.0, -1.0, 0.0 }, //Multiply the z value by -1.0,
{ 0.0, 0.0, 0.0, scale }
})) //scale the object's coordinate to within a [-0.5, 0.5] cube
{
CvInvoke.ReprojectImageTo3D(outputDisparityMap, points, q, handleMissingValues, DepthType.Cv32F);
//CvInvoke.ReprojectImageTo3D(leftDisparity, points, q, false, DepthType.Cv32F);
//CvInvoke.ReprojectImageTo3D(leftDisparity, points, q, handleMissingValues, DepthType.Cv32F);
}
//points = PointCollection.ReprojectImageTo3D(outputDisparityMap, q);
}
}
public override void SetImage(IInputArray image)
{
if (image == null)
{
Xamarin.Forms.Device.BeginInvokeOnMainThread(() =>
{
_imageView.SetImageBitmap(null);
});
return;
}
int bufferIdx = _renderBufferIdx;
Bitmap buffer;
_renderBufferIdx = (_renderBufferIdx + 1) % _renderBuffer.Length;
using (InputArray iaImage = image.GetInputArray())
using (Mat mat = iaImage.GetMat())
{
if (_renderBuffer[bufferIdx] == null)
{
buffer = mat.ToBitmap();
_renderBuffer[bufferIdx] = buffer;
}
else
{
var size = iaImage.GetSize();
buffer = _renderBuffer[bufferIdx];
if (buffer.Width != size.Width || buffer.Height != size.Height)
{
buffer.Dispose();
_renderBuffer[bufferIdx] = mat.ToBitmap();
}
else
{
mat.ToBitmap(buffer);
}
}
}
Xamarin.Forms.Device.BeginInvokeOnMainThread(() =>
{
_imageView.SetImageBitmap(buffer);
});
}
/// <summary>
/// Create an ImageViewer from the specific <paramref name="image"/>
/// </summary>
/// <param name="image">The image to be displayed in this viewer</param>
public ImageViewer(IInputArray image)
: this()
{
if (image != null)
{
using (InputArray iaImage = image.GetInputArray())
{
Size size = iaImage.GetSize();
size.Width += 12;
size.Height += 38;
if (!Size.Equals(size))
{
Size = size;
}
}
}
Image = image;
}
/*
* public void InvokeOnImagesLoaded(Mat[] images)
* {
* if (OnImagesLoaded != null)
* OnImagesLoaded(this, images);
* }
*
* public event EventHandler<Mat[]> OnImagesLoaded;
*/
//private byte[] _imageData;
//private MemoryStream _imageStream;
public virtual void SetImage(IInputArray image)
{
if (image == null)
{
Xamarin.Forms.Device.BeginInvokeOnMainThread(
() =>
{
this.DisplayImage.Source = null;
this.DisplayImage.IsVisible = false;
});
return;
}
int width = 0;
int height = 0;
using (InputArray iaImage = image.GetInputArray())
{
System.Drawing.Size s = iaImage.GetSize();
width = s.Width;
height = s.Height;
}
using (VectorOfByte vb = new VectorOfByte())
{
//CvInvoke.Imencode(".jpg", image, vb);
CvInvoke.Imencode(".png", image, vb);
byte[] rawData = vb.ToArray();
//_imageData = vb.ToArray();
//_imageStream = new MemoryStream(_imageData);
Xamarin.Forms.Device.BeginInvokeOnMainThread(
() =>
{
this.DisplayImage.IsVisible = true;
this.DisplayImage.Source = ImageSource.FromStream(() => new MemoryStream(rawData));
this.DisplayImage.WidthRequest = Math.Min(this.Width, width);
this.DisplayImage.HeightRequest = height;
//this.MainLayout.ForceLayout();
//this.ForceLayout();
//var bounds = this.DisplayImage.Bounds;
});
}
}
public void SetImage(IInputArray image)
{
if (image == null)
{
this.DisplayImage.Source = null;
return;
}
using (VectorOfByte vb = new VectorOfByte())
{
CvInvoke.Imencode(".jpg", image, vb);
byte[] rawData = vb.ToArray();
this.DisplayImage.Source = ImageSource.FromStream(() => new MemoryStream(rawData));
#if __MACOS__
using (InputArray iaImage = image.GetInputArray())
this.DisplayImage.HeightRequest = iaImage.GetSize().Height;
#endif
}
}
/// <summary>
/// Detect faces on the image
/// </summary>
/// <param name="image">The image.</param>
/// <param name="fullFaceRegions">The faces where a full facial region is detected. These images can be send to facial landmark recognition for further processing.</param>
/// <param name="partialFaceRegions">The face region of which is close to the edge of the images. Because if may not contains all the facial landmarks, it is not recommended to send these regions to facial landmark detection.</param>
/// <param name="confidenceThreshold">The confident threshold for face detection</param>
/// <param name="nmsThreshold">The non maximum suppression threshold for face detection.</param>
public void Detect(IInputArray image, List <DetectedObject> fullFaceRegions, List <DetectedObject> partialFaceRegions, float confidenceThreshold = 0.5f, float nmsThreshold = 0.0f)
{
using (InputArray iaImage = image.GetInputArray())
{
DetectedObject[] detectedFaces = _faceDetectionModel.Detect(image, confidenceThreshold, nmsThreshold);
Rectangle imageRegion = new Rectangle(Point.Empty, iaImage.GetSize());
foreach (DetectedObject face in detectedFaces)
{
if (imageRegion.Contains(face.Region))
{
fullFaceRegions.Add(face);
}
else
{
partialFaceRegions.Add(face);
}
}
}
}
/// <summary>
/// Get the color at the specific location of the image
/// </summary>
/// <param name="image">The image to obtain pixel value from</param>
/// <param name="location">The location to sample a pixel</param>
/// <returns>The color at the specific location</returns>
public static IColor GetPixelColor(IInputArray image, Point location)
{
using (InputArray ia = image.GetInputArray())
{
Size size = ia.GetSize();
location.X = Math.Min(location.X, size.Width - 1);
location.Y = Math.Min(location.Y, size.Height - 1);
MethodInfo indexers =
image.GetType()
.GetMethod("get_Item", new Type[2] {
typeof(int), typeof(int)
});
return(indexers == null
? new Bgra()
: indexers.Invoke(image, new object[2] {
location.Y, location.X
}) as IColor);
}
}
private void AddLabelAndImage(ref Point startPoint, String labelText, IInputArray image)
{
Label label = new Label();
panel1.Controls.Add(label);
label.Text = labelText;
label.Width = 100;
label.Height = 30;
label.Location = startPoint;
startPoint.Y += label.Height;
ImageBox box = new ImageBox();
panel1.Controls.Add(box);
using (InputArray iaImage = image.GetInputArray())
{
box.ClientSize = iaImage.GetSize();
box.Image = image;
box.Location = startPoint;
startPoint.Y += box.Height + 10;
}
}
/// <summary>
/// Set the mouse position over the image.
/// It also set the color intensity of the pixel on the image where is mouse is at
/// </summary>
/// <param name="location">The location of the mouse on the image</param>
public void SetMousePositionOnImage(Point location)
{
IInputArray img = _imageBox.DisplayedImage;
using (InputArray iaImage = img.GetInputArray())
{
Size size = iaImage.GetSize();
location.X = Math.Max(Math.Min(location.X, size.Width - 1), 0);
location.Y = Math.Max(Math.Min(location.Y, size.Height - 1), 0);
mousePositionTextbox.Text = location.ToString();
if (_imageType == typeof(CvArray <>))
{
using (Mat mat = iaImage.GetMat())
{
RenderIntensityForMat(mat, location);
}
}
else if (_imageType == typeof(Mat))
{
Mat mat = img as Mat;
RenderIntensityForMat(mat, location);
}
else if (_imageType == typeof(UMat))
{
UMat umat = img as UMat;
using (Mat mat = umat.GetMat(AccessType.Read))
{
RenderIntensityForMat(mat, location);
}
}
else
{
colorIntensityTextbox.Text = String.Empty;
}
}
}
public void SetImage(IInputArray image)
{
#region display the size of the image
if (image != null)
{
using (InputArray iaImage = image.GetInputArray())
{
Size size = iaImage.GetSize();
widthTextbox.Text = size.Width.ToString();
heightTextBox.Text = size.Height.ToString();
}
}
else
{
widthTextbox.Text = String.Empty;
heightTextBox.Text = string.Empty;
}
#endregion
#region display the color type of the image
if (image != null)
{
Type colorType = Reflection.ReflectIImage.GetTypeOfColor(image);
Object[] colorAttributes = colorType.GetCustomAttributes(typeof(ColorInfoAttribute), true);
if (colorAttributes.Length > 0)
{
ColorInfoAttribute info = (ColorInfoAttribute)colorAttributes[0];
typeOfColorTexbox.Text = info.ConversionCodename;
}
else
{
typeOfColorTexbox.Text = Properties.StringTable.Unknown;
}
Type colorDepth = Reflection.ReflectIImage.GetTypeOfDepth(image);
typeOfDepthTextBox.Text = colorDepth.Name;
}
else
{
typeOfColorTexbox.Text = string.Empty;
typeOfDepthTextBox.Text = string.Empty;
}
#endregion
#region check if image is a subclass of CvArr type
if (image != null)
{
Type imgType = image.GetType();
if (IsSubTypeOf(imgType, typeof(CvArray <>)))
{
_imageType = typeof(CvArray <>);
}
else if (IsSubTypeOf(imgType, typeof(Mat)))
{
_imageType = typeof(Mat);
}
else if (IsSubTypeOf(imgType, typeof(UMat)))
{
_imageType = typeof(UMat);
}
else
{
_imageType = null;
}
}
else
{
_imageType = null;
}
#endregion
UpdateHistogram();
UpdateZoomScale();
}
/// <summary>
/// Perform detection on the input image and return the results
/// </summary>
/// <param name="m">The input image</param>
/// <param name="matchScoreThreshold">A threshold used to filter boxes by score.</param>
/// <param name="nmsThreshold">A threshold used in non maximum suppression.</param>
/// <returns>The detected objects</returns>
public MaskedObject[] Detect(IInputArray m, float matchScoreThreshold = 0.5f, float nmsThreshold = 0.4f)
{
using (InputArray iaM = m.GetInputArray())
using (Mat blob = DnnInvoke.BlobFromImage(m))
using (VectorOfMat tensors = new VectorOfMat())
{
_maskRcnnDetector.SetInput(blob, "image_tensor");
_maskRcnnDetector.Forward(tensors, new string[] { "detection_out_final", "detection_masks" });
using (Mat boxes = tensors[0])
using (Mat masks = tensors[1])
{
System.Drawing.Size imgSize = iaM.GetSize();
float[,,,] boxesData = boxes.GetData(true) as float[, , , ];
int numDetections = boxesData.GetLength(2);
List <int> classIds = new List <int>();
List <Rectangle> regions = new List <Rectangle>();
List <float> scores = new List <float>();
for (int i = 0; i < numDetections; i++)
{
int classId = (int)boxesData[0, 0, i, 1];
if (_objectsOfInterest == null || _objectsOfInterest.Contains(_labels[classId]))
{
float score = boxesData[0, 0, i, 2];
Rectangle rect = DetectedObject.GetRectangle(
boxesData[0, 0, i, 3],
boxesData[0, 0, i, 4],
boxesData[0, 0, i, 5],
boxesData[0, 0, i, 6],
imgSize.Width,
imgSize.Height);
rect.Intersect(new Rectangle(Point.Empty, imgSize));
regions.Add(rect);
scores.Add(score);
classIds.Add(classId);
}
}
int[] validIdx = DnnInvoke.NMSBoxes(regions.ToArray(), scores.ToArray(), matchScoreThreshold, nmsThreshold);
List <MaskedObject> maskedObjects = new List <MaskedObject>();
for (int i = 0; i < validIdx.Length; i++)
{
int idx = validIdx[i];
int classId = classIds[idx];
Rectangle rect = regions[idx];
float score = scores[idx];
int[] masksDim = masks.SizeOfDimension;
using (Mat mask = new Mat(
masksDim[2],
masksDim[3],
DepthType.Cv32F,
1,
masks.GetDataPointer(i, classId),
masksDim[3] * masks.ElementSize))
{
MaskedObject mo = new MaskedObject(classId, _labels[classId], score, rect, mask);
maskedObjects.Add(mo);
}
}
return(maskedObjects.ToArray());
}
}
}