public double[] GrabScreen(Size resultSize = default(Size))
{
_sightService.GrabScreen();
using (var playArea = _sightService.LookAt(playAreaRectangle)) //Crop Region of Interest(ROI)
//using (var playArea = playAreaRaw.CvtColor(ColorConversionCodes.BGR2GRAY)) //RGB to Grey Scale
{
using (var playAreaOfDoubles = new MatOfDouble(playArea))
{
double[] result;
if (resultSize != default(Size))
{
using (var playAreaResized = playArea.Resize(resultSize))
{
result = playAreaResized.GetArray(0, 0);
}
}
else
{
result = playAreaOfDoubles.GetArray(0, 0);
}
Cv2.ImWrite($"{Guid.NewGuid()}.png", playArea);
return(result);
}
}
}
private void detectShapeCandidates(ref Bitmap bitmap, Boolean saveShapes)
{
Debug.WriteLine("Running OpenCV");
string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
Mat colorMat = BitmapConverter.ToMat(bitmap);
MatOfDouble mu = new MatOfDouble();
MatOfDouble sigma = new MatOfDouble();
Cv2.MeanStdDev(colorMat, mu, sigma);
double mean = mu.GetArray(0, 0)[0];
mu.Dispose();
sigma.Dispose();
Mat greyMat = new Mat();
Cv2.CvtColor(colorMat, greyMat, ColorConversion.BgraToGray, 0);
greyMat = greyMat.GaussianBlur(new OpenCvSharp.CPlusPlus.Size(1, 1), 5, 5, BorderType.Default);
greyMat = greyMat.Canny(0.5 * mean, 1.2 * mean, 3, true);
Mat contourMat = new Mat(greyMat.Size(), colorMat.Type());
greyMat.CopyTo(contourMat);
var contours = contourMat.FindContoursAsArray(ContourRetrieval.List, ContourChain.ApproxSimple);
for (int j = 0; j < contours.Length; j++)
{
var poly = Cv2.ApproxPolyDP(contours[j], 0.01 * Cv2.ArcLength(contours[j], true), true);
int num = poly.Length;
if (num >= 4 && num < 20)
{
var color = Scalar.Blue;
var rect = Cv2.BoundingRect(poly);
if (rect.Height < 20 || rect.Width < 20)
{
continue;
}
if (saveShapes)
{
string path = Path.Combine(myPhotos, "shape_samples");
path = Path.Combine(path, "shape_sample_" + Path.GetRandomFileName() + ".png");
var matRect = new OpenCvSharp.CPlusPlus.Rect(0, 0, greyMat.Width, greyMat.Height);
rect.Inflate((int)(rect.Width * 0.1), (int)(rect.Height * 0.1));
rect = rect.Intersect(matRect);
Mat shapeMat = greyMat.SubMat(rect);
var size = new OpenCvSharp.CPlusPlus.Size(128, 128);
shapeMat = shapeMat.Resize(size);
Bitmap shape = shapeMat.ToBitmap();
shape.Save(path);
shape.Dispose();
shapeMat.Dispose();
continue;
}
Cv2.Rectangle(colorMat, rect, color, 2);
}
}
bitmap = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(colorMat);
colorMat.Dispose();
greyMat.Dispose();
contourMat.Dispose();
}
private void detectShapeCandidates(ref Bitmap bitmap, Boolean saveShapes)
{
string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
Mat colorMat = BitmapConverter.ToMat(bitmap);
MatOfDouble mu = new MatOfDouble();
MatOfDouble sigma = new MatOfDouble();
Cv2.MeanStdDev(colorMat, mu, sigma);
double mean = mu.GetArray(0, 0)[0];
mu.Dispose();
sigma.Dispose();
Mat greyMat = new Mat();
Cv2.CvtColor(colorMat, greyMat, ColorConversion.BgraToGray, 0);
greyMat = greyMat.GaussianBlur(new OpenCvSharp.CPlusPlus.Size(1, 1), 5, 5, BorderType.Default);
greyMat = greyMat.Canny(0.5 * mean, 1.2 * mean, 3, true);
Mat contourMat = new Mat(greyMat.Size(), colorMat.Type());
greyMat.CopyTo(contourMat);
var contours = contourMat.FindContoursAsArray(ContourRetrieval.List, ContourChain.ApproxSimple);
this.controls.Clear();
for (int j = 0; j < contours.Length; j++)
{
var poly = Cv2.ApproxPolyDP(contours[j], 0.01 * Cv2.ArcLength(contours[j], true), true);
int num = poly.Length;
if (num >= 4 && num < 20)
{
var color = Scalar.Blue;
var rect = Cv2.BoundingRect(poly);
if (rect.Height < 20 || rect.Width < 20)
{
continue;
}
if (saveShapes)
{
string path = Path.Combine(myPhotos, "shape_samples");
path = Path.Combine(path, "shape_sample_" + Path.GetRandomFileName() + ".png");
Mat shapeMat = preprocessShape(rect, greyMat);
Bitmap shape = shapeMat.ToBitmap();
shape.Save(path);
shape.Dispose();
shapeMat.Dispose();
continue;
}
if (shapeSVM != null)
{
Mat shapeMat = preprocessShape(rect, greyMat);
float shapeClass = classifyShape(shapeMat, shapeSVM);
if (shapeClass >= 0)
{
Shape shape = null;
switch ((int)shapeClass)
{
case 0:
color = Scalar.Red;
shape = new Shape(Shape.ShapeType.SQUARE, rect);
break;
case 1:
color = Scalar.Yellow;
shape = new Shape(Shape.ShapeType.CIRCLE, rect);
break;
case 2:
color = Scalar.Green;
shape = new Shape(Shape.ShapeType.SLIDER, rect);
break;
}
Cv2.Rectangle(colorMat, rect, color, 2);
this.controls.Add(shape);
}
shapeMat.Dispose();
}
else
{
Cv2.Rectangle(colorMat, rect, color, 2);
}
}
}
bitmap = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(colorMat);
colorMat.Dispose();
greyMat.Dispose();
contourMat.Dispose();
}
private unsafe void OpenCV(ref Bitmap bitmap)
{
Mat testMat = BitmapConverter.ToMat(bitmap);
MatOfDouble mu = new MatOfDouble();
MatOfDouble sigma = new MatOfDouble();
Cv2.MeanStdDev(testMat, mu, sigma);
double mean = mu.GetArray(0, 0)[0];
mu.Dispose();
sigma.Dispose();
SimpleBlobDetector.Params circleParameters = new SimpleBlobDetector.Params();
circleParameters.FilterByCircularity = true;
circleParameters.MinCircularity = (float)0.85;
circleParameters.MaxCircularity = (float)1;
circleParameters.MinArea = 30; // Modify the value on the fly (TODO use bigger circle)
SimpleBlobDetector detectCircleBlobs = new SimpleBlobDetector(circleParameters);
fingerPoints = detectCircleBlobs.Detect(testMat);
detectCircleBlobs.Dispose();
// If Finger found basically
if (fingerPoints != null)
{
this.fingerSize = 0;
int fingerIndex = -1;
for (int i = 0; i < fingerPoints.Length; i++)
{
if (fingerPoints[i].Size >= this.fingerSize)
{
this.fingerSize = (int)fingerPoints[i].Size;
fingerIndex = i;
}
}
if (fingerIndex != -1)
{
OpenCvSharp.CPlusPlus.Point coordinate = fingerPoints[fingerIndex].Pt;
this.fingerSize = (int)((fingerPoints[fingerIndex].Size) * Math.Sqrt(2));
testMat.Set <Vec3b>(coordinate.Y, coordinate.X, new Vec3b(0, 255, 0));
RotatedRect rRect = new RotatedRect(new Point2f(coordinate.X, coordinate.Y), new Size2f(this.fingerSize, this.fingerSize), 0);
Point2f[] circleVerticies = rRect.Points();
//this.fingerCoordinates[0] = coordinate.X;
//this.fingerCoordinates[1] = coordinate.Y;
int height = (int)(circleVerticies[0].Y - circleVerticies[1].Y);
int width = (int)(circleVerticies[2].X - circleVerticies[1].X);
int startX = (int)(circleVerticies[0].X);
int startY = (int)(circleVerticies[1].Y);
this.fingerDepth = MapColortoDepth(startX, startY, this.fingerSize, this.fingerSize);
OpenCvSharp.CPlusPlus.Rect featureRect = new OpenCvSharp.CPlusPlus.Rect(startX, startY, this.fingerSize, this.fingerSize);
// Draw box around finger
for (int j = 0; j < 4; j++)
{
Cv2.Line(testMat, circleVerticies[j], circleVerticies[(j + 1) % 4], new Scalar(0, 255, 0));
}
Boolean intersectOccurance = false;
List <int> intersectIndicies = new List <int>();
for (int i = 0; i < this.controls.Count; i++)
{
if (this.controls[i].boundingRect.IntersectsWith(featureRect))
{
double diff = fingerDepth - this.controls[i].depth;
if (Math.Abs(diff) < 0.5)
{
intersectOccurance = true;
intersectIndicies.Add(i);
}
}
}
System.Text.StringBuilder append = new System.Text.StringBuilder();
if (intersectOccurance)
{
for (int i = 0; i < intersectIndicies.Count; i++)
{
append.Append(" " + this.controls[intersectIndicies[i]].title + " " + intersectIndicies[i].ToString());
}
this.OutputText = "Pressed Button" + append; //TODO Make this more obvious
}
else
{
this.OutputText = "No State";
}
}
}
bitmap = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(testMat);
testMat.Dispose();
}
