/// <summary>
///
/// </summary>
/// <param name="glyphs"></param>
/// <param name="gaze"></param>
/// <param name="diameter"></param>in percent (e.g. 150% of marker size)
/// <returns></returns>
public string GetGazedGlyph(AForge.Point gaze, double diameter)
{
ExtractedGlyphData myGlyph = null; //foundGlyphs[0];
double min = 100000;
AForge.IntPoint minXY, maxXY;
AForge.IntPoint center;
foreach (ExtractedGlyphData glyphData in foundGlyphs)
{
// get glyph's center point
PointsCloud.GetBoundingRectangle(glyphData.Quadrilateral, out minXY, out maxXY);
center = (minXY + maxXY) / 2;
double dis = GetDistance(new Point(center.X, center.Y), new Point((int)gaze.X, (int)gaze.Y));
if (dis < min)
{
myGlyph = glyphData;
min = dis;
}
}
if (myGlyph != null)
{
PointsCloud.GetBoundingRectangle(myGlyph.Quadrilateral, out minXY, out maxXY);
double glyphDiameter = Math.Sqrt(2) * ((maxXY - minXY).X);
double acceptedDis = (diameter * (glyphDiameter / 2));
if (min > acceptedDis)
{
myGlyph = null;
}
}
gazedMarkerHistory_Update(myGlyph, gaze);
string name = "";
if (myGlyph != null)
{
name = myGlyph.RecognizedGlyph.Name;
}
return(name);
}
public void gazedMarkerHistory_Update(ExtractedGlyphData glyph, AForge.Point gaze)
{
gazedMarkerHistory_Shift();
gazedMarkerHistory[0].name = "";
gazedMarkerHistory[0].tag = "";
if (glyph != null)
{
AForge.IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(glyph.Quadrilateral, out minXY, out maxXY);
AForge.IntPoint center = (minXY + maxXY) / 2;
gazedMarkerHistory[0].tag = "";
gazedMarkerHistory[0].gaze = gaze;
gazedMarkerHistory[0].markerCenter = new Point(center.X, center.Y);
gazedMarkerHistory[0].name = glyph.RecognizedGlyph != null? glyph.RecognizedGlyph.Name:"";
}
}
/// <summary>
/// distance is metured in Marker diameter unit. e.g. 1 diameter 2 diameter ....
/// </summary>
/// <param name="glyph"></param>
/// <param name="gaze"></param>
/// <returns></returns>
public double getGazeDistance(ExtractedGlyphData glyph, AForge.Point gaze)
{
double dis = 0;
AForge.IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(glyph.Quadrilateral, out minXY, out maxXY);
AForge.IntPoint center = (minXY + maxXY) / 2;
dis = GetDistance(new Point(center.X, center.Y), new Point((int)gaze.X, (int)gaze.Y));
PointsCloud.GetBoundingRectangle(glyph.Quadrilateral, out minXY, out maxXY);
double glyphDiameter = Math.Sqrt(2) * ((maxXY - minXY).X);
dis = dis / glyphDiameter;
return(dis);
}
/// <summary>
/// anglebetween gazepoint, glyph center and top point of the glyph
/// </summary>
/// <param name="glyph"></param>
/// <param name="gaze"></param>
/// <returns></returns>
public int getGazeAngle(ExtractedGlyphData glyph, AForge.Point gaze)
{
int angle = 0;
//gaze
AForge.IntPoint g = new AForge.IntPoint(Convert.ToInt32(gaze.X), Convert.ToInt32(gaze.Y));
//top point
Point topCenter = new Point(0, 0);
int i = 0, j = 1;
topCenter.X = (glyph.Quadrilateral[i].X + glyph.Quadrilateral[j].X) / 2;
topCenter.Y = (glyph.Quadrilateral[i].Y + glyph.Quadrilateral[j].Y) / 2;
AForge.IntPoint top = new AForge.IntPoint(topCenter.X, topCenter.Y);
//center
AForge.IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(glyph.Quadrilateral, out minXY, out maxXY);
AForge.IntPoint center = (minXY + maxXY) / 2;
AForge.IntPoint cnt = new AForge.IntPoint(center.X, center.Y);
angle = (int)(AForge.Math.Geometry.GeometryTools.GetAngleBetweenVectors(cnt, g, top));
int direc = 1;
if ((cnt.X - top.X) != 0 && (g.Y - cnt.Y - ((cnt.Y - top.Y) / (cnt.X - top.X)) * (g.X - cnt.X)) < 0)
{
direc = -1;
}
// if (top.X < cnt.X) direc *= -1;
angle *= direc;
return(angle);
}
private void process_image(PictureBox box, bool toinvert)
{
Bitmap original = (Bitmap)Bitmap.FromFile(_fname);
original.Save("C:\\users\\alberto geniola\\desktop\\dbg\\original_" + toinvert + ".bmp");
// Setup the Blob counter
BlobCounter blobCounter = new BlobCounter();
blobCounter.BackgroundThreshold = Color.FromArgb(10, 10, 10);
blobCounter.FilterBlobs = true;
blobCounter.CoupledSizeFiltering = false;
blobCounter.MinHeight = minHeight;
blobCounter.MinWidth = minWidth;
List <Blob> blobs = new List <Blob>();
// Button scanning
// Apply the grayscale. This is needed for AForge's filters
using (Bitmap grey_scaled = Grayscale.CommonAlgorithms.BT709.Apply(original))
{
// Invert the image if requested
if (toinvert)
{
Invert invert = new Invert();
invert.ApplyInPlace(grey_scaled);
}
using (Bitmap t1 = new Threshold(64).Apply(grey_scaled))
{
using (var tmp = new Bitmap(t1.Width, t1.Height))
{
using (Graphics g = Graphics.FromImage(tmp))
{
g.DrawImage(t1, 0, 0);
}
tmp.Save("C:\\users\\alberto geniola\\desktop\\dbg\\filtered_" + toinvert + ".bmp");
// The blob counter will analyze the bitmap looking for shapes
blobCounter.ProcessImage(tmp);
var tmparr = blobCounter.GetObjectsInformation();
blobs.AddRange(tmparr);
for (int i = 0, n = tmparr.Length; i < n; i++)
{
List <IntPoint> edgePoints = blobCounter.GetBlobsEdgePoints(blobs[i]);
if (edgePoints.Count > 1)
{
IntPoint p0, p1;
PointsCloud.GetBoundingRectangle(edgePoints, out p0, out p1);
var r = new Rectangle(p0.X, p0.Y, p1.X - p0.X, p1.Y - p0.Y);
// Skip any shape representing the border of the whole window ( +10px padding)
if (r.Width >= (original.Width - 10))
{
continue;
}
using (var g = Graphics.FromImage(tmp))
{
g.DrawRectangle(_marker, r);
}
}
}
tmp.Save("C:\\users\\alberto geniola\\desktop\\dbg\\processed_" + toinvert + ".bmp");
}
}
using (Bitmap t2 = new SISThreshold().Apply(grey_scaled))
{
using (var tmp = new Bitmap(t2.Width, t2.Height))
{
using (Graphics g = Graphics.FromImage(tmp))
{
g.DrawImage(t2, 0, 0);
}
tmp.Save("C:\\users\\alberto geniola\\desktop\\dbg\\t2_" + toinvert + ".bmp");
// The blob counter will analyze the bitmap looking for shapes
blobCounter.ProcessImage(tmp);
var tmparr = blobCounter.GetObjectsInformation();
blobs.AddRange(tmparr);
for (int i = 0, n = tmparr.Length; i < n; i++)
{
List <IntPoint> edgePoints = blobCounter.GetBlobsEdgePoints(blobs[i]);
if (edgePoints.Count > 1)
{
IntPoint p0, p1;
PointsCloud.GetBoundingRectangle(edgePoints, out p0, out p1);
var r = new Rectangle(p0.X, p0.Y, p1.X - p0.X, p1.Y - p0.Y);
// Skip any shape representing the border of the whole window ( +10px padding)
if (r.Width >= (original.Width - 10))
{
continue;
}
using (var g = Graphics.FromImage(tmp))
{
g.DrawRectangle(_marker, r);
}
}
}
tmp.Save("C:\\users\\alberto geniola\\desktop\\dbg\\t1_" + toinvert + ".bmp");
}
}
}
Bitmap test = (Bitmap)original.Clone();
// Let's analyze every single shape
for (int i = 0, n = blobs.Count; i < n; i++)
{
List <IntPoint> edgePoints = blobCounter.GetBlobsEdgePoints(blobs[i]);
if (edgePoints.Count > 1)
{
IntPoint p0, p1;
PointsCloud.GetBoundingRectangle(edgePoints, out p0, out p1);
var r = new Rectangle(p0.X, p0.Y, p1.X - p0.X, p1.Y - p0.Y);
// Skip any shape representing the border of the whole window ( +10px padding)
if (r.Width >= (original.Width - 10))
{
continue;
}
using (var g = Graphics.FromImage(test))
{
g.DrawRectangle(_marker, r);
}
// This is most-likely a button!
// Crop the image and pass it to the OCR engine for text recognition
using (Bitmap button = new Bitmap(r.Width, r.Height))
{
// Scan the original shape
String txt = null;
using (var g1 = Graphics.FromImage(button))
{
g1.DrawImage(original, 0, 0, r, GraphicsUnit.Pixel);
}
// Process OCR on that image
txt = scanButton(button);
if (String.IsNullOrEmpty(txt))
{
using (Bitmap tmp = Grayscale.CommonAlgorithms.BT709.Apply(button))
{
if (toinvert)
{
new Invert().ApplyInPlace(tmp);
}
new SISThreshold().ApplyInPlace(tmp);
txt = scanButton(tmp);
}
}
// If still nothing is found, repeat the analysis with the second version of the filter
if (String.IsNullOrEmpty(txt))
{
using (Bitmap tmp = Grayscale.CommonAlgorithms.BT709.Apply(button))
{
if (toinvert)
{
new Invert().ApplyInPlace(tmp);
}
new Threshold(64).ApplyInPlace(tmp);
txt = scanButton(tmp);
}
}
if (!String.IsNullOrEmpty(txt))
{
using (var g = Graphics.FromImage(test))
{
int SPACING = 5;
double angle = 45 * 2 * Math.PI / 360; // 45 degrees to radiants
for (int x = 0; x < r.Width; x += SPACING)
{
PointF start = new PointF(r.X + x, r.Y);
PointF end = new PointF((float)(r.X + x + r.Height * Math.Tan(angle)), r.Y + r.Height);
if (end.X > (r.X + r.Width))
{
// Calculate midpoint
var delta = end.X - r.Width;
end.X = r.X + r.Width;
end.Y = r.Y + (float)(Math.Tan(angle) * r.Width) - x;
// Draw the overflow line
g.DrawLine(_marker2, r.X, end.Y, delta, r.Y + r.Height);
}
g.DrawLine(_marker2, start, end);
}
g.FillRectangle(_b, r);
var dim = g.MeasureString(txt.Trim(), _f);
g.DrawString(txt.Trim().ToUpper(), _f, _b2, r.X + (r.Width - dim.Width) / 2, r.Y + (r.Height - dim.Height) / 2);
}
}
test.Save("C:\\users\\alberto geniola\\desktop\\dbg\\processed_" + toinvert + ".bmp");
/*
* // At this point we should have a result. Add it to list if it does not overlap any UIAutomated element
* UIControlCandidate t = new UIControlCandidate();
* t.PositionWindowRelative = r;
* var winLoc = w.WindowLocation;
* t.PositionScreenRelative = new Rectangle(r.X + winLoc.X, r.Y + winLoc.Y, r.Width, r.Height);
* t.Text = txt;
* t.Score = policy.RankElement(t);
*
* // If the item falls into the same area of a UI element, ignore it.
* bool overlaps = false;
* foreach (var el in res)
* {
* if (el.AutoElementRef != null && el.PositionScreenRelative.IntersectsWith(t.PositionScreenRelative))
* {
* overlaps = true;
* break;
* }
* }
* if (!overlaps)
* res.Add(t);
*/
}
}
box.Image = test;
}
}
// Process image searching for glyphs and highlighting them
public List <ExtractedGlyphData> ProcessImage(Bitmap image)
{
var glyphs = new List <ExtractedGlyphData>();
lock (_sync)
{
_glyphTracker.ImageSize = image.Size;
// get list of recognized glyphs
glyphs.AddRange(_recognizer.FindGlyphs(image));
if (glyphs.Count <= 0)
{
return(glyphs);
}
// visualizing glyphs
var glyphIDs = _glyphTracker.TrackGlyphs(glyphs);
var g = Graphics.FromImage(image);
var i = 0;
// highlight each found glyph
foreach (var glyphData in glyphs)
{
var glyphPoints = (glyphData.RecognizedGlyph == null)
? glyphData.Quadrilateral
: glyphData.RecognizedQuadrilateral;
var pen = new Pen(Color.Red);
// highlight border
g.DrawPolygon(pen, ToPointsArray(glyphPoints));
// prepare glyph's title
var glyphTitle = glyphData.RecognizedGlyph != null
? glyphData.RecognizedGlyph.Name
: string.Format("ID: {0}", glyphIDs[i]);
// show glyph's title
if (!string.IsNullOrEmpty(glyphTitle))
{
// get glyph's center point
IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(glyphPoints, out minXY, out maxXY);
var center = (minXY + maxXY) / 2;
// glyph's name size
var nameSize = g.MeasureString(glyphTitle, _defaultFont);
// paint the name
var brush = new SolidBrush(pen.Color);
g.DrawString(glyphTitle, _defaultFont, brush,
new System.Drawing.Point(center.X - (int)nameSize.Width / 2,
center.Y - (int)nameSize.Height / 2));
brush.Dispose();
}
i++;
pen.Dispose();
}
}
return(glyphs);
}
private void altProcess(Bitmap bm, int level)
{
var img = new Image <Bgr, byte>(bm);
if (level == 1)
{
var resImage = new Image <Bgr, byte>(img.Bitmap);
CvInvoke.BilateralFilter(resImage, img, 30, 80, 80);
CvInvoke.MedianBlur(img, img, 5);
resImage = img;
}
else if (level == 2)
{
CvInvoke.MedianBlur(img, img, 5);
var resImage = new Image <Bgr, byte>(img.Bitmap);
CvInvoke.BilateralFilter(resImage, img, 25, 75, 75);
CvInvoke.Blur(img, img, new Size(5, 5), new Point(0, 0));
}
var grayimage = new Image <Gray, byte>(bm);
CvInvoke.CvtColor(img, grayimage, ColorConversion.Bgr2Gray);
BlackBG(grayimage);
Console.WriteLine("Filtering done");
var cannyThreshold = GetKMeansThreshold(grayimage);
label2.Text = cannyThreshold.ToString();
Thresholding(grayimage, cannyThreshold);
Console.WriteLine("Canny threshold using KMEANS found " + cannyThreshold);
//Convert the image to grayscale and filter out the noise
var cannyEdges = new UMat();
Console.WriteLine("Canny threshold using KMEANS found " + cannyThreshold);
var uimage = new UMat();
CvInvoke.CvtColor(img, uimage, ColorConversion.Bgr2Gray);
CvInvoke.Canny(uimage, cannyEdges, cannyThreshold, cannyThreshold);
BlobCounter blobCounter = new BlobCounter( );
if (level == 1)
{
blobCounter.FilterBlobs = true;
blobCounter.MinHeight = 25;
blobCounter.MinWidth = 25;
blobCounter.ProcessImage(cannyEdges.Bitmap);
}
else
{
blobCounter.ProcessImage(grayimage.ToBitmap());
}
//blobCounter.ProcessImage(grayimage.ToBitmap());
Blob[] blobs = blobCounter.GetObjectsInformation( );
SimpleShapeChecker shapeChecker = new SimpleShapeChecker();
var triangleList = new List <Triangle2DF>();
var boxList = new List <RotatedRect>();
var circleList = new List <CircleF>();
Bitmap newBM = new Bitmap(img.Bitmap);
Graphics g = Graphics.FromImage(newBM);
Pen redPen = new Pen(Color.Red, 2);
Pen yellowPen = new Pen(Color.Yellow, 2);
Pen greenPen = new Pen(Color.Green, 2);
Pen bluePen = new Pen(Color.Blue, 2);
for (int i = 0, n = blobs.Length; i < n; i++)
{
List <IntPoint> edgePoints =
blobCounter.GetBlobsEdgePoints(blobs[i]);
AForge.Point center;
float radius;
if (shapeChecker.IsCircle(edgePoints, out center, out radius))
{
//g.DrawEllipse(bluePen,
// (float)(center.X - radius), (float)(center.Y - radius),
// (float)(radius * 2), (float)(radius * 2));
circleList.Add(new CircleF(new PointF(center.X, center.Y), radius));
}
else
{
List <IntPoint> corners;
if (edgePoints.Count > 1)
{
if (shapeChecker.IsQuadrilateral(edgePoints, out corners))
{
System.Console.WriteLine(corners.Count);
if (shapeChecker.CheckPolygonSubType(corners) ==
PolygonSubType.Square || shapeChecker.CheckPolygonSubType(corners) ==
PolygonSubType.Rectangle)
{
IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(corners, out minXY, out maxXY);
AForge.Point c = PointsCloud.GetCenterOfGravity(corners);
//g.DrawPolygon(greenPen, ToPointsArray(corners));
boxList.Add(new RotatedRect(new PointF(c.X, c.Y), new SizeF(maxXY.X - minXY.X, maxXY.Y - minXY.Y), 0));
}
}
else
{
corners = PointsCloud.FindQuadrilateralCorners(edgePoints);
if (corners.Count == 3)
{
Triangle2DF tri = new Triangle2DF(new PointF(corners[0].X, corners[0].Y), new PointF(corners[1].X, corners[1].Y), new PointF(corners[2].X, corners[2].Y));
triangleList.Add(tri);
//g.DrawPolygon(yellowPen, ToPointsArray(corners));
}
//g.DrawPolygon(redPen, ToPointsArray(corners));
}
}
}
}
Console.WriteLine("boxes " + boxList.Count);
Console.WriteLine("triangles " + triangleList.Count);
Console.WriteLine("circles " + circleList.Count);
redPen.Dispose();
greenPen.Dispose();
bluePen.Dispose();
yellowPen.Dispose();
//g.Dispose();
resPicBox.Image = newBM;
CircleF[] circles = circleList.ToArray();
var cList = circles.ToList();
FilterSame(boxList, triangleList, cList, img.Width * img.Height);
circles = cList.ToArray();
var points = new List <PointF>();
var Image = img.CopyBlank();
foreach (var triangle in triangleList)
{
Image.Draw(triangle, new Bgr(Color.Red), 3);
points.Add(triangle.Centeroid);
}
foreach (var box in boxList)
{
Image.Draw(box, new Bgr(Color.Blue), 3);
points.Add(box.Center);
}
foreach (var circle in circles)
{
Image.Draw(circle, new Bgr(Color.DarkCyan), 3);
points.Add(circle.Center);
}
var listPoints = SortPoints(points, img);
for (var i = 0; i < listPoints.Length; i++)
{
Console.WriteLine(listPoints[i].X.ToString() + " " + listPoints[i].Y.ToString());
}
System.Console.WriteLine("Points sorted, num of objects " + listPoints.Length.ToString());
resPicBox.Image = (Image + img).ToBitmap();
if (listPoints.Length > 3)
{
var bezSegList = InterpolatePointWithBeizerCurves(listPoints.ToList <PointF>());
var gr = Graphics.FromImage(resPicBox.Image);
var p = new Pen(Color.Red);
foreach (BeizerCurveSegment seg in bezSegList)
{
var bezierList = GetBez(new PointF[]
{ seg.StartPoint, seg.FirstControlPoint, seg.SecondControlPoint, seg.EndPoint });
for (var i = 0; i < bezierList.Length - 1; i++)
{
gr.DrawLine(p, bezierList[i], bezierList[i + 1]);
}
}
}
else
{
var gr = Graphics.FromImage(resPicBox.Image);
var p = new Pen(Color.Red);
for (var i = 0; i < listPoints.Length - 1; i++)
{
gr.DrawLine(p, listPoints[i], listPoints[i + 1]);
}
}
//var bezierList = GetBezierCurve1(listPoints);
}
public void VisualizeGlyph(Bitmap bitmap, Dictionary <string, string> names)
{
lock (this)
{
if (foundGlyphs.Count > 0)
{
if ((visualizationType == VisualizationType.BorderOnly) ||
(visualizationType == VisualizationType.Name))
{
Graphics g = Graphics.FromImage(bitmap);
// highlight each found glyph
foreach (ExtractedGlyphData glyphData in foundGlyphs)
{
if ((glyphData.RecognizedGlyph == null) || (glyphData.RecognizedGlyph.UserData == null))
{
// highlight with default pen
g.DrawPolygon(defaultPen, ToPointsArray(glyphData.Quadrilateral));
}
else
{
GlyphVisualizationData visualization =
(GlyphVisualizationData)glyphData.RecognizedGlyph.UserData;
Pen pen = new Pen(visualization.Color, 3);
// highlight border
g.DrawPolygon(pen, ToPointsArray(glyphData.Quadrilateral));
// show glyph's name
if (visualizationType == VisualizationType.Name)
{
// get glyph's center point
AForge.IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(glyphData.Quadrilateral, out minXY, out maxXY);
AForge.IntPoint center = (minXY + maxXY) / 2;
string GyphName = "";
if (names.ContainsKey(glyphData.RecognizedGlyph.Name))
{
GyphName = names[glyphData.RecognizedGlyph.Name];
}
// glyph's name size
SizeF nameSize = g.MeasureString(GyphName, defaultFont);
// paint the name
Brush brush = new SolidBrush(visualization.Color);
g.DrawString(GyphName, defaultFont, brush,
new Point(center.X - (int)nameSize.Width / 2, center.Y - (int)nameSize.Height / 2));
brush.Dispose( );
}
pen.Dispose( );
}
}
}
}
}
}
private void ProcessImage(Bitmap bitmap)
{
// lock image
BitmapData bitmapData = bitmap.LockBits(
new Rectangle(0, 0, bitmap.Width, bitmap.Height),
ImageLockMode.ReadWrite, bitmap.PixelFormat);
// step 1 - turn background to black
ColorFiltering colorFilter = new ColorFiltering();
colorFilter.Red = new IntRange(Properties.Settings.Default.camFilterRed1, Properties.Settings.Default.camFilterRed2);
colorFilter.Green = new IntRange(Properties.Settings.Default.camFilterGreen1, Properties.Settings.Default.camFilterGreen2);
colorFilter.Blue = new IntRange(Properties.Settings.Default.camFilterBlue1, Properties.Settings.Default.camFilterBlue2);
colorFilter.FillOutsideRange = Properties.Settings.Default.camFilterOutside;
colorFilter.ApplyInPlace(bitmapData);
// step 2 - locating objects
BlobCounter blobCounter = new BlobCounter();
blobCounter.FilterBlobs = true;
blobCounter.MinHeight = (int)Properties.Settings.Default.camShapeSizeMin * (int)cameraZoom;
blobCounter.MinWidth = (int)Properties.Settings.Default.camShapeSizeMin * (int)cameraZoom;
blobCounter.MaxHeight = (int)Properties.Settings.Default.camShapeSizeMax * (int)cameraZoom;
blobCounter.MaxWidth = (int)Properties.Settings.Default.camShapeSizeMax * (int)cameraZoom;
blobCounter.ProcessImage(bitmapData);
Blob[] blobs = blobCounter.GetObjectsInformation();
bitmap.UnlockBits(bitmapData);
// step 3 - check objects' type and highlight
SimpleShapeChecker shapeChecker = new SimpleShapeChecker();
shapeChecker.MinAcceptableDistortion = (float)Properties.Settings.Default.camShapeDist;
shapeChecker.RelativeDistortionLimit = (float)Properties.Settings.Default.camShapeDistMax;
Graphics g = Graphics.FromImage(bitmap);
Pen yellowPen = new Pen(Color.Yellow, 5); // circles
Pen redPen = new Pen(Color.Red, 10); // circles
Pen greenPen = new Pen(Color.Green, 5); // known triangle
double lowestDistance = xmid;
double distance;
shapeFound = false;
AForge.Point center;
double shapeRadius = 1;
for (int i = 0, n = blobs.Length; i < n; i++)
{
List <IntPoint> edgePoints = blobCounter.GetBlobsEdgePoints(blobs[i]);
System.Single radius;
// is circle ?
// g.DrawPolygon(greenPen, ToPointsArray(edgePoints));
if (Properties.Settings.Default.camShapeCircle && shapeChecker.IsCircle(edgePoints, out center, out radius))
{
shapeFound = true;
distance = center.DistanceTo((AForge.Point)picCenter);
g.DrawEllipse(yellowPen,
(float)(shapeCenter.X - shapeRadius), (float)(shapeCenter.Y - shapeRadius),
(float)(shapeRadius * 2), (float)(shapeRadius * 2));
if (lowestDistance > Math.Abs(distance))
{
lowestDistance = Math.Abs(distance);
shapeCenter = center;
shapeRadius = radius;
}
}
List <IntPoint> corners;
if (Properties.Settings.Default.camShapeRect && shapeChecker.IsQuadrilateral(edgePoints, out corners)) //.IsConvexPolygon
{
IntPoint minxy, maxxy, centxy;
shapeFound = true;
PolygonSubType subType = shapeChecker.CheckPolygonSubType(corners);
g.DrawPolygon(yellowPen, ToPointsArray(corners));
PointsCloud.GetBoundingRectangle(corners, out minxy, out maxxy);
centxy = (minxy + maxxy) / 2;
distance = picCenter.DistanceTo(centxy);// PointsCloud.GetCenterOfGravity(corners));
if (lowestDistance > Math.Abs(distance))
{
lowestDistance = Math.Abs(distance);
shapeCenter = centxy; // PointsCloud.GetCenterOfGravity(corners);
shapeRadius = maxxy.DistanceTo(minxy) / 2; // 50;
}
}
}
if (shapeFound)
{
g.DrawEllipse(redPen,
(float)(shapeCenter.X - shapeRadius * 1.2), (float)(shapeCenter.Y - shapeRadius * 1.2),
(float)(shapeRadius * 2.4), (float)(shapeRadius * 2.4));
}
yellowPen.Dispose();
redPen.Dispose();
greenPen.Dispose();
g.Dispose();
pictureBoxVideo.BackgroundImage = bitmap;
}
// Process both images transforming source image into quadrilateral in destination image
private unsafe void ProcessFilter(UnmanagedImage dstImage, UnmanagedImage srcImage)
{
// get source and destination images size
int srcWidth = srcImage.Width;
int srcHeight = srcImage.Height;
int dstWidth = dstImage.Width;
int dstHeight = dstImage.Height;
int pixelSize = Image.GetPixelFormatSize(srcImage.PixelFormat) / 8;
int srcStride = srcImage.Stride;
int dstStride = dstImage.Stride;
// get bounding rectangle of the quadrilateral
IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(destinationQuadrilateral, out minXY, out maxXY);
// make sure the rectangle is inside of destination image
if ((maxXY.X < 0) || (maxXY.Y < 0) || (minXY.X >= dstWidth) || (minXY.Y >= dstHeight))
{
return; // nothing to do, since quadrilateral is completely outside
}
// correct rectangle if required
if (minXY.X < 0)
{
minXY.X = 0;
}
if (minXY.Y < 0)
{
minXY.Y = 0;
}
if (maxXY.X >= dstWidth)
{
maxXY.X = dstWidth - 1;
}
if (maxXY.Y >= dstHeight)
{
maxXY.Y = dstHeight - 1;
}
int startX = minXY.X;
int startY = minXY.Y;
int stopX = maxXY.X + 1;
int stopY = maxXY.Y + 1;
int offset = dstStride - (stopX - startX) * pixelSize;
// calculate tranformation matrix
List <IntPoint> srcRect = new List <IntPoint>();
srcRect.Add(new IntPoint(0, 0));
srcRect.Add(new IntPoint(srcWidth - 1, 0));
srcRect.Add(new IntPoint(srcWidth - 1, srcHeight - 1));
srcRect.Add(new IntPoint(0, srcHeight - 1));
double[,] matrix = QuadTransformationCalcs.MapQuadToQuad(destinationQuadrilateral, srcRect);
// do the job
byte *ptr = (byte *)dstImage.ImageData.ToPointer();
byte *baseSrc = (byte *)srcImage.ImageData.ToPointer();
// allign pointer to the first pixel to process
ptr += (startY * dstStride + startX * pixelSize);
if (!useInterpolation)
{
byte *p;
// for each row
for (int y = startY; y < stopY; y++)
{
// for each pixel
for (int x = startX; x < stopX; x++)
{
double factor = matrix[2, 0] * x + matrix[2, 1] * y + matrix[2, 2];
double srcX = (matrix[0, 0] * x + matrix[0, 1] * y + matrix[0, 2]) / factor;
double srcY = (matrix[1, 0] * x + matrix[1, 1] * y + matrix[1, 2]) / factor;
if ((srcX >= 0) && (srcY >= 0) && (srcX < srcWidth) && (srcY < srcHeight))
{
// get pointer to the pixel in the source image
p = baseSrc + (int)srcY * srcStride + (int)srcX * pixelSize;
// copy pixel's values
for (int i = 0; i < pixelSize; i++, ptr++, p++)
{
*ptr = *p;
}
}
else
{
// skip the pixel
ptr += pixelSize;
}
}
ptr += offset;
}
}
else
{
int srcWidthM1 = srcWidth - 1;
int srcHeightM1 = srcHeight - 1;
// coordinates of source points
double dx1, dy1, dx2, dy2;
int sx1, sy1, sx2, sy2;
// temporary pointers
byte *p1, p2, p3, p4;
// for each row
for (int y = startY; y < stopY; y++)
{
// for each pixel
for (int x = startX; x < stopX; x++)
{
double factor = matrix[2, 0] * x + matrix[2, 1] * y + matrix[2, 2];
double srcX = (matrix[0, 0] * x + matrix[0, 1] * y + matrix[0, 2]) / factor;
double srcY = (matrix[1, 0] * x + matrix[1, 1] * y + matrix[1, 2]) / factor;
if ((srcX >= 0) && (srcY >= 0) && (srcX < srcWidth) && (srcY < srcHeight))
{
sx1 = (int)srcX;
sx2 = (sx1 == srcWidthM1) ? sx1 : sx1 + 1;
dx1 = srcX - sx1;
dx2 = 1.0 - dx1;
sy1 = (int)srcY;
sy2 = (sy1 == srcHeightM1) ? sy1 : sy1 + 1;
dy1 = srcY - sy1;
dy2 = 1.0 - dy1;
// get four points
p1 = p2 = baseSrc + sy1 * srcStride;
p1 += sx1 * pixelSize;
p2 += sx2 * pixelSize;
p3 = p4 = baseSrc + sy2 * srcStride;
p3 += sx1 * pixelSize;
p4 += sx2 * pixelSize;
// interpolate using 4 points
for (int i = 0; i < pixelSize; i++, ptr++, p1++, p2++, p3++, p4++)
{
*ptr = (byte)(
dy2 * (dx2 * (*p1) + dx1 * (*p2)) +
dy1 * (dx2 * (*p3) + dx1 * (*p4)));
}
}
else
{
// skip the pixel
ptr += pixelSize;
}
}
ptr += offset;
}
}
}
private unsafe void ProcessFilter(UnmanagedImage dstImage, UnmanagedImage srcImage)
{
int width = srcImage.Width;
int height = srcImage.Height;
int width2 = dstImage.Width;
int height2 = dstImage.Height;
int num = System.Drawing.Image.GetPixelFormatSize(srcImage.PixelFormat) / 8;
int stride = srcImage.Stride;
int stride2 = dstImage.Stride;
PointsCloud.GetBoundingRectangle(destinationQuadrilateral, out IntPoint minXY, out IntPoint maxXY);
if (maxXY.X < 0 || maxXY.Y < 0 || minXY.X >= width2 || minXY.Y >= height2)
{
return;
}
if (minXY.X < 0)
{
minXY.X = 0;
}
if (minXY.Y < 0)
{
minXY.Y = 0;
}
if (maxXY.X >= width2)
{
maxXY.X = width2 - 1;
}
if (maxXY.Y >= height2)
{
maxXY.Y = height2 - 1;
}
int x = minXY.X;
int y = minXY.Y;
int num2 = maxXY.X + 1;
int num3 = maxXY.Y + 1;
int num4 = stride2 - (num2 - x) * num;
List <IntPoint> list = new List <IntPoint>();
list.Add(new IntPoint(0, 0));
list.Add(new IntPoint(width - 1, 0));
list.Add(new IntPoint(width - 1, height - 1));
list.Add(new IntPoint(0, height - 1));
double[,] array = QuadTransformationCalcs.MapQuadToQuad(destinationQuadrilateral, list);
byte *ptr = (byte *)dstImage.ImageData.ToPointer();
byte *ptr2 = (byte *)srcImage.ImageData.ToPointer();
ptr += y * stride2 + x * num;
if (!useInterpolation)
{
for (int i = y; i < num3; i++)
{
for (int j = x; j < num2; j++)
{
double num5 = array[2, 0] * (double)j + array[2, 1] * (double)i + array[2, 2];
double num6 = (array[0, 0] * (double)j + array[0, 1] * (double)i + array[0, 2]) / num5;
double num7 = (array[1, 0] * (double)j + array[1, 1] * (double)i + array[1, 2]) / num5;
if (num6 >= 0.0 && num7 >= 0.0 && num6 < (double)width && num7 < (double)height)
{
byte *ptr3 = ptr2 + (long)(int)num7 * (long)stride + (long)(int)num6 * (long)num;
int num8 = 0;
while (num8 < num)
{
*ptr = *ptr3;
num8++;
ptr++;
ptr3++;
}
}
else
{
ptr += num;
}
}
ptr += num4;
}
return;
}
int num9 = width - 1;
int num10 = height - 1;
for (int k = y; k < num3; k++)
{
for (int l = x; l < num2; l++)
{
double num11 = array[2, 0] * (double)l + array[2, 1] * (double)k + array[2, 2];
double num12 = (array[0, 0] * (double)l + array[0, 1] * (double)k + array[0, 2]) / num11;
double num13 = (array[1, 0] * (double)l + array[1, 1] * (double)k + array[1, 2]) / num11;
if (num12 >= 0.0 && num13 >= 0.0 && num12 < (double)width && num13 < (double)height)
{
int num14 = (int)num12;
int num15 = (num14 == num9) ? num14 : (num14 + 1);
double num16 = num12 - (double)num14;
double num17 = 1.0 - num16;
int num18 = (int)num13;
int num19 = (num18 == num10) ? num18 : (num18 + 1);
double num20 = num13 - (double)num18;
double num21 = 1.0 - num20;
byte * ptr4;
byte * ptr5 = ptr4 = ptr2 + (long)num18 * (long)stride;
ptr5 += (long)num14 * (long)num;
ptr4 += (long)num15 * (long)num;
byte *ptr6;
byte *ptr7 = ptr6 = ptr2 + (long)num19 * (long)stride;
ptr7 += (long)num14 * (long)num;
ptr6 += (long)num15 * (long)num;
int num22 = 0;
while (num22 < num)
{
*ptr = (byte)(num21 * (num17 * (double)(int)(*ptr5) + num16 * (double)(int)(*ptr4)) + num20 * (num17 * (double)(int)(*ptr7) + num16 * (double)(int)(*ptr6)));
num22++;
ptr++;
ptr5++;
ptr4++;
ptr7++;
ptr6++;
}
}
else
{
ptr += num;
}
}
ptr += num4;
}
}
private unsafe void ProcessFilter(UnmanagedImage dstImage, UnmanagedImage srcImage)
{
IntPoint point;
IntPoint point2;
int width = srcImage.Width;
int height = srcImage.Height;
int num3 = dstImage.Width;
int num4 = dstImage.Height;
int num5 = Image.GetPixelFormatSize(srcImage.PixelFormat) / 8;
int stride = srcImage.Stride;
int num7 = dstImage.Stride;
PointsCloud.GetBoundingRectangle(this.destinationQuadrilateral, ref point, ref point2);
if (((point2.X >= 0) && (point2.Y >= 0)) && ((point.X < num3) && (point.Y < num4)))
{
if (point.X < 0)
{
point.X = 0;
}
if (point.Y < 0)
{
point.Y = 0;
}
if (point2.X >= num3)
{
point2.X = num3 - 1;
}
if (point2.Y >= num4)
{
point2.Y = num4 - 1;
}
int x = point.X;
int y = point.Y;
int num10 = point2.X + 1;
int num11 = point2.Y + 1;
int num12 = num7 - ((num10 - x) * num5);
List <IntPoint> output = new List <IntPoint> {
new IntPoint(0, 0),
new IntPoint(width - 1, 0),
new IntPoint(width - 1, height - 1),
new IntPoint(0, height - 1)
};
double[,] numArray = QuadTransformationCalcs.MapQuadToQuad(this.destinationQuadrilateral, output);
byte *numPtr = (byte *)dstImage.ImageData.ToPointer();
byte *numPtr2 = (byte *)srcImage.ImageData.ToPointer();
numPtr += (y * num7) + (x * num5);
if (!this.useInterpolation)
{
for (int i = y; i < num11; i++)
{
for (int j = x; j < num10; j++)
{
double num15 = ((numArray[2, 0] * j) + (numArray[2, 1] * i)) + numArray[2, 2];
double num16 = (((numArray[0, 0] * j) + (numArray[0, 1] * i)) + numArray[0, 2]) / num15;
double num17 = (((numArray[1, 0] * j) + (numArray[1, 1] * i)) + numArray[1, 2]) / num15;
if (((num16 >= 0.0) && (num17 >= 0.0)) && ((num16 < width) && (num17 < height)))
{
byte *numPtr3 = (numPtr2 + (((int)num17) * stride)) + (((int)num16) * num5);
int num18 = 0;
while (num18 < num5)
{
numPtr[0] = numPtr3[0];
num18++;
numPtr++;
numPtr3++;
}
}
else
{
numPtr += num5;
}
}
numPtr += num12;
}
}
else
{
int num19 = width - 1;
int num20 = height - 1;
for (int k = y; k < num11; k++)
{
for (int m = x; m < num10; m++)
{
double num31 = ((numArray[2, 0] * m) + (numArray[2, 1] * k)) + numArray[2, 2];
double num32 = (((numArray[0, 0] * m) + (numArray[0, 1] * k)) + numArray[0, 2]) / num31;
double num33 = (((numArray[1, 0] * m) + (numArray[1, 1] * k)) + numArray[1, 2]) / num31;
if (((num32 >= 0.0) && (num33 >= 0.0)) && ((num32 < width) && (num33 < height)))
{
byte * numPtr5;
byte * numPtr7;
int num25 = (int)num32;
int num27 = (num25 == num19) ? num25 : (num25 + 1);
double num21 = num32 - num25;
double num23 = 1.0 - num21;
int num26 = (int)num33;
int num28 = (num26 == num20) ? num26 : (num26 + 1);
double num22 = num33 - num26;
double num24 = 1.0 - num22;
byte * numPtr4 = numPtr5 = numPtr2 + (num26 * stride);
numPtr4 += num25 * num5;
numPtr5 += num27 * num5;
byte *numPtr6 = numPtr7 = numPtr2 + (num28 * stride);
numPtr6 += num25 * num5;
numPtr7 += num27 * num5;
int num34 = 0;
while (num34 < num5)
{
numPtr[0] = (byte)((num24 * ((num23 * numPtr4[0]) + (num21 * numPtr5[0]))) + (num22 * ((num23 * numPtr6[0]) + (num21 * numPtr7[0]))));
num34++;
numPtr++;
numPtr4++;
numPtr5++;
numPtr6++;
numPtr7++;
}
}
else
{
numPtr += num5;
}
}
numPtr += num12;
}
}
}
}
// Process image searching for glyphs and highlighting them
public List <ExtractedGlyphData> ProcessImage(Bitmap bitmap)
{
List <ExtractedGlyphData> glyphs = new List <ExtractedGlyphData>( );
lock ( sync )
{
glyphTracker.ImageSize = bitmap.Size;
// get list of recognized glyphs
glyphs.AddRange(recognizer.FindGlyphs(bitmap));
List <int> glyphIDs = glyphTracker.TrackGlyphs(glyphs);
if (glyphs.Count > 0)
{
if ((visualizationType == VisualizationType.BorderOnly) ||
(visualizationType == VisualizationType.Name))
{
Graphics g = Graphics.FromImage(bitmap);
int i = 0;
// highlight each found glyph
foreach (ExtractedGlyphData glyphData in glyphs)
{
List <IntPoint> glyphPoints = (glyphData.RecognizedGlyph == null) ?
glyphData.Quadrilateral : glyphData.RecognizedQuadrilateral;
Pen pen = new Pen(((glyphData.RecognizedGlyph == null) || (glyphData.RecognizedGlyph.UserData == null)) ?
Color.Red : ((GlyphVisualizationData)glyphData.RecognizedGlyph.UserData).Color, 3);
// highlight border
g.DrawPolygon(pen, ToPointsArray(glyphPoints));
string glyphTitle = null;
// prepare glyph's title
if ((visualizationType == VisualizationType.Name) && (glyphData.RecognizedGlyph != null))
{
glyphTitle = string.Format("{0}: {1}",
glyphIDs[i], glyphData.RecognizedGlyph.Name);
}
else
{
glyphTitle = string.Format("Tracking ID: {0}", glyphIDs[i]);
}
// show glyph's title
if (!string.IsNullOrEmpty(glyphTitle))
{
// get glyph's center point
IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(glyphPoints, out minXY, out maxXY);
IntPoint center = (minXY + maxXY) / 2;
// glyph's name size
SizeF nameSize = g.MeasureString(glyphTitle, defaultFont);
// paint the name
Brush brush = new SolidBrush(pen.Color);
g.DrawString(glyphTitle, defaultFont, brush,
new System.Drawing.Point(center.X - (int)nameSize.Width / 2, center.Y - (int)nameSize.Height / 2));
brush.Dispose( );
}
i++;
pen.Dispose( );
}
}
else if (visualizationType == VisualizationType.Image)
{
// lock image for further processing
BitmapData bitmapData = bitmap.LockBits(new Rectangle(0, 0, bitmap.Width, bitmap.Height),
ImageLockMode.ReadWrite, bitmap.PixelFormat);
UnmanagedImage unmanagedImage = new UnmanagedImage(bitmapData);
// highlight each found glyph
foreach (ExtractedGlyphData glyphData in glyphs)
{
if ((glyphData.RecognizedGlyph != null) && (glyphData.RecognizedGlyph.UserData != null))
{
GlyphVisualizationData visualization =
(GlyphVisualizationData)glyphData.RecognizedGlyph.UserData;
if (visualization.ImageName != null)
{
// get image associated with the glyph
Bitmap glyphImage = EmbeddedImageCollection.Instance.GetImage(visualization.ImageName);
if (glyphImage != null)
{
// put glyph's image onto the glyph using quadrilateral transformation
quadrilateralTransformation.SourceImage = glyphImage;
quadrilateralTransformation.DestinationQuadrilateral = glyphData.RecognizedQuadrilateral;
quadrilateralTransformation.ApplyInPlace(unmanagedImage);
}
}
}
}
bitmap.UnlockBits(bitmapData);
}
}
}
return(glyphs);
}
/// <summary>
/// Given the Window UI Screenshot, tries to detect all the buttons in it and returns all of them.
/// </summary>
/// <param name="res">Where to put candidate controls</param>
/// <param name="bitmap">Bitmap containing the pixels of the UI to analyze</param>
/// <param name="shouldInvert">If true, the bitmap will be inverted before analysis</param>
/// <param name="minWidth">Minimum width of patterns to recognize</param>
/// <param name="minHeight">Minimum height of patterns to recognize</param>
/// <returns>void</returns>
private void ScanWithVisualRecognition(ref CandidateSet res, Bitmap original, bool shouldInvert, int minWidth, int minHeight, Window w, IRankingPolicy policy)
{
// Setup the Blob counter
BlobCounter blobCounter = new BlobCounter();
blobCounter.BackgroundThreshold = Color.FromArgb(10, 10, 10);
blobCounter.FilterBlobs = true;
blobCounter.CoupledSizeFiltering = false;
blobCounter.MinHeight = minHeight;
blobCounter.MinWidth = minWidth;
List <Blob> blobs = new List <Blob>();
// Button scanning
// Apply the grayscale. This is needed for AForge's filters
using (Bitmap grey_scaled = Grayscale.CommonAlgorithms.BT709.Apply(original))
{
// Invert the image if requested
if (shouldInvert)
{
Invert invert = new Invert();
invert.ApplyInPlace(grey_scaled);
}
//grey_scaled.Save("C:\\users\\alberto geniola\\desktop\\dbg\\gray_scaled_" + toinvert + ".bmp");
using (Bitmap t1 = new Threshold(64).Apply(grey_scaled))
{
using (var tmp = new Bitmap(t1.Width, t1.Height))
{
using (Graphics g = Graphics.FromImage(tmp))
{
g.DrawImage(t1, 0, 0);
}
// The blob counter will analyze the bitmap looking for shapes
blobCounter.ProcessImage(tmp);
var tmparr = blobCounter.GetObjectsInformation();
blobs.AddRange(tmparr);
}
}
using (Bitmap t2 = new SISThreshold().Apply(grey_scaled))
{
using (var tmp = new Bitmap(t2.Width, t2.Height))
{
using (Graphics g = Graphics.FromImage(tmp))
{
g.DrawImage(t2, 0, 0);
}
//tmp.Save("C:\\users\\alberto geniola\\desktop\\dbg\\t2_" + toinvert + ".bmp");
// The blob counter will analyze the bitmap looking for shapes
blobCounter.ProcessImage(tmp);
var tmparr = blobCounter.GetObjectsInformation();
blobs.AddRange(tmparr);
}
}
}
// Let's analyze every single shape
for (int i = 0, n = blobs.Count; i < n; i++)
{
List <IntPoint> edgePoints = blobCounter.GetBlobsEdgePoints(blobs[i]);
if (edgePoints.Count > 1)
{
IntPoint p0, p1;
PointsCloud.GetBoundingRectangle(edgePoints, out p0, out p1);
var r = new Rectangle(p0.X, p0.Y, p1.X - p0.X, p1.Y - p0.Y);
// Skip any shape representing the border of the whole window ( +10px padding)
if (r.Width >= (original.Width - 10))
{
continue;
}
// This is most-likely a button!
// Crop the image and pass it to the OCR engine for text recognition
using (Bitmap button = new Bitmap(r.Width, r.Height))
{
// Scan the original shape
String txt = null;
using (var g1 = Graphics.FromImage(button))
{
g1.DrawImage(original, 0, 0, r, GraphicsUnit.Pixel);
}
// Process OCR on that image
txt = scanButton(button);
if (String.IsNullOrEmpty(txt))
{
using (Bitmap tmp = Grayscale.CommonAlgorithms.BT709.Apply(button))
{
if (shouldInvert)
{
new Invert().ApplyInPlace(tmp);
}
new SISThreshold().ApplyInPlace(tmp);
txt = scanButton(tmp);
}
}
// If still nothing is found, repeat the analysis with the second version of the filter
if (String.IsNullOrEmpty(txt))
{
using (Bitmap tmp = Grayscale.CommonAlgorithms.BT709.Apply(button))
{
if (shouldInvert)
{
new Invert().ApplyInPlace(tmp);
}
new Threshold(64).ApplyInPlace(tmp);
txt = scanButton(tmp);
}
}
// At this point we should have a result. Add it to list if it does not overlap any UIAutomated element
UIControlCandidate t = new UIControlCandidate();
t.PositionWindowRelative = r;
var winLoc = w.WindowLocation;
t.PositionScreenRelative = new Rectangle(r.X + winLoc.X, r.Y + winLoc.Y, r.Width, r.Height);
t.Text = txt;
t.Score = policy.RankElement(t);
// If the item falls into the same area of a UI element, ignore it.
bool overlaps = false;
foreach (var el in res)
{
if (el.AutoElementRef != null && el.PositionScreenRelative.IntersectsWith(t.PositionScreenRelative))
{
overlaps = true;
break;
}
}
if (!overlaps)
{
res.Add(t);
}
}
}
}
}
/* Odświezanie okna z obrazem */
void RefreshWindow(object sender, EventArgs arg)
{
//Pobieranie ramki
image = capture.QueryFrame();
image = image.Flip(FLIP.HORIZONTAL);
imageBox1.Image = image;
//YCbCr or Bgr(RGB)
//Warto zwrócić uwagę na to że Ycc to Y,Cr,Cb a nie Y,Cb,Cr, oraz Bgr to Blue,Green,Red
if (radioButton1.Checked)
{
imageGray = image.Resize((double)nupScale.Value, INTER.CV_INTER_CUBIC).Convert <Ycc, Byte>().
InRange(new Ycc((double)nudW1.Value, (double)nudW3.Value, (double)nudW2.Value), new Ycc((double)nudW4.Value, (double)nudW6.Value, (double)nudW5.Value));
}
else
{
imageGray = image.InRange(new Bgr((double)nudW3.Value, (double)nudW2.Value, (double)nudW1.Value), new Bgr((double)nudW6.Value, (double)nudW5.Value, (double)nudW4.Value));
}
if (medianCB.Checked)
{
imageGray = imageGray.SmoothMedian((int)nudMedian.Value);
}
//Image<Gray, Byte> sgm = new Image<Gray, Byte>(imageGray.Size);
Bitmap bmp = imageGray.ToBitmap();
bc.ProcessImage(bmp);
Blob[] blob = bc.GetObjectsInformation();
//one hand version
//int iters = bc.ObjectsCount > 1 ? 1 : bc.ObjectsCount;
int iters = bc.ObjectsCount > 2 ? 2 : bc.ObjectsCount;
if (iters > 1)
{
//both hands version
//lewa reka to ta z prawej strony obrazu (zwierciadlo), nie zakladamy ze user gestykuluje na krzyz, keep it simple
blob = blob.Take(2).OrderByDescending(a => a.CenterOfGravity.X).ToArray <Blob>();
}
int centerOfGravityLHandX = 0, centerOfGravityLHandY = 0, centerOfGravityRHandX = 0, centerOfGravityRHandY = 0;
string[] gestureLabel = new string[2];
int i = 0;
for (; i < iters; ++i)
{
IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(bc.GetBlobsEdgePoints(blob[i]), out minXY, out maxXY);
Bitmap clonimage = (Bitmap)bmp.Clone();
BitmapData data = bmp.LockBits(new Rectangle(0, 0, imageGray.Width, imageGray.Height), ImageLockMode.ReadWrite, bmp.PixelFormat);
Drawing.Rectangle(data, blob[i].Rectangle, Color.White);
bmp.UnlockBits(data);
int X = maxXY.X, Y = maxXY.Y;
int x = minXY.X, y = minXY.Y;
observed[0, i] = blob[i].Fullness;
/* malinowska kryjaka liczy obwod ze wzoru na prostokąt, nasza liczy piksele krawedziowe */
//Malinowska(i) = (2*bb(3)+2*bb(4))/(2*sqrt(pi*S)) - 1;
observed[2, i] = (double)(bc.GetBlobsEdgePoints(blob[i]).Count) / 2 / Math.Sqrt(Math.PI * blob[i].Area) - 1;
//MalinowskaZ(i) = 2*sqrt(pi*S)/(2*bb(3)+2*bb(4));
observed[3, i] = 2 * Math.Sqrt(Math.PI * blob[i].Area) / (double)(bc.GetBlobsEdgePoints(blob[i]).Count);
int gx = (int)blob[i].CenterOfGravity.X, gy = (int)blob[i].CenterOfGravity.Y;
//Sprawdzenie która ręka prawa, a która lewa
if (gx > centerOfGravityRHandX)
{
centerOfGravityLHandX = centerOfGravityRHandX;
centerOfGravityLHandY = centerOfGravityRHandY;
centerOfGravityRHandX = gx;
centerOfGravityRHandY = gy;
}
else
{
centerOfGravityLHandX = gx;
centerOfGravityLHandY = gy;
}
double blairsum = 0;
int ftx = 0, ftxMax = 0;
byte[, ,] dd = imageGray.Data;
for (int j = y; j < Y; ++j)
{
if (ftx > ftxMax)
{
ftxMax = ftx;
}
ftx = 0;//bo moze sie zdazyc ze zliczy wiecej linii naraz, patrz: idealny prostokat
for (int k = x; k < X; ++k)
{
if (dd[j, k, 0] != 0)
{
++ftx;
blairsum += (k - gx) * (k - gx) + (j - gy) * (j - gy);//distance squared
}
else
{
if (ftx > ftxMax)
{
ftxMax = ftx;
}
ftx = 0;
}
dd[j, k, 0] = 255;
}
}
/* aby policzyc ftyMax trzeba puscic jeszcze jedna petle tak aby wewnetrzna szla po y-kach
* ale mozna tez aproksymowac ftYmax przez boundingbox.Y, wtedy
* przewidywalem najwieksze rozbieznosci przy skosnych lub dziurawych obiektach;
* ale blad byl ponizej procenta, wiec szkoda tracic czas na kolejne O(n*n)
*
* int fty = 0, ftyMax = 0;
* for (int j = x; j < X; ++j) {
* if (fty > ftyMax) ftyMax = fty;
* fty = 0;
* for (int i = y; i < Y; ++i)
* if (dd[i, j, 0] != 0)
++fty;
* else {
* if (fty > ftyMax) ftyMax = fty;
* fty = 0;
* }
* }
* feret = (double)ftxMax / ftyMax; */
observed[4, i] = (double)ftxMax / (Y - y); //feret
observed[1, i] = (double)(blob[i].Area) / Math.Sqrt(2 * Math.PI * blairsum); //blair
gestChance[GEST.SLAYER] = dist(slayer, i);
gestChance[GEST.THUMBLEFT] = dist(thumbleft, i);
gestChance[GEST.THUMBUP] = dist(thumbup, i);
gestChance[GEST.SHAKA] = dist(shaka, i);
gestChance[GEST.FIST] = dist(fist, i);
gestChance[GEST.VICTORY] = dist(victory, i);
gestChance[GEST.VOPEN] = dist(vopen, i);
gestChance[GEST.HOPEN] = dist(hopen, i);
gestChance[GEST.FINGERS] = dist(fingers, i);
gestChance[GEST.SCISSORS] = dist(scissors, i);
//list fold - get key of minimal value
KeyValuePair <GEST, double> elem = gestChance.Aggregate((l, r) => l.Value < r.Value ? l : r);
found[i] = (elem.Value < TOLERANCE) ? elem.Key : GEST.BLANK;
if (elem.Key == GEST.FIST && (double)(X - x) / (Y - y) < .6)
{
found[i] = GEST.VOPEN;
}
gestureLabel[i] = labels[(int)found[i]];
}
g1value.Text = gestureLabel[1];
g2value.Text = gestureLabel[0];
compactnessLbl.Text = observed[0, 1].ToString(format);
blairLbl.Text = observed[1, 1].ToString(format);
malinowskaLbl.Text = observed[2, 1].ToString(format);
malzmodLbl.Text = observed[3, 1].ToString(format);
feretLbl.Text = observed[4, 1].ToString(format);
comp2.Text = observed[0, 0].ToString(format);
blair2.Text = observed[1, 0].ToString(format);
mal2.Text = observed[2, 0].ToString(format);
malz2.Text = observed[3, 0].ToString(format);
feret2.Text = observed[4, 0].ToString(format);
/* for blobs not detected */
for (; i < 2; ++i)
{
observed[0, i] = observed[1, i] = observed[2, i] = observed[3, i] = observed[4, i] = NOT_FOUND;
}
imageGray = new Image <Gray, Byte>(bmp);
imageGray = imageGray.Erode((int)nudErode.Value);
imageGray = imageGray.Dilate((int)nudDilate.Value);
imageBox2.Image = imageGray;
//Zmiana pozycji myszki od środka ciężkości lewej ręki
if (centerOfGravityLHandX * centerOfGravityLHandY != 0 && !blockMouseControl)
{
double smoothness = (double)nudSmoothness.Value;
double sensitivity = (double)nudSensitivity.Value;
int newPositionX = screenWidth - (int)(centerOfGravityLHandX / (imageGray.Width * .2) * sensitivity * screenWidth); //- imageGray.Width*1/5
int newPositionY = (int)((centerOfGravityLHandY - imageGray.Height * .5) / (imageGray.Height * .25) * sensitivity * screenHeight);
int diffX = Cursor.Position.X + newPositionX;
int diffY = Cursor.Position.Y - newPositionY;
newPositionX = Cursor.Position.X - (int)(diffX / smoothness);
newPositionY = Cursor.Position.Y - (int)(diffY / smoothness);
MouseSimulating.SetMousePosition(newPositionX, newPositionY);
//Wyliczanie akcji do podjęcia
if (found[1] == GEST.BLANK || prevGestureLeft != found[1])
{
frameCounterLeft = 0;
prevGestureLeft = found[1];
}
if (found[0] == GEST.BLANK || prevGestureRight != found[0])
{
frameCounterRight = 0;
prevGestureRight = found[0];
}
if (frameCounterLeft == 30) //ile klatek musi - 30 kl/s
{
if (prevGestureLeft == GEST.FIST)
{
MouseSimulating.PressLPM();
}
else if (prevGestureLeft == GEST.VOPEN)
{
MouseSimulating.ReleaseLPM();
}
frameCounterLeft = 0;
}
else
{
frameCounterLeft++;
}
if (frameCounterRight == 30)
{
if (prevGestureRight == GEST.FIST)
{
MouseSimulating.ClickLPM();
}
else if (prevGestureRight == GEST.SLAYER)
{
MouseSimulating.ScrollUP(200);
}
else if (prevGestureRight == GEST.VICTORY)
{
MouseSimulating.ScrollDOWN(200);
}
else if (prevGestureRight == GEST.FINGERS)
{
MouseSimulating.ClickPPM();
}
else if (prevGestureRight == GEST.THUMBUP)
{
KeyboardSimulating.SendCtrlC();
}
else if (prevGestureRight == GEST.THUMBLEFT)
{
KeyboardSimulating.SendCtrlV();
}
else if (prevGestureRight == GEST.SCISSORS)
{
KeyboardSimulating.SendCtrlX();
}
else if (prevGestureRight == GEST.HOPEN)
{
MouseSimulating.ClickLPM(); MouseSimulating.ClickLPM();
}
else if (prevGestureRight == GEST.SHAKA)
{
MouseSimulating.ClickMouseButton4();
}
frameCounterRight = 0;
}
else
{
frameCounterRight++;
}
}
}