public IEnumerable <HullDefectVertice> getDefects()
{
int[] hullIndices = getHullIndices();
var defects = Cv2.ConvexityDefects(contour, hullIndices);
var hullPointDefectNeighbors = hullIndices.ToDictionary(indices => indices, i => new List <int>());
foreach (var defect in defects)
{
int startPoint = defect.Item0;
int endPoint = defect.Item1;
int defectPoint = defect.Item2;
hullPointDefectNeighbors[startPoint].Add(defectPoint);
hullPointDefectNeighbors[endPoint].Add(defectPoint);
}
return(hullIndices
.Where((indice) => hullPointDefectNeighbors[indice].Count() > 1)
.Select((indice) => {
var defectNeighbor = hullPointDefectNeighbors[indice];
return new HullDefectVertice {
pt = contour[indice],
d1 = contour[defectNeighbor[0]],
d2 = contour[defectNeighbor[1]]
};
}
));
}
private static void DrawContours(IReadOnlyList <Point[]> contours, IList <Point[]> contoursPoly, Mat drawing)
{
var center = new Point2f[contours.Count];
var radius = new float[contours.Count];
for (var i = 0; i < contours.Count; i++)
{
if (Cv2.ContourArea(contours[i]) >= 5000)
{
contoursPoly[i] = Cv2.ApproxPolyDP(contours[i], 3, true);
Cv2.MinEnclosingCircle(contoursPoly[i], out center[i], out radius[i]);
var tempContour = contours[i];
var hulls = new Point[1][];
var hullsI = new int[1][];
hulls[0] = Cv2.ConvexHull(tempContour);
hullsI[0] = Cv2.ConvexHullIndices(tempContour);
Cv2.DrawContours(drawing, hulls, -1, Scalar.Gold, 2);
if (hullsI[0].Length > 0)
{
var defects = Cv2.ConvexityDefects(tempContour, hullsI[0]);
if (defects.Length > 0)
{
for (var j = 1; j < defects.Length; j++)
{
var startIdx = defects[j][0];
var ptStart = tempContour[startIdx];
var farIdx = defects[j][2];
var ptFar = tempContour[farIdx];
if (GetDistance(ptStart, ptFar) > 1000 && ptStart.Y < center[i].Y && radius[i] >= 70)
{
Cv2.Circle(drawing, ptStart, 10, Scalar.Yellow, 2);
Cv2.Line(drawing, ptStart, ptFar, Scalar.Pink, 2);
_numberOfFingers++;
}
}
if (radius[i] > 50)
{
Cv2.DrawContours(drawing, contoursPoly, i, Scalar.Red);
Cv2.Circle(drawing, center[i], (int)radius[i], Scalar.White, 2);
Cv2.Circle(drawing, center[i], 5, Scalar.Red, 2);
if (Program.ControlMode)
{
_posX = (int)(4 * (center[i].X - 100));
_posY = (int)(4 * (center[i].Y - 100));
WinApiUtils.SetCursorPos(_posX, _posY);
}
}
}
}
}
}
}
public static void Run()
{
Mat image = new Mat();
frameSource = Cv2.CreateFrameSource_Camera(0);
while (true)
{
//Grab the current frame
frameSource.NextFrame(image);
Mat gray = new Mat();
Cv2.CvtColor(image, gray, ColorConversionCodes.BGR2GRAY);
Mat blur = new Mat();
Cv2.GaussianBlur(gray, blur, new Size(19, 19), 0);
Mat threshImg = new Mat();
double thresh = Cv2.Threshold(gray, threshImg, 150, 255, ThresholdTypes.Otsu);
Mat[] contours;
Mat hierarchy = new Mat();
Cv2.FindContours(threshImg, out contours, hierarchy, RetrievalModes.Tree, ContourApproximationModes.ApproxSimple);
double maxArea = 100;
Mat handContour = new Mat();
handContour = contours.OrderByDescending(x => (x.ContourArea())).ToList()[0];
//foreach (var item in contours)
//{
// if(item.ContourArea() > maxArea)
// {
// handContour = item;
// break;
// }
//}
Mat hull = new Mat();
Cv2.ConvexHull(handContour, hull);
Mat defects = new Mat();
Cv2.ConvexityDefects(handContour, hull, defects);
Cv2.ImShow("frame", hull);
//Cv2.WaitKey(0);
if (Cv2.WaitKey(1) == (int)ConsoleKey.Enter)
{
break;
}
}
}
public void ConvexityDefects()
{
var contour = new[]
{
// ‰š
new Point(0, 0),
new Point(0, 10),
new Point(3, 10),
new Point(3, 5),
new Point(6, 5),
new Point(6, 10),
new Point(10, 10),
new Point(10, 0),
};
var convexHull = Cv2.ConvexHullIndices(contour);
Assert.Equal(4, convexHull.Length);
// Note: ConvexityDefects does not support Point2f contour
var convexityDefects = Cv2.ConvexityDefects(contour, convexHull);
Assert.Single(convexityDefects);
Assert.Equal(new Vec4i(1, 6, 3, 1280), convexityDefects[0]);
}
// uzimamo vrhove koji nisu pravi vrhovi (imaju smetnje i sl.)
private IEnumerable <DefectVertex> CreateHullDefectVertices(Point[] contour, IEnumerable <int> indices)
{
if (contour == null || !indices.Any())
{
return(Enumerable.Empty <DefectVertex>());
}
indices = indices.OrderByDescending(x => x);
var defects = Cv2.ConvexityDefects(contour, indices);
var neighbours = new Dictionary <int, List <int> >();
foreach (var induce in indices)
{
neighbours[induce] = new List <int>();
}
foreach (var defect in defects)
{
var startPoint = defect.Item0;
var endPoint = defect.Item1;
var defectPoint = defect.Item2;
neighbours[startPoint].Add(defectPoint);
neighbours[endPoint].Add(defectPoint);
}
return(neighbours.Keys
.Where(x => neighbours[x].Count > 1)
.Select(hull =>
{
var defectNeighbourId = neighbours[hull];
return new DefectVertex(contour[hull], contour[defectNeighbourId[0]],
contour[defectNeighbourId[1]]);
}));
}
private static Vec4i[] GetDefects(OpenCvSharp.Point[] contour)
{
var hull = Cv2.ConvexHull(contour);
return(Cv2.ConvexityDefects(contour, PointsToIndex(contour, hull)));
}
public Mat getHandContour(Mat input, Mat frame)
{
Mat detectImage = frame.Clone();
if (input.Empty())
{
return(detectImage);
}
// we work only on the 1 channel result, since this function is called inside a loop we are not sure that this is always the case
if (input.Channels() != 1)
{
return(detectImage);
}
Point[][] contours;
HierarchyIndex[] hierarchy;
Cv2.FindContours(input,
out contours,
out hierarchy,
RetrievalModes.External,
ContourApproximationModes.ApproxSimple);
if (contours.Length <= 0)
{
return(detectImage);
}
int biggest_contour_index = -1;
double biggest_area = 0.0;
for (int i = 0; i < contours.GetLength(0); i++)
{
double area = Cv2.ContourArea(contours[i], false);
if (area > biggest_area)
{
biggest_area = area;
biggest_contour_index = i;
}
}
if (biggest_contour_index < 0)
{
return(detectImage);
}
// Debug.Log(" " + biggest_area + " " + biggest_contour_index);
Point[] hull_points;
int[] hull_ints;
hull_points = Cv2.ConvexHull(contours[biggest_contour_index]);
hull_ints = Cv2.ConvexHullIndices(contours[biggest_contour_index]);
//Debug.Log("hull " + hull_points.Length);
//Debug.Log("hull " + hull_ints.Length);
Vec4i[] defects;
if (hull_ints.Length > 3)
{
defects = Cv2.ConvexityDefects(contours[biggest_contour_index], hull_ints);
}
else
{
return(detectImage);
}
///////////////
OpenCvSharp.Rect bounding_rectangle = Cv2.BoundingRect(hull_points);
// we find the center of the bounding rectangle, this should approximately also be the center of the hand
Point center_bounding_rect = new Point(
(bounding_rectangle.TopLeft.X + bounding_rectangle.BottomRight.X) / 2,
(bounding_rectangle.TopLeft.Y + bounding_rectangle.BottomRight.Y) / 2
);
// we separate the defects keeping only the ones of intrest
List <Point> start_points = new List <Point>();
List <Point> far_points = new List <Point>();
// Debug.Log(" " + defects.Length);
//Debug.Log(" " + contours[biggest_contour_index].Length);
//Debug.Log(" " + contours.Length);
for (int i = 0; i < defects.Length; i++)
{
start_points.Add(contours[biggest_contour_index][defects[i].Item0]);
// filtering the far point based on the distance from the center of the bounding rectangle
if (findPointsDistance(contours[biggest_contour_index][defects[i].Item2], center_bounding_rect) < bounding_rectangle.Height * BOUNDING_RECT_FINGER_SIZE_SCALING)
{
far_points.Add(contours[biggest_contour_index][defects[i].Item2]);
}
}
// we compact them on their medians
List <Point> filtered_start_points = compactOnNeighborhoodMedian(start_points, bounding_rectangle.Height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING);
List <Point> filtered_far_points = compactOnNeighborhoodMedian(far_points, bounding_rectangle.Height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING);
// now we try to find the fingers
List <Point> filtered_finger_points = new List <Point>();
if (filtered_far_points.Count > 1)
{
List <Point> finger_points = new List <Point>();
for (int i = 0; i < filtered_start_points.Count; i++)
{
List <Point> closest_points = findClosestOnX(filtered_far_points, filtered_start_points[i]);
if (isFinger(closest_points[0], filtered_start_points[i], closest_points[1], LIMIT_ANGLE_INF, LIMIT_ANGLE_SUP, center_bounding_rect, bounding_rectangle.Height * BOUNDING_RECT_FINGER_SIZE_SCALING))
{
finger_points.Add(filtered_start_points[i]);
}
}
if (finger_points.Count > 0)
{
// we have at most five fingers usually :)
while (finger_points.Count > 5)
{
finger_points.RemoveAt(0);
}
// filter out the points too close to each other
for (int i = 0; i < finger_points.Count - 1; i++)
{
if (findPointsDistanceOnX(finger_points[i], finger_points[i + 1]) > bounding_rectangle.Height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING * 1.5)
{
filtered_finger_points.Add(finger_points[i]);
}
}
if (finger_points.Count > 2)
{
if (findPointsDistanceOnX(finger_points[0], finger_points[finger_points.Count - 1]) > bounding_rectangle.Height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING * 1.5)
{
filtered_finger_points.Add(finger_points[finger_points.Count - 1]);
}
}
else
{
filtered_finger_points.Add(finger_points[finger_points.Count - 1]);
}
}
}
// we draw what found on the returned image
Cv2.DrawContours(detectImage, contours, biggest_contour_index, new Scalar(0, 255, 0), 8, LineTypes.Link8, hierarchy);
//Cv2.Polylines(input, new IEnumerable<IEnumerable<Point>>( hull_points), true, new Scalar(1, 0, 0));
Cv2.Rectangle(detectImage, bounding_rectangle.TopLeft, bounding_rectangle.BottomRight, new Scalar(0, 0, 255), 8, LineTypes.Link8);
Cv2.Circle(detectImage, center_bounding_rect.X, center_bounding_rect.Y, 15, new Scalar(255, 0, 255), 3, LineTypes.Link8);
drawVectorPoints(detectImage, filtered_start_points, new Scalar(255, 0, 0), true);
drawVectorPoints(detectImage, filtered_far_points, new Scalar(0, 0, 25), true);
drawVectorPoints(detectImage, filtered_finger_points, new Scalar(0, 255, 255), false);
Cv2.PutText(detectImage, filtered_finger_points.Count.ToString(), center_bounding_rect, HersheyFonts.HersheyTriplex, 10, new Scalar(255, 0, 25));
// and on the starting frame
Cv2.DrawContours(frame, contours, biggest_contour_index, new Scalar(0, 255, 0), 8, LineTypes.Link8, hierarchy);
Cv2.Circle(frame, center_bounding_rect.X, center_bounding_rect.Y, 15, new Scalar(255, 0, 255), 8, LineTypes.Link8);
drawVectorPoints(frame, filtered_finger_points, new Scalar(0, 255, 255), false);
Cv2.PutText(frame, filtered_finger_points.Count.ToString(), center_bounding_rect, HersheyFonts.HersheyTriplex, 10, new Scalar(255, 0, 255));
fingerCount = filtered_finger_points.Count;
if (fingerCount <= 1)
{
finalPose = "rock";
}
if (fingerCount == 2 && fingerCount <= 3)
{
finalPose = "scissor";
}
if (fingerCount >= 4)
{
finalPose = "paper";
}
return(detectImage);
}