/// <summary> /// Gets the color of the approximate scalar from. /// </summary> /// <param name="handColor">Hand color.</param> /// <param name="mLowerBound">M lower bound.</param> /// <param name="mUpperBound">M upper bound.</param> private static void _getApproximateScalarFromColor(Color handColor, Scalar mLowerBound, Scalar mUpperBound) { //色の範囲を指定する Scalar mColorRadius = new Scalar(25, 50, 50, 0); Scalar hsvColor = HGColorSpuiter.ColorToScalar(handColor); double minH = (hsvColor.val [0] >= mColorRadius.val [0]) ? hsvColor.val [0] - mColorRadius.val [0] : 0; double maxH = (hsvColor.val [0] + mColorRadius.val [0] <= 255) ? hsvColor.val [0] + mColorRadius.val [0] : 255; mLowerBound.val [0] = minH; mUpperBound.val [0] = maxH; mLowerBound.val [1] = hsvColor.val [1] - mColorRadius.val [1]; mUpperBound.val [1] = hsvColor.val [1] + mColorRadius.val [1]; mLowerBound.val [2] = hsvColor.val [2] - mColorRadius.val [2]; mUpperBound.val [2] = hsvColor.val [2] + mColorRadius.val [2]; mLowerBound.val [3] = 0; mUpperBound.val [3] = 255; }
//手を検出して画像に描画する private static void _handPoseEstimationProcess(Mat rgbaMat, Color handColor) { Imgproc.GaussianBlur(rgbaMat, rgbaMat, new OpenCVForUnity.Size(3, 3), 1, 1); //検出器に色を設定 detector.setHsvColor(HGColorSpuiter.ColorToScalar(handColor)); List <MatOfPoint> contours = detector.getContours(); detector.process(rgbaMat); if (contours.Count <= 0) { return; } //手の角度に傾いた外接矩形を作成 RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[0].toArray())); double boundWidth = rect.size.width; double boundHeight = rect.size.height; int boundPos = 0; for (int i = 1; i < contours.Count; i++) { rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[i].toArray())); if (rect.size.width * rect.size.height > boundWidth * boundHeight) { boundWidth = rect.size.width; boundHeight = rect.size.height; boundPos = i; } } OpenCVForUnity.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contours[boundPos].toArray())); //手首までの範囲を描画 Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), HGColorSpuiter.ColorToScalar(WristRangeColor), 2, 8, 0); double a = boundRect.br().y - boundRect.tl().y; a = a * 0.7; a = boundRect.tl().y + a; //手のひらの範囲を描画 Imgproc.rectangle(rgbaMat, boundRect.tl(), new Point(boundRect.br().x, a), HGColorSpuiter.ColorToScalar(PalmsRangeColor), 2, 8, 0); //折れ線カーブまたはポリゴンを,互いの距離が指定された精度以下になるように,より少ない頂点数のカーブやポリゴンで近似します MatOfPoint2f pointMat = new MatOfPoint2f(); Imgproc.approxPolyDP(new MatOfPoint2f(contours[boundPos].toArray()), pointMat, 3, true); contours[boundPos] = new MatOfPoint(pointMat.toArray()); //点とポリゴンの最短距離を計算 MatOfInt hull = new MatOfInt(); MatOfInt4 convexDefect = new MatOfInt4(); Imgproc.convexHull(new MatOfPoint(contours[boundPos].toArray()), hull); if (hull.toArray().Length < 3) { return; } Imgproc.convexityDefects(new MatOfPoint(contours[boundPos].toArray()), hull, convexDefect); //手の範囲を取得 List <MatOfPoint> hullPoints = new List <MatOfPoint>(); List <Point> listPo = new List <Point>(); for (int j = 0; j < hull.toList().Count; j++) { listPo.Add(contours[boundPos].toList()[hull.toList()[j]]); } MatOfPoint e = new MatOfPoint(); e.fromList(listPo); hullPoints.Add(e); //手の範囲を描画 Imgproc.drawContours(rgbaMat, hullPoints, -1, HGColorSpuiter.ColorToScalar(HandRangeColor), 3); //指と認識した場所を取得 List <MatOfPoint> defectPoints = new List <MatOfPoint>(); List <Point> listPoDefect = new List <Point>(); for (int j = 0; j < convexDefect.toList().Count; j = j + 4) { Point farPoint = contours[boundPos].toList()[convexDefect.toList()[j + 2]]; int depth = convexDefect.toList()[j + 3]; if (depth > depthThreashold && farPoint.y < a) { listPoDefect.Add(contours[boundPos].toList()[convexDefect.toList()[j + 2]]); } } MatOfPoint e2 = new MatOfPoint(); e2.fromList(listPo); defectPoints.Add(e2); //検出した指の本数を更新 numberOfFingers = listPoDefect.Count; if (numberOfFingers > 5) { numberOfFingers = 5; } //指の間に点を描画 foreach (Point p in listPoDefect) { Imgproc.circle(rgbaMat, p, 6, HGColorSpuiter.ColorToScalar(BetweenFingersColor), -1); } }
/*=============================================* * 輪郭ごとの頂点から手を判別するまで *=============================================*/ /// <summary> /// Contours to hand gesture. /// </summary> /// <param name="rgbaMat">Rgba mat.</param> /// <param name="contour">Contour.</param> private static void _contourToHandGesture(Mat rgbaMat, MatOfPoint contour) { try { //頂点を調査する準備をする _pointOfVertices(rgbaMat, contour); //基準輪郭のサイズの取得と描画(長方形) OpenCVForUnity.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray())); Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), HGColorSpuiter.ColorToScalar(ContourRangeColor), 2, 8, 0); /*=============================================* * 腕まで含んだ手の大きさを取得する **=============================================*/ //腕まで含んだ手の大きさを識別する MatOfInt hull = new MatOfInt(); Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull); //腕まで含んだ手の範囲を取得 List <Point> armPointList = new List <Point>(); for (int j = 0; j < hull.toList().Count; j++) { Point armPoint = contour.toList()[hull.toList()[j]]; bool addFlag = true; foreach (Point point in armPointList.ToArray()) { //輪郭の1/10より近い頂点は誤差としてまとめる double distance = Mathf.Sqrt((float)((armPoint.x - point.x) * (armPoint.x - point.x) + (armPoint.y - point.y) * (armPoint.y - point.y))); if (distance <= Mathf.Min((float)boundRect.width, (float)boundRect.height) / 10) { addFlag = false; break; } } if (addFlag) { armPointList.Add(armPoint); } } MatOfPoint armMatOfPoint = new MatOfPoint(); armMatOfPoint.fromList(armPointList); List <MatOfPoint> armPoints = new List <MatOfPoint>(); armPoints.Add(armMatOfPoint); //腕まで含んだ手の範囲を描画 Imgproc.drawContours(rgbaMat, armPoints, -1, HGColorSpuiter.ColorToScalar(ArmRangeColor), 3); //腕まで含んだ手が三角形の場合はそれ以上の識別が難しい if (hull.toArray().Length < 3) { return; } /*=============================================* * 掌の大きさを取得する **=============================================*/ //凸面の頂点から凹面の点のみを取得し、掌の範囲を取得する MatOfInt4 convexDefect = new MatOfInt4(); Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect); //凹面の点をフィルタリングして取得 List <Point> palmPointList = new List <Point>(); for (int j = 0; j < convexDefect.toList().Count; j = j + 4) { Point farPoint = contour.toList()[convexDefect.toList()[j + 2]]; int depth = convexDefect.toList()[j + 3]; if (depth > depthThreashold && farPoint.y < boundRect.br().y - boundRect.tl().y) { palmPointList.Add(contour.toList()[convexDefect.toList()[j + 2]]); } } MatOfPoint palmMatOfPoint = new MatOfPoint(); palmMatOfPoint.fromList(palmPointList); List <MatOfPoint> palmPoints = new List <MatOfPoint>(); palmPoints.Add(palmMatOfPoint); //掌の範囲を描画 Imgproc.drawContours(rgbaMat, palmPoints, -1, HGColorSpuiter.ColorToScalar(PalmRangeColor), 3); /*=============================================* * 掌+指先の大きさを取得する **=============================================*/ //掌の位置を元に手首を除いた範囲を取得する List <Point> handPointList = new List <Point>(); handPointList.AddRange(armPointList.ToArray()); handPointList.Reverse(); handPointList.RemoveAt(0); handPointList.Insert(0, palmPointList.ToArray()[0]); handPointList.RemoveAt(handPointList.Count - 1); handPointList.Insert(handPointList.Count, palmPointList.ToArray()[palmPointList.Count - 1]); MatOfPoint handMatOfPoint = new MatOfPoint(); handMatOfPoint.fromList(handPointList); List <MatOfPoint> handPoints = new List <MatOfPoint>(); handPoints.Add(handMatOfPoint); Imgproc.drawContours(rgbaMat, handPoints, -1, HGColorSpuiter.ColorToScalar(HandRangeColor), 3); /*=============================================* * 指先の位置を取得する **=============================================*/ //掌の各頂点の中心を求める List <Point> palmCenterPoints = new List <Point>(); for (int i = 0; i < palmPointList.Count; i++) { Point palmPoint = palmPointList.ToArray()[i]; Point palmPointNext = new Point(); if (i + 1 < palmPointList.Count) { palmPointNext = palmPointList.ToArray()[i + 1]; } else { palmPointNext = palmPointList.ToArray()[0]; } Point palmCenterPoint = new Point((palmPoint.x + palmPointNext.x) / 2, (palmPoint.y + palmPointNext.y) / 2); palmCenterPoints.Add(palmCenterPoint); } //掌の頂点から最も近い手の頂点を求める for (int i = 0; i < palmCenterPoints.Count && i + 1 < handPointList.Count && i < 5; i++) { Point palmPoint = palmCenterPoints.ToArray()[i]; List <Point> fingerList = new List <Point>(); fingerList.Add(palmPoint); fingerList.Add(handPointList.ToArray()[i + 1]); MatOfPoint fingerPoint = new MatOfPoint(); fingerPoint.fromList(fingerList); List <MatOfPoint> fingerPoints = new List <MatOfPoint>(); fingerPoints.Add(fingerPoint); Imgproc.drawContours(rgbaMat, fingerPoints, -1, HGColorSpuiter.ColorToScalar(FingerRangeColor), 3); } // Imgproc.putText(rgbaMat, "", new Point(2, rgbaMat.rows()-30), Core.FONT_HERSHEY_SIMPLEX, 1.0, HGColorSpuiter.ColorToScalar(Color.black), 2, Imgproc.LINE_AA, false); } catch (System.Exception e) { Debug.Log(e.Message); } }