private void UpdateConvexityDefection(
MatOfPoint contour, MatOfInt hullIndices, double defectMinY, RecordHandDetectResult resultSetter
)
{
var contourArray = contour.toArray();
var convexDefect = new MatOfInt4();
Imgproc.convexityDefects(contour, hullIndices, convexDefect);
resultSetter.ConvexDefectVectors.Clear();
int convexDefectCount = convexDefect.rows();
if (convexDefectCount > 0)
{
for (int i = 0; i < convexDefectCount; i++)
{
convexDefect.get(i, 0, _convexityDefectSetValues);
Point farPoint = contourArray[_convexityDefectSetValues[2]];
int depth = _convexityDefectSetValues[3];
if (depth > DefectThreasholdValue && farPoint.y < defectMinY)
{
var nearPoint1 = contourArray[_convexityDefectSetValues[0]];
var nearPoint2 = contourArray[_convexityDefectSetValues[1]];
resultSetter.ConvexDefectVectors.Add(new Vector2(
(float)(nearPoint1.x * 0.5f + nearPoint2.x * 0.5 - farPoint.x),
(float)(nearPoint1.y * 0.5f + nearPoint2.y * 0.5 - farPoint.y)
));
}
}
}
//ここまでやりきると全データが有効に更新されている。
resultSetter.HasValidHandArea = true;
}
private void HandPoseEstimationProcess(Mat rgbaMat)
{
//Imgproc.blur(mRgba, mRgba, new Size(5,5));
Imgproc.GaussianBlur(rgbaMat, rgbaMat, new OpenCVForUnity.Size(3, 3), 1, 1);
//Imgproc.medianBlur(mRgba, mRgba, 3);
if (!isColorSelected)
{
return;
}
List <MatOfPoint> contours = detector.GetContours();
detector.Process(rgbaMat);
// Debug.Log ("Contours count: " + contours.Count);
if (contours.Count <= 0)
{
return;
}
RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours [0].toArray()));
double boundWidth = rect.size.width;
double boundHeight = rect.size.height;
int boundPos = 0;
for (int i = 1; i < contours.Count; i++)
{
rect = Imgproc.minAreaRect(new MatOfPoint2f(contours [i].toArray()));
if (rect.size.width * rect.size.height > boundWidth * boundHeight)
{
boundWidth = rect.size.width;
boundHeight = rect.size.height;
boundPos = i;
}
}
MatOfPoint contour = contours [boundPos];
OpenCVForUnity.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray()));
Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), CONTOUR_COLOR_WHITE, 2, 8, 0);
// Debug.Log (
// " Row start [" +
// (int)boundRect.tl ().y + "] row end [" +
// (int)boundRect.br ().y + "] Col start [" +
// (int)boundRect.tl ().x + "] Col end [" +
// (int)boundRect.br ().x + "]");
double a = boundRect.br().y - boundRect.tl().y;
a = a * 0.7;
a = boundRect.tl().y + a;
// Debug.Log (
// " A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]");
Imgproc.rectangle(rgbaMat, boundRect.tl(), new Point(boundRect.br().x, a), CONTOUR_COLOR, 2, 8, 0);
MatOfPoint2f pointMat = new MatOfPoint2f();
Imgproc.approxPolyDP(new MatOfPoint2f(contour.toArray()), pointMat, 3, true);
contour = new MatOfPoint(pointMat.toArray());
MatOfInt hull = new MatOfInt();
MatOfInt4 convexDefect = new MatOfInt4();
Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull);
if (hull.toArray().Length < 3)
{
return;
}
Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect);
List <MatOfPoint> hullPoints = new List <MatOfPoint> ();
List <Point> listPo = new List <Point> ();
for (int j = 0; j < hull.toList().Count; j++)
{
listPo.Add(contour.toList() [hull.toList() [j]]);
}
MatOfPoint e = new MatOfPoint();
e.fromList(listPo);
hullPoints.Add(e);
List <Point> listPoDefect = new List <Point> ();
if (convexDefect.rows() > 0)
{
List <int> convexDefectList = convexDefect.toList();
List <Point> contourList = contour.toList();
for (int j = 0; j < convexDefectList.Count; j = j + 4)
{
Point farPoint = contourList [convexDefectList [j + 2]];
int depth = convexDefectList [j + 3];
if (depth > threasholdSlider.value && farPoint.y < a)
{
listPoDefect.Add(contourList [convexDefectList [j + 2]]);
}
// Debug.Log ("convexDefectList [" + j + "] " + convexDefectList [j + 3]);
}
}
// Debug.Log ("hull: " + hull.toList ());
// if (convexDefect.rows () > 0) {
// Debug.Log ("defects: " + convexDefect.toList ());
// }
Imgproc.drawContours(rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3);
// int defectsTotal = (int)convexDefect.total();
// Debug.Log ("Defect total " + defectsTotal);
this.numberOfFingers = listPoDefect.Count;
if (this.numberOfFingers > 5)
{
this.numberOfFingers = 5;
}
// Debug.Log ("numberOfFingers " + numberOfFingers);
// Core.putText (mRgba, "" + numberOfFingers, new Point (mRgba.cols () / 2, mRgba.rows () / 2), Core.FONT_HERSHEY_PLAIN, 4.0, new Scalar (255, 255, 255, 255), 6, Core.LINE_AA, false);
numberOfFingersText.text = numberOfFingers.ToString();
foreach (Point p in listPoDefect)
{
Imgproc.circle(rgbaMat, p, 6, new Scalar(255, 0, 255, 255), -1);
}
}
private void HandPoseEstimationProcess(Mat rgbaMat)
{
//Imgproc.blur(mRgba, mRgba, new Size(5,5));
Imgproc.GaussianBlur(rgbaMat, rgbaMat, new Size(3, 3), 1, 1);
//Imgproc.medianBlur(mRgba, mRgba, 3);
if (!isColorSelected)
{
return;
}
List <MatOfPoint> contours = detector.GetContours();
detector.Process(rgbaMat);
//Debug.Log ("Contours count: " + contours.Count);
if (contours.Count <= 0)
{
return;
}
RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[0].toArray()));
double boundWidth = rect.size.width;
double boundHeight = rect.size.height;
int boundPos = 0;
for (int i = 1; i < contours.Count; i++)
{
rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[i].toArray()));
if (rect.size.width * rect.size.height > boundWidth * boundHeight)
{
boundWidth = rect.size.width;
boundHeight = rect.size.height;
boundPos = i;
}
}
MatOfPoint contour = contours[boundPos];
OpenCVForUnity.CoreModule.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray()));
Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), CONTOUR_COLOR_WHITE, 2, 8, 0);
// Debug.Log (
// " Row start [" +
//(int)boundRect.tl ().y + "] row end [" +
// (int)boundRect.br ().y + "] Col start [" +
// (int)boundRect.tl ().x + "] Col end [" +
// (int)boundRect.br ().x + "]");
Point bottomLeft = new Point(boundRect.x, boundRect.y + boundRect.height);
Point topLeft = new Point(boundRect.x, boundRect.y);
Point bottomRight = new Point(boundRect.x + boundRect.width, boundRect.y + boundRect.height);
Point topRight = new Point(boundRect.x + boundRect.width, boundRect.y);
rectPoints = new MatOfPoint2f(new Point(boundRect.x, boundRect.y), //topleft
new Point(boundRect.x + boundRect.width, boundRect.y), //Top Right
new Point(boundRect.x + boundRect.width, boundRect.y + boundRect.height), //Bottom Right
new Point(boundRect.x, boundRect.y + boundRect.height) //Bottom Left
);
//double a = boundRect.br ().y - boundRect.tl ().y;
//a = a * 0.7;
//a = boundRect.tl ().y + a;
//Debug.Log (" A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]");
//Imgproc.rectangle (rgbaMat, boundRect.tl (), new Point (boundRect.br ().x, a), CONTOUR_COLOR, 2, 8, 0);
List <Point3> m_markerCorners3dList = new List <Point3>();
m_markerCorners3dList.Add(new Point3(-0.5f, -0.5f, 0)); //Top, Left (A)
m_markerCorners3dList.Add(new Point3(+0.5f, -0.5f, 0)); //Top, Right (B)
m_markerCorners3dList.Add(new Point3(+0.5f, +0.5f, 0)); //Bottom, Right (C)
m_markerCorners3dList.Add(new Point3(-0.5f, +0.5f, 0)); //Bottom, Left (D)
m_markerCorners3d.fromList(m_markerCorners3dList);
//estimate pose
Mat Rvec = new Mat();
Mat Tvec = new Mat();
Mat raux = new Mat();
Mat taux = new Mat();
Calib3d.solvePnP(m_markerCorners3d, rectPoints, camMatrix, distCoeff, raux, taux);
raux.convertTo(Rvec, CvType.CV_32F);
taux.convertTo(Tvec, CvType.CV_32F);
rotMat = new Mat(3, 3, CvType.CV_64FC1);
Calib3d.Rodrigues(Rvec, rotMat);
transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0)[0], (float)rotMat.get(0, 1)[0], (float)rotMat.get(0, 2)[0], (float)Tvec.get(0, 0)[0]));
transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0)[0], (float)rotMat.get(1, 1)[0], (float)rotMat.get(1, 2)[0], (float)Tvec.get(1, 0)[0]));
transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0)[0], (float)rotMat.get(2, 1)[0], (float)rotMat.get(2, 2)[0], (float)Tvec.get(2, 0)[0]));
transformationM.SetRow(3, new Vector4(0, 0, 0, 1));
//Debug.Log ("transformationM " + transformationM.ToString ());
Rvec.Dispose();
Tvec.Dispose();
raux.Dispose();
taux.Dispose();
rotMat.Dispose();
ARM = ARCamera.transform.localToWorldMatrix * invertYM * transformationM * invertZM;
//Debug.Log("arM " + ARM.ToString());
if (ARGameObject != null)
{
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
if (deactivateCoroutine == null)
{
deactivateCoroutine = StartCoroutine(Wait(10.0f));
}
ARGameObject.SetActive(true);
}
//end pose estimation
MatOfPoint2f pointMat = new MatOfPoint2f();
Imgproc.approxPolyDP(new MatOfPoint2f(contour.toArray()), pointMat, 3, true);
contour = new MatOfPoint(pointMat.toArray());
MatOfInt hull = new MatOfInt();
MatOfInt4 convexDefect = new MatOfInt4();
Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull);
if (hull.toArray().Length < 3)
{
return;
}
Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect);
List <MatOfPoint> hullPoints = new List <MatOfPoint>();
List <Point> listPo = new List <Point>();
for (int j = 0; j < hull.toList().Count; j++)
{
listPo.Add(contour.toList()[hull.toList()[j]]);
}
MatOfPoint e = new MatOfPoint();
e.fromList(listPo);
hullPoints.Add(e);
List <Point> listPoDefect = new List <Point>();
if (convexDefect.rows() > 0)
{
List <int> convexDefectList = convexDefect.toList();
List <Point> contourList = contour.toList();
for (int j = 0; j < convexDefectList.Count; j = j + 4)
{
Point farPoint = contourList[convexDefectList[j + 2]];
int depth = convexDefectList[j + 3];
//if (depth > threasholdSlider.value && farPoint.y < a)
//{
// listPoDefect.Add(contourList[convexDefectList[j + 2]]);
//}
//Debug.Log ("convexDefectList [" + j + "] " + convexDefectList [j + 3]);
}
}
Debug.Log("hull: " + hull.toList());
if (convexDefect.rows() > 0)
{
Debug.Log("defects: " + convexDefect.toList());
}
//use these contours to do heart detection
Imgproc.drawContours(rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3);
int defectsTotal = (int)convexDefect.total();
Debug.Log("Defect total " + defectsTotal);
this.numberOfFingers = listPoDefect.Count;
if (this.numberOfFingers > 5)
{
this.numberOfFingers = 5;
}
Debug.Log("numberOfFingers " + numberOfFingers);
Imgproc.putText(rgbaMat, "" + numberOfFingers, new Point(rgbaMat.cols() / 2, rgbaMat.rows() / 2), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 255, 255, 255), 6, Imgproc.LINE_AA, false);
numberOfFingersText.text = numberOfFingers.ToString();
foreach (Point p in listPoDefect)
{
Imgproc.circle(rgbaMat, p, 6, new Scalar(255, 0, 255, 255), -1);
}
}
