private void HandPoseEstimationProcess(Mat rgbaMat)
{
//Imgproc.blur(mRgba, mRgba, new Size(5,5));
Imgproc.GaussianBlur(rgbaMat, rgbaMat, new Size(3, 3), 1, 1);
//Imgproc.medianBlur(mRgba, mRgba, 3);
if (!isColorSelected)
{
return;
}
List <MatOfPoint> contours = detector.GetContours();
detector.Process(rgbaMat);
//Debug.Log ("Contours count: " + contours.Count);
if (contours.Count <= 0)
{
return;
}
RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[0].toArray()));
double boundWidth = rect.size.width;
double boundHeight = rect.size.height;
int boundPos = 0;
for (int i = 1; i < contours.Count; i++)
{
rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[i].toArray()));
if (rect.size.width * rect.size.height > boundWidth * boundHeight)
{
boundWidth = rect.size.width;
boundHeight = rect.size.height;
boundPos = i;
}
}
MatOfPoint contour = contours[boundPos];
OpenCVForUnity.CoreModule.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray()));
Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), CONTOUR_COLOR_WHITE, 2, 8, 0);
// Debug.Log (
// " Row start [" +
//(int)boundRect.tl ().y + "] row end [" +
// (int)boundRect.br ().y + "] Col start [" +
// (int)boundRect.tl ().x + "] Col end [" +
// (int)boundRect.br ().x + "]");
Point bottomLeft = new Point(boundRect.x, boundRect.y + boundRect.height);
Point topLeft = new Point(boundRect.x, boundRect.y);
Point bottomRight = new Point(boundRect.x + boundRect.width, boundRect.y + boundRect.height);
Point topRight = new Point(boundRect.x + boundRect.width, boundRect.y);
rectPoints = new MatOfPoint2f(new Point(boundRect.x, boundRect.y), //topleft
new Point(boundRect.x + boundRect.width, boundRect.y), //Top Right
new Point(boundRect.x + boundRect.width, boundRect.y + boundRect.height), //Bottom Right
new Point(boundRect.x, boundRect.y + boundRect.height) //Bottom Left
);
//double a = boundRect.br ().y - boundRect.tl ().y;
//a = a * 0.7;
//a = boundRect.tl ().y + a;
//Debug.Log (" A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]");
//Imgproc.rectangle (rgbaMat, boundRect.tl (), new Point (boundRect.br ().x, a), CONTOUR_COLOR, 2, 8, 0);
List <Point3> m_markerCorners3dList = new List <Point3>();
m_markerCorners3dList.Add(new Point3(-0.5f, -0.5f, 0)); //Top, Left (A)
m_markerCorners3dList.Add(new Point3(+0.5f, -0.5f, 0)); //Top, Right (B)
m_markerCorners3dList.Add(new Point3(+0.5f, +0.5f, 0)); //Bottom, Right (C)
m_markerCorners3dList.Add(new Point3(-0.5f, +0.5f, 0)); //Bottom, Left (D)
m_markerCorners3d.fromList(m_markerCorners3dList);
//estimate pose
Mat Rvec = new Mat();
Mat Tvec = new Mat();
Mat raux = new Mat();
Mat taux = new Mat();
Calib3d.solvePnP(m_markerCorners3d, rectPoints, camMatrix, distCoeff, raux, taux);
raux.convertTo(Rvec, CvType.CV_32F);
taux.convertTo(Tvec, CvType.CV_32F);
rotMat = new Mat(3, 3, CvType.CV_64FC1);
Calib3d.Rodrigues(Rvec, rotMat);
transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0)[0], (float)rotMat.get(0, 1)[0], (float)rotMat.get(0, 2)[0], (float)Tvec.get(0, 0)[0]));
transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0)[0], (float)rotMat.get(1, 1)[0], (float)rotMat.get(1, 2)[0], (float)Tvec.get(1, 0)[0]));
transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0)[0], (float)rotMat.get(2, 1)[0], (float)rotMat.get(2, 2)[0], (float)Tvec.get(2, 0)[0]));
transformationM.SetRow(3, new Vector4(0, 0, 0, 1));
//Debug.Log ("transformationM " + transformationM.ToString ());
Rvec.Dispose();
Tvec.Dispose();
raux.Dispose();
taux.Dispose();
rotMat.Dispose();
ARM = ARCamera.transform.localToWorldMatrix * invertYM * transformationM * invertZM;
//Debug.Log("arM " + ARM.ToString());
if (ARGameObject != null)
{
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
if (deactivateCoroutine == null)
{
deactivateCoroutine = StartCoroutine(Wait(10.0f));
}
ARGameObject.SetActive(true);
}
//end pose estimation
MatOfPoint2f pointMat = new MatOfPoint2f();
Imgproc.approxPolyDP(new MatOfPoint2f(contour.toArray()), pointMat, 3, true);
contour = new MatOfPoint(pointMat.toArray());
MatOfInt hull = new MatOfInt();
MatOfInt4 convexDefect = new MatOfInt4();
Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull);
if (hull.toArray().Length < 3)
{
return;
}
Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect);
List <MatOfPoint> hullPoints = new List <MatOfPoint>();
List <Point> listPo = new List <Point>();
for (int j = 0; j < hull.toList().Count; j++)
{
listPo.Add(contour.toList()[hull.toList()[j]]);
}
MatOfPoint e = new MatOfPoint();
e.fromList(listPo);
hullPoints.Add(e);
List <Point> listPoDefect = new List <Point>();
if (convexDefect.rows() > 0)
{
List <int> convexDefectList = convexDefect.toList();
List <Point> contourList = contour.toList();
for (int j = 0; j < convexDefectList.Count; j = j + 4)
{
Point farPoint = contourList[convexDefectList[j + 2]];
int depth = convexDefectList[j + 3];
//if (depth > threasholdSlider.value && farPoint.y < a)
//{
// listPoDefect.Add(contourList[convexDefectList[j + 2]]);
//}
//Debug.Log ("convexDefectList [" + j + "] " + convexDefectList [j + 3]);
}
}
Debug.Log("hull: " + hull.toList());
if (convexDefect.rows() > 0)
{
Debug.Log("defects: " + convexDefect.toList());
}
//use these contours to do heart detection
Imgproc.drawContours(rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3);
int defectsTotal = (int)convexDefect.total();
Debug.Log("Defect total " + defectsTotal);
this.numberOfFingers = listPoDefect.Count;
if (this.numberOfFingers > 5)
{
this.numberOfFingers = 5;
}
Debug.Log("numberOfFingers " + numberOfFingers);
Imgproc.putText(rgbaMat, "" + numberOfFingers, new Point(rgbaMat.cols() / 2, rgbaMat.rows() / 2), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 255, 255, 255), 6, Imgproc.LINE_AA, false);
numberOfFingersText.text = numberOfFingers.ToString();
foreach (Point p in listPoDefect)
{
Imgproc.circle(rgbaMat, p, 6, new Scalar(255, 0, 255, 255), -1);
}
}
/// <summary>
/// Hands the pose estimation process.
/// </summary>
public void handPoseEstimationProcess()
{
//Imgproc.blur(mRgba, mRgba, new Size(5,5));
Imgproc.GaussianBlur(rgbaMat, rgbaMat, new OpenCVForUnity.Size(3, 3), 1, 1);
//Imgproc.medianBlur(mRgba, mRgba, 3);
if (!isColorSelected)
{
return;
}
List <MatOfPoint> contours = detector.getContours();
detector.process(rgbaMat);
// Debug.Log ("Contours count: " + contours.Count);
if (contours.Count <= 0)
{
return;
}
RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours [0].toArray()));
double boundWidth = rect.size.width;
double boundHeight = rect.size.height;
int boundPos = 0;
for (int i = 1; i < contours.Count; i++)
{
rect = Imgproc.minAreaRect(new MatOfPoint2f(contours [i].toArray()));
if (rect.size.width * rect.size.height > boundWidth * boundHeight)
{
boundWidth = rect.size.width;
boundHeight = rect.size.height;
boundPos = i;
}
}
OpenCVForUnity.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contours [boundPos].toArray()));
Core.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), CONTOUR_COLOR_WHITE, 2, 8, 0);
// Debug.Log (
// " Row start [" +
// (int)boundRect.tl ().y + "] row end [" +
// (int)boundRect.br ().y + "] Col start [" +
// (int)boundRect.tl ().x + "] Col end [" +
// (int)boundRect.br ().x + "]");
int rectHeightThresh = 0;
double a = boundRect.br().y - boundRect.tl().y;
a = a * 0.7;
a = boundRect.tl().y + a;
// Debug.Log (
// " A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]");
//Core.rectangle( mRgba, boundRect.tl(), boundRect.br(), CONTOUR_COLOR, 2, 8, 0 );
Core.rectangle(rgbaMat, boundRect.tl(), new Point(boundRect.br().x, a), CONTOUR_COLOR, 2, 8, 0);
MatOfPoint2f pointMat = new MatOfPoint2f();
Imgproc.approxPolyDP(new MatOfPoint2f(contours [boundPos].toArray()), pointMat, 3, true);
contours [boundPos] = new MatOfPoint(pointMat.toArray());
MatOfInt hull = new MatOfInt();
MatOfInt4 convexDefect = new MatOfInt4();
Imgproc.convexHull(new MatOfPoint(contours [boundPos].toArray()), hull);
if (hull.toArray().Length < 3)
{
return;
}
Imgproc.convexityDefects(new MatOfPoint(contours [boundPos].toArray()), hull, convexDefect);
List <MatOfPoint> hullPoints = new List <MatOfPoint> ();
List <Point> listPo = new List <Point> ();
for (int j = 0; j < hull.toList().Count; j++)
{
listPo.Add(contours [boundPos].toList() [hull.toList() [j]]);
}
MatOfPoint e = new MatOfPoint();
e.fromList(listPo);
hullPoints.Add(e);
List <MatOfPoint> defectPoints = new List <MatOfPoint> ();
List <Point> listPoDefect = new List <Point> ();
for (int j = 0; j < convexDefect.toList().Count; j = j + 4)
{
Point farPoint = contours [boundPos].toList() [convexDefect.toList() [j + 2]];
int depth = convexDefect.toList() [j + 3];
if (depth > threasholdSlider.value && farPoint.y < a)
{
listPoDefect.Add(contours [boundPos].toList() [convexDefect.toList() [j + 2]]);
}
// Debug.Log ("defects [" + j + "] " + convexDefect.toList () [j + 3]);
}
MatOfPoint e2 = new MatOfPoint();
e2.fromList(listPo);
defectPoints.Add(e2);
// Debug.Log ("hull: " + hull.toList ());
// Debug.Log ("defects: " + convexDefect.toList ());
Imgproc.drawContours(rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3);
int defectsTotal = (int)convexDefect.total();
// Debug.Log ("Defect total " + defectsTotal);
this.numberOfFingers = listPoDefect.Count;
if (this.numberOfFingers > 5)
{
this.numberOfFingers = 5;
}
// Debug.Log ("numberOfFingers " + numberOfFingers);
// Core.putText (mRgba, "" + numberOfFingers, new Point (mRgba.cols () / 2, mRgba.rows () / 2), Core.FONT_HERSHEY_PLAIN, 4.0, new Scalar (255, 255, 255, 255), 6, Core.LINE_AA, false);
numberOfFingersText.text = numberOfFingers.ToString();
foreach (Point p in listPoDefect)
{
Core.circle(rgbaMat, p, 6, new Scalar(255, 0, 255, 255), -1);
}
}
/// <summary>
/// Hands the pose estimation process.
/// </summary>
public void handPoseEstimationProcess ()
{
//Imgproc.blur(mRgba, mRgba, new Size(5,5));
Imgproc.GaussianBlur (rgbaMat, rgbaMat, new OpenCVForUnity.Size (3, 3), 1, 1);
//Imgproc.medianBlur(mRgba, mRgba, 3);
if (!isColorSelected)
return;
List<MatOfPoint> contours = detector.getContours ();
detector.process (rgbaMat);
// Debug.Log ("Contours count: " + contours.Count);
if (contours.Count <= 0) {
return;
}
RotatedRect rect = Imgproc.minAreaRect (new MatOfPoint2f (contours [0].toArray ()));
double boundWidth = rect.size.width;
double boundHeight = rect.size.height;
int boundPos = 0;
for (int i = 1; i < contours.Count; i++) {
rect = Imgproc.minAreaRect (new MatOfPoint2f (contours [i].toArray ()));
if (rect.size.width * rect.size.height > boundWidth * boundHeight) {
boundWidth = rect.size.width;
boundHeight = rect.size.height;
boundPos = i;
}
}
OpenCVForUnity.Rect boundRect = Imgproc.boundingRect (new MatOfPoint (contours [boundPos].toArray ()));
Core.rectangle (rgbaMat, boundRect.tl (), boundRect.br (), CONTOUR_COLOR_WHITE, 2, 8, 0);
// Debug.Log (
// " Row start [" +
// (int)boundRect.tl ().y + "] row end [" +
// (int)boundRect.br ().y + "] Col start [" +
// (int)boundRect.tl ().x + "] Col end [" +
// (int)boundRect.br ().x + "]");
int rectHeightThresh = 0;
double a = boundRect.br ().y - boundRect.tl ().y;
a = a * 0.7;
a = boundRect.tl ().y + a;
// Debug.Log (
// " A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]");
//Core.rectangle( mRgba, boundRect.tl(), boundRect.br(), CONTOUR_COLOR, 2, 8, 0 );
Core.rectangle (rgbaMat, boundRect.tl (), new Point (boundRect.br ().x, a), CONTOUR_COLOR, 2, 8, 0);
MatOfPoint2f pointMat = new MatOfPoint2f ();
Imgproc.approxPolyDP (new MatOfPoint2f (contours [boundPos].toArray ()), pointMat, 3, true);
contours [boundPos] = new MatOfPoint (pointMat.toArray ());
MatOfInt hull = new MatOfInt ();
MatOfInt4 convexDefect = new MatOfInt4 ();
Imgproc.convexHull (new MatOfPoint (contours [boundPos].toArray ()), hull);
if (hull.toArray ().Length < 3)
return;
Imgproc.convexityDefects (new MatOfPoint (contours [boundPos] .toArray ()), hull, convexDefect);
List<MatOfPoint> hullPoints = new List<MatOfPoint> ();
List<Point> listPo = new List<Point> ();
for (int j = 0; j < hull.toList().Count; j++) {
listPo.Add (contours [boundPos].toList () [hull.toList () [j]]);
}
MatOfPoint e = new MatOfPoint ();
e.fromList (listPo);
hullPoints.Add (e);
List<MatOfPoint> defectPoints = new List<MatOfPoint> ();
List<Point> listPoDefect = new List<Point> ();
for (int j = 0; j < convexDefect.toList().Count; j = j+4) {
Point farPoint = contours [boundPos].toList () [convexDefect.toList () [j + 2]];
int depth = convexDefect.toList () [j + 3];
if (depth > threasholdSlider.value && farPoint.y < a) {
listPoDefect.Add (contours [boundPos].toList () [convexDefect.toList () [j + 2]]);
}
// Debug.Log ("defects [" + j + "] " + convexDefect.toList () [j + 3]);
}
MatOfPoint e2 = new MatOfPoint ();
e2.fromList (listPo);
defectPoints.Add (e2);
// Debug.Log ("hull: " + hull.toList ());
// Debug.Log ("defects: " + convexDefect.toList ());
Imgproc.drawContours (rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3);
int defectsTotal = (int)convexDefect.total ();
// Debug.Log ("Defect total " + defectsTotal);
this.numberOfFingers = listPoDefect.Count;
if (this.numberOfFingers > 5)
this.numberOfFingers = 5;
// Debug.Log ("numberOfFingers " + numberOfFingers);
// Core.putText (mRgba, "" + numberOfFingers, new Point (mRgba.cols () / 2, mRgba.rows () / 2), Core.FONT_HERSHEY_PLAIN, 4.0, new Scalar (255, 255, 255, 255), 6, Core.LINE_AA, false);
numberOfFingersText.text = numberOfFingers.ToString ();
foreach (Point p in listPoDefect) {
Core.circle (rgbaMat, p, 6, new Scalar (255, 0, 255, 255), -1);
}
}
