/// <summary>
/// To release the resources for the initialized method.
/// </summary>
public override void Dispose()
{
DisposeOpticalFlow();
if (prevTrackPtsMat != null)
{
prevTrackPtsMat.Dispose();
}
}
/// <summary>
/// Raises the web cam texture to mat helper disposed event.
/// </summary>
public void OnWebCamTextureToMatHelperDisposed()
{
Debug.Log("OnWebCamTextureToMatHelperDisposed");
matOpFlowThis.Dispose();
matOpFlowPrev.Dispose();
MOPcorners.Dispose();
mMOP2fptsThis.Dispose();
mMOP2fptsPrev.Dispose();
mMOP2fptsSafe.Dispose();
mMOBStatus.Dispose();
mMOFerr.Dispose();
}
/// <summary>
/// To release the resources for the initialized method.
/// </summary>
public override void Dispose()
{
if (src_points != null)
{
src_points.Clear();
}
DisposeKalmanFilter();
if (prevTrackPtsMat != null)
{
prevTrackPtsMat.Dispose();
}
}
//辨識輪廓
private bool analysisContoursRect(int index, List <MatOfPoint> contours, Mat result, List <MatchObject> matchObject)
{
OpenCVForUnity.Rect _testDepthRect = Imgproc.boundingRect(contours[index]);
float minAreaSize = _minDepthObjectSizePer * _drawBlock.MatchHeight * _drawBlock.MatchWidth;
if (_testDepthRect.area() > minAreaSize)
{
//宣告放置點資料
MatOfInt hullInt = new MatOfInt();
List <Point> hullPointList = new List <Point>();
MatOfPoint hullPointMat = new MatOfPoint();
List <MatOfPoint> hullPoints = new List <MatOfPoint>();
MatOfInt4 defects = new MatOfInt4();
//篩選點資料
MatOfPoint2f Temp2f = new MatOfPoint2f();
//Convert contours(i) from MatOfPoint to MatOfPoint2f
contours[index].convertTo(Temp2f, CvType.CV_32FC2);
//Processing on mMOP2f1 which is in type MatOfPoint2f
Imgproc.approxPolyDP(Temp2f, Temp2f, 30, true);
//Convert back to MatOfPoint and put the new values back into the contours list
Temp2f.convertTo(contours[index], CvType.CV_32S);
//计算轮廓围绕的凸形壳
Imgproc.convexHull(contours[index], hullInt);
List <Point> pointMatList = contours[index].toList();
List <int> hullIntList = hullInt.toList();
for (int j = 0; j < hullInt.toList().Count; j++)
{
hullPointList.Add(pointMatList[hullIntList[j]]);
hullPointMat.fromList(hullPointList);
hullPoints.Add(hullPointMat);
}
if (hullInt.toList().Count == 4)
{
if (!setMatchObject(index, pointMatList, contours, hullPoints, result, matchObject))
{
//Debug.Log("setMatchObject fail");
}
}
//清空記憶體
defects.Dispose();
hullPointList.Clear();
hullPointMat.Dispose();
hullInt.Dispose();
hullPoints.Clear();
return(true);
}
return(false);
}
/// <summary>
/// Raises the web cam texture to mat helper disposed event.
/// </summary>
public void OnWebCamTextureToMatHelperDisposed()
{
Debug.Log("OnWebCamTextureToMatHelperDisposed");
if (texture != null)
{
Texture2D.Destroy(texture);
texture = null;
}
if (matOpFlowThis != null)
{
matOpFlowThis.Dispose();
}
if (matOpFlowPrev != null)
{
matOpFlowPrev.Dispose();
}
if (MOPcorners != null)
{
MOPcorners.Dispose();
}
if (mMOP2fptsThis != null)
{
mMOP2fptsThis.Dispose();
}
if (mMOP2fptsPrev != null)
{
mMOP2fptsPrev.Dispose();
}
if (mMOP2fptsSafe != null)
{
mMOP2fptsSafe.Dispose();
}
if (mMOBStatus != null)
{
mMOBStatus.Dispose();
}
if (mMOFerr != null)
{
mMOFerr.Dispose();
}
}
void OnDisable()
{
Debug.Log("OnDisable");
if (matOpFlowThis != null)
{
matOpFlowThis.Dispose();
}
if (matOpFlowPrev != null)
{
matOpFlowPrev.Dispose();
}
if (MOPcorners != null)
{
MOPcorners.Dispose();
}
if (mMOP2fptsThis != null)
{
mMOP2fptsThis.Dispose();
}
if (mMOP2fptsPrev != null)
{
mMOP2fptsPrev.Dispose();
}
if (mMOP2fptsSafe != null)
{
mMOP2fptsSafe.Dispose();
}
if (mMOBStatus != null)
{
mMOBStatus.Dispose();
}
if (mMOFerr != null)
{
mMOFerr.Dispose();
}
//webCamTextureToMatHelper.Dispose();
}
private IEnumerator init()
{
if (webCamTexture != null)
{
webCamTexture.Stop();
initDone = false;
rgbaMat.Dispose();
matOpFlowThis.Dispose();
matOpFlowPrev.Dispose();
MOPcorners.Dispose();
mMOP2fptsThis.Dispose();
mMOP2fptsPrev.Dispose();
mMOP2fptsSafe.Dispose();
mMOBStatus.Dispose();
mMOFerr.Dispose();
}
// Checks how many and which cameras are available on the device
for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++)
{
if (WebCamTexture.devices [cameraIndex].isFrontFacing == isFrontFacing)
{
Debug.Log(cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing);
webCamDevice = WebCamTexture.devices [cameraIndex];
webCamTexture = new WebCamTexture(webCamDevice.name, width, height);
break;
}
}
if (webCamTexture == null)
{
webCamDevice = WebCamTexture.devices [0];
webCamTexture = new WebCamTexture(webCamDevice.name, width, height);
}
Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
// Starts the camera
webCamTexture.Play();
while (true)
{
//If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/)
#if UNITY_IPHONE && !UNITY_EDITOR
if (webCamTexture.width > 16 && webCamTexture.height > 16)
{
#else
if (webCamTexture.didUpdateThisFrame)
{
#endif
Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
Debug.Log("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing);
colors = new Color32[webCamTexture.width * webCamTexture.height];
rgbaMat = new Mat(webCamTexture.height, webCamTexture.width, CvType.CV_8UC4);
matOpFlowThis = new Mat();
matOpFlowPrev = new Mat();
MOPcorners = new MatOfPoint();
mMOP2fptsThis = new MatOfPoint2f();
mMOP2fptsPrev = new MatOfPoint2f();
mMOP2fptsSafe = new MatOfPoint2f();
mMOBStatus = new MatOfByte();
mMOFerr = new MatOfFloat();
texture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
gameObject.transform.eulerAngles = new Vector3(0, 0, 0);
#if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR
gameObject.transform.eulerAngles = new Vector3(0, 0, -90);
#endif
// gameObject.transform.rotation = gameObject.transform.rotation * Quaternion.AngleAxis (webCamTexture.videoRotationAngle, Vector3.back);
gameObject.transform.localScale = new Vector3(webCamTexture.width, webCamTexture.height, 1);
// bool videoVerticallyMirrored = webCamTexture.videoVerticallyMirrored;
// float scaleX = 1;
// float scaleY = videoVerticallyMirrored ? -1.0f : 1.0f;
// if (webCamTexture.videoRotationAngle == 270)
// scaleY = -1.0f;
// gameObject.transform.localScale = new Vector3 (scaleX * gameObject.transform.localScale.x, scaleY * gameObject.transform.localScale.y, 1);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
#if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR
Camera.main.orthographicSize = webCamTexture.width / 2;
#else
Camera.main.orthographicSize = webCamTexture.height / 2;
#endif
initDone = true;
break;
}
else
{
yield return(0);
}
}
}
// Update is called once per frame
void Update()
{
if (!initDone)
{
return;
}
#if UNITY_IPHONE && !UNITY_EDITOR
if (webCamTexture.width > 16 && webCamTexture.height > 16)
{
#else
if (webCamTexture.didUpdateThisFrame)
{
#endif
Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors);
if (webCamTexture.videoVerticallyMirrored)
{
if (webCamDevice.isFrontFacing)
{
if (webCamTexture.videoRotationAngle == 0)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
else if (webCamTexture.videoRotationAngle == 90)
{
Core.flip(rgbaMat, rgbaMat, 0);
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
}
else
{
if (webCamTexture.videoRotationAngle == 90)
{
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, -1);
}
}
}
else
{
if (webCamDevice.isFrontFacing)
{
if (webCamTexture.videoRotationAngle == 0)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
else if (webCamTexture.videoRotationAngle == 90)
{
Core.flip(rgbaMat, rgbaMat, 0);
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
}
else
{
if (webCamTexture.videoRotationAngle == 90)
{
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, -1);
}
}
}
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
}
/*
* Parameters:
* prevImg first 8-bit input image
* nextImg second input image
* prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
* nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
* status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
* err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (!mMOBStatus.empty())
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++)
{
if (byteStatus [x] == 1)
{
Point pt = cornersThis [x];
Point pt2 = cornersPrev [x];
Core.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1);
Core.line(rgbaMat, pt, pt2, colorRed, iLineThickness);
}
}
}
Utils.matToTexture2D(rgbaMat, texture, colors);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}
}
void OnDisable()
{
webCamTexture.Stop();
}
void OnGUI()
{
float screenScale = Screen.width / 240.0f;
Matrix4x4 scaledMatrix = Matrix4x4.Scale(new Vector3(screenScale, screenScale, screenScale));
GUI.matrix = scaledMatrix;
GUILayout.BeginVertical();
if (GUILayout.Button("back"))
{
Application.LoadLevel("OpenCVForUnitySample");
}
if (GUILayout.Button("change camera"))
{
isFrontFacing = !isFrontFacing;
StartCoroutine(init());
}
GUILayout.EndVertical();
}
}
}
//利用深度的輪廓做RGB的顏色判斷
public Mat getContours(Mat srcColorMat, Mat srcDepthMat)
{
Mat ColorMat = new Mat();
Mat DepthMat = new Mat();
Mat HsvMat = new Mat();
srcColorMat.copyTo(ColorMat);
srcDepthMat.copyTo(DepthMat);
Imgproc.cvtColor(ColorMat, HsvMat, Imgproc.COLOR_BGR2HSV);
List <ColorObject> colorObjects = new List <ColorObject>();
Mat resultMat = new Mat(DepthMat.height(), DepthMat.width(), CvType.CV_8UC1);
Mat hierarchy = new Mat();
List <Point> ConsistP = new List <Point>();
List <MatOfPoint> contours = new List <MatOfPoint>();
List <List <Point> > trianglePointList = new List <List <Point> >();
Imgproc.findContours(DepthMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
int numObjects = contours.Count;
List <Scalar> clickRGB = new List <Scalar>();
List <Scalar> clickHSV = new List <Scalar>();
List <int> HullCountList = new List <int>();
for (int i = 0; i < numObjects; i++)
{
Imgproc.drawContours(resultMat, contours, i, new Scalar(255), 1);
}
double[] GetRGB = new double[10];
float minAreaSize = _minDepthObjectSizePer * _drawBlock.MatchHeight * _drawBlock.MatchWidth;
if (numObjects > 0)
{
for (int index = 0; index < numObjects; index++)
{
OpenCVForUnity.Rect R0 = Imgproc.boundingRect(contours[index]);
if (R0.area() > minAreaSize)
{
//宣告放置點資料
MatOfInt hullInt = new MatOfInt();
List <Point> hullPointList = new List <Point>();
MatOfPoint hullPointMat = new MatOfPoint();
List <MatOfPoint> hullPoints = new List <MatOfPoint>();
MatOfInt4 defects = new MatOfInt4();
//篩選點資料
MatOfPoint2f Temp2f = new MatOfPoint2f();
//Convert contours(i) from MatOfPoint to MatOfPoint2f
contours[index].convertTo(Temp2f, CvType.CV_32FC2);
//Processing on mMOP2f1 which is in type MatOfPoint2f
Imgproc.approxPolyDP(Temp2f, Temp2f, 30, true);
//Convert back to MatOfPoint and put the new values back into the contours list
Temp2f.convertTo(contours[index], CvType.CV_32S);
//计算轮廓围绕的凸形壳
Imgproc.convexHull(contours[index], hullInt);
List <Point> pointMatList = contours[index].toList();
List <int> hullIntList = hullInt.toList();
for (int j = 0; j < hullInt.toList().Count; j++)
{
hullPointList.Add(pointMatList[hullIntList[j]]);
hullPointMat.fromList(hullPointList);
hullPoints.Add(hullPointMat);
}
ConsistP.Add(new Point(R0.x, R0.y));
ConsistP.Add(new Point(R0.x + R0.width, R0.y + R0.height));
ConsistP.Add(new Point(R0.x + R0.width, R0.y));
ConsistP.Add(new Point(R0.x, R0.y + R0.height));
clickRGB.Add(clickcolor(ColorMat, R0));
clickHSV.Add(clickcolor(HsvMat, R0));
HullCountList.Add(hullIntList.Count);
trianglePointList.Add(pointMatList);
//清空記憶體
defects.Dispose();
hullPointList.Clear();
hullPointMat.Dispose();
hullInt.Dispose();
hullPoints.Clear();
//Debug.Log("ID = " + index + " Color = " + clickcolor(ColorMat, R0));
}
}
//使用顏色找尋物體
_matchColorObjectList = setColorMatchObject(ConsistP, trianglePointList, clickRGB, clickHSV, resultMat, HullCountList);
}
return(resultMat);
}
private IEnumerator init()
{
if (webCamTexture != null)
{
webCamTexture.Stop();
initDone = false;
rgbaMat.Dispose();
matOpFlowThis.Dispose();
matOpFlowPrev.Dispose();
MOPcorners.Dispose();
mMOP2fptsThis.Dispose();
mMOP2fptsPrev.Dispose();
mMOP2fptsSafe.Dispose();
mMOBStatus.Dispose();
mMOFerr.Dispose();
}
// Checks how many and which cameras are available on the device
for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++)
{
if (WebCamTexture.devices [cameraIndex].isFrontFacing == shouldUseFrontFacing)
{
Debug.Log(cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing);
webCamDevice = WebCamTexture.devices [cameraIndex];
webCamTexture = new WebCamTexture(webCamDevice.name, width, height);
break;
}
}
if (webCamTexture == null)
{
webCamDevice = WebCamTexture.devices [0];
webCamTexture = new WebCamTexture(webCamDevice.name, width, height);
}
Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
// Starts the camera
webCamTexture.Play();
while (true)
{
//If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/)
#if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1)
if (webCamTexture.width > 16 && webCamTexture.height > 16)
{
#else
if (webCamTexture.didUpdateThisFrame)
{
#if UNITY_IOS && !UNITY_EDITOR && UNITY_5_2
while (webCamTexture.width <= 16)
{
webCamTexture.GetPixels32();
yield return(new WaitForEndOfFrame());
}
#endif
#endif
Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
Debug.Log("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing);
colors = new Color32[webCamTexture.width * webCamTexture.height];
rgbaMat = new Mat(webCamTexture.height, webCamTexture.width, CvType.CV_8UC4);
matOpFlowThis = new Mat();
matOpFlowPrev = new Mat();
MOPcorners = new MatOfPoint();
mMOP2fptsThis = new MatOfPoint2f();
mMOP2fptsPrev = new MatOfPoint2f();
mMOP2fptsSafe = new MatOfPoint2f();
mMOBStatus = new MatOfByte();
mMOFerr = new MatOfFloat();
texture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
updateLayout();
screenOrientation = Screen.orientation;
initDone = true;
break;
}
else
{
yield return(0);
}
}
}
private void FindDefects(Mat maskImage, ref int cx, ref int cy, int min_defects_count, int max_defects_count)
{
int erosion_size = 1;
Mat element = Imgproc.getStructuringElement(
Imgproc.MORPH_ELLIPSE,
new Size(2 * erosion_size + 1, 2 * erosion_size + 1),
new Point(erosion_size, erosion_size));
// dilate and erode
Imgproc.dilate(maskImage, maskImage, element);
Imgproc.erode(maskImage, maskImage, element);
element.Dispose();
//Find Contours in image
List <MatOfPoint> contours = new List <MatOfPoint>();
Imgproc.findContours(maskImage, contours, new MatOfPoint(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
//Loop to find the biggest contour; If no contour is found index=-1
int index = -1;
double area = 2000;
for (int i = 0; i < contours.Count; i++)
{
var tempsize = Imgproc.contourArea(contours[i]);
if (tempsize > area)
{
area = tempsize;
index = i;
}
}
if (index == -1)
{
return;
}
else
{
var points = new MatOfPoint(contours[index].toArray());
var hull = new MatOfInt();
Imgproc.convexHull(points, hull, false);
var defects = new MatOfInt4();
Imgproc.convexityDefects(points, hull, defects);
var start_points = new MatOfPoint2f();
var far_points = new MatOfPoint2f();
for (int i = 0; i < defects.size().height; i++)
{
int ind_start = (int)defects.get(i, 0)[0];
int ind_end = (int)defects.get(i, 0)[1];
int ind_far = (int)defects.get(i, 0)[2];
double depth = defects.get(i, 0)[3] / 256;
double a = Core.norm(contours[index].row(ind_start) - contours[index].row(ind_end));
double b = Core.norm(contours[index].row(ind_far) - contours[index].row(ind_start));
double c = Core.norm(contours[index].row(ind_far) - contours[index].row(ind_end));
double angle = Math.Acos((b * b + c * c - a * a) / (2 * b * c)) * 180.0 / Math.PI;
double threshFingerLength = ((double)maskImage.height()) / 8.0;
double threshAngle = 80;
if (angle < threshAngle && depth > threshFingerLength)
{
//start point
var aa = contours[index].row(ind_start);
start_points.push_back(contours[index].row(ind_start));
far_points.push_back(contours[index].row(ind_far));
}
}
points.Dispose();
hull.Dispose();
defects.Dispose();
// when no finger found
if (far_points.size().height < min_defects_count || far_points.size().height > max_defects_count)
{
return;
}
var cnts = new List <MatOfPoint>();
cnts.Add(contours[index]);
Mat mm = new Mat();
Imgproc.cvtColor(maskImage, mm, Imgproc.COLOR_GRAY2BGR);
Imgproc.drawContours(mm, cnts, 0, new Scalar(0, 0, 255));
// OpenCVForUnity.ImgcodecsModule.Imgcodecs.imwrite("D:/tempImg.jpg", mm)
//var rotatedRect = Imgproc.minAreaRect(far_points);
var boundingRect = Imgproc.boundingRect(far_points);
cx = (int)(boundingRect.x + boundingRect.width / 2);
cy = (int)(boundingRect.y + boundingRect.height / 2);
}
}
