private void drawPoints() { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus[x] == 1) { Point pt = cornersThis[x]; Point pt2 = cornersPrev[x]; Core.circle(rgbaMat, pt, 5, colorRed, lineThickness - 1); Core.line(rgbaMat, pt, pt2, colorRed, lineThickness); } } }
//public RawImage document; void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat mainMat = webCamTextureToMatHelper.GetMat(); if (!selectTarget) //find paper by contours { grayMat = new Mat(); // convert texture to matrix mainMat.copyTo(grayMat); mainMat = findPaper(mainMat); // display matrix on the screen Utils.fastMatToTexture2D(mainMat, texture); } else { // using optical flow // set the currentGrayMat mat currentGrayMat = new Mat(mainMat.rows(), mainMat.cols(), Imgproc.COLOR_RGB2GRAY); Imgproc.cvtColor(mainMat, currentGrayMat, Imgproc.COLOR_RGBA2GRAY); if (initOpticalFlow == true) // doing the init setting for optical flow { // create 40 points Point[] points = new Point[40]; // set those points near the corner // paperCornerMatOfPoint is the corner of the paper for (int i = 0; i < 4; i++) { points[i * 10] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 1] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 2] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y + 1); points[i * 10 + 3] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y + 1); points[i * 10 + 4] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 1); points[i * 10 + 5] = new Point(paperCornerMatOfPoint.toList()[i].x - 1, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 6] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 1); points[i * 10 + 7] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 2); points[i * 10 + 8] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 2); points[i * 10 + 9] = new Point(paperCornerMatOfPoint.toList()[i].x + 2, paperCornerMatOfPoint.toList()[i].y + 2); } // make the points closer to the corners (Harris Corner Detection ) //Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40, qualityLevel, minDistance, none, blockSize, false, 0.04); //Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40,0.05,20); corners.fromArray(points); prevFeatures.fromList(corners.toList()); currentFeatures.fromList(corners.toList()); prevGrayMat = currentGrayMat.clone(); // won't go back t again initOpticalFlow = false; // not that useful lol // create random color // not working now for (int i = 0; i < maxCorners; i++) { color.Add(new Scalar((int)(Random.value * 255), (int)(Random.value * 255), (int)(Random.value * 255), 255)); } } else { // Don't want ball move //currentFeatures.fromArray(prevFeatures.toArray()); // want ball move prevFeatures.fromArray(currentFeatures.toArray()); // optical flow it will changes the valu of currentFeatures Video.calcOpticalFlowPyrLK(prevGrayMat, currentGrayMat, prevFeatures, currentFeatures, mMOBStatus, err); //Debug.Log(st.rows()); // change to points list List <Point> prevList = prevFeatures.toList(), nextList = currentFeatures.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus[x] == 1) { Point pt = nextList[x]; Point pt2 = prevList[x]; Imgproc.circle(mainMat, pt, 10, new Scalar(0, 0, 255), -1); Imgproc.line(mainMat, pt, pt2, new Scalar(0, 0, 255)); } } // draw the data //for (int i = 0; i < prevList.Count; i++) //{ // //Imgproc.circle(frame, prevList[i], 5, color[10]); // Imgproc.circle(mainMat, nextList[i], 10, new Scalar(0, 0, 255), -1); // Imgproc.line(mainMat, prevList[i], nextList[i], color[20]); //} List <List <Point> > cornersFeatures = new List <List <Point> >(40); cornersFeatures.Add(new List <Point>(10)); // put the corners features data into the list int tmp = 0; bool last = true; for (int i = 0; i < nextList.Count - 1; i++) { if (Mathf.Abs((float)(nextList[i].x - nextList[i + 1].x)) < 10 && Mathf.Abs((float)(nextList[i].y - nextList[i + 1].y)) < 10) { if (last == true) { cornersFeatures[tmp].Add(nextList[i]); } else { cornersFeatures.Add(new List <Point>(10)); tmp = tmp + 1; cornersFeatures[tmp].Add(nextList[i]); } last = true; } else { last = false; } } // count corners int manyCornersFeatures = 0; for (int i = 0; i < cornersFeatures.Count; i++) { Debug.Log(cornersFeatures[i].Count); if (cornersFeatures[i].Count < 5) { cornersFeatures.RemoveAt(i); } else { manyCornersFeatures++; } } //Debug.Log("Length" + manyCornersFeatures); // if corners equal 4 then diplay virtual docunment into the frame // doing the perspective transform if (manyCornersFeatures == 4) { Mat documentMat = new Mat(document.height, document.width, CvType.CV_8UC3); Utils.texture2DToMat(document, documentMat); List <Point> srcPoints = new List <Point>(); srcPoints.Add(new Point(0, 0)); srcPoints.Add(new Point(documentMat.cols(), 0)); srcPoints.Add(new Point(documentMat.cols(), documentMat.rows())); srcPoints.Add(new Point(0, documentMat.rows())); Mat srcPointsMat = Converters.vector_Point_to_Mat(srcPoints, CvType.CV_32F); List <Point> dstPoints = new List <Point>() { cornersFeatures[0][0], cornersFeatures[1][0], cornersFeatures[2][0], cornersFeatures[3][0] }; Mat dstPointsMat = Converters.vector_Point_to_Mat(dstPoints, CvType.CV_32F); //Make perspective transform Mat m = Imgproc.getPerspectiveTransform(srcPointsMat, dstPointsMat); Mat warpedMat = new Mat(new Size(), documentMat.type()); Debug.Log((cornersFeatures[1][0].x - cornersFeatures[0][0].x) + " " + (cornersFeatures[2][0].y - cornersFeatures[1][0].y)); Imgproc.warpPerspective(documentMat, warpedMat, m, mainMat.size(), Imgproc.INTER_LINEAR); //warpedMat.convertTo(warpedMat, CvType.CV_32F); //warpedMat.convertTo(warpedMat, CvType.CV_8UC3); warpedMat.convertTo(warpedMat, CvType.CV_8UC3); // same size as frame Mat dst = new Mat(mainMat.size(), CvType.CV_8UC3); //Mat dst = new Mat(frame.size(), CvType.CV_8UC3); //Mat dst2 = new Mat(); Imgproc.cvtColor(mainMat, dst, Imgproc.COLOR_RGBA2RGB); //dst.setTo(new Scalar(0, 255, 0)); //currentGrayMat.copyTo(dst); //dst.convertTo(dst, CvType.CV_8UC3); //Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA); Mat img1 = new Mat(); Mat mask = new Mat(mainMat.size(), CvType.CV_8UC1, new Scalar(0)); Imgproc.cvtColor(warpedMat, img1, Imgproc.COLOR_RGB2GRAY); Imgproc.Canny(img1, img1, 100, 200); List <MatOfPoint> doc_contours = new List <MatOfPoint>();; Imgproc.findContours(img1, doc_contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_NONE); Imgproc.drawContours(mask, doc_contours, -1, new Scalar(255), Core.FILLED); warpedMat.copyTo(dst, mask); dst.convertTo(dst, CvType.CV_8UC3); Debug.Log("dst" + dst.size()); Imgproc.cvtColor(dst, mainMat, Imgproc.COLOR_RGB2RGBA); // display on the right Texture2D finalTextue = new Texture2D(dst.width(), dst.height(), TextureFormat.RGB24, false); Utils.matToTexture2D(dst, finalTextue); targetRawImage.texture = finalTextue; } // current frame to old frame prevGrayMat = currentGrayMat.clone(); //Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA); // display matrix on the screen Utils.fastMatToTexture2D(mainMat, texture); } } } }
/// <summary> /// Refines the matches with homography. /// </summary> /// <returns><c>true</c>, if matches with homography was refined, <c>false</c> otherwise.</returns> /// <param name="queryKeypoints">Query keypoints.</param> /// <param name="trainKeypoints">Train keypoints.</param> /// <param name="reprojectionThreshold">Reprojection threshold.</param> /// <param name="matches">Matches.</param> /// <param name="homography">Homography.</param> static bool refineMatchesWithHomography ( MatOfKeyPoint queryKeypoints, MatOfKeyPoint trainKeypoints, float reprojectionThreshold, MatOfDMatch matches, Mat homography ) { //Debug.Log ("matches " + matches.ToString ()); int minNumberMatchesAllowed = 8; List <KeyPoint> queryKeypointsList = queryKeypoints.toList(); List <KeyPoint> trainKeypointsList = trainKeypoints.toList(); List <DMatch> matchesList = matches.toList(); if (matchesList.Count < minNumberMatchesAllowed) { return(false); } // Prepare data for cv::findHomography List <Point> srcPointsList = new List <Point>(matchesList.Count); List <Point> dstPointsList = new List <Point>(matchesList.Count); for (int i = 0; i < matchesList.Count; i++) { srcPointsList.Add(trainKeypointsList[matchesList[i].trainIdx].pt); dstPointsList.Add(queryKeypointsList[matchesList[i].queryIdx].pt); } // Find homography matrix and get inliers mask using (MatOfPoint2f srcPoints = new MatOfPoint2f()) using (MatOfPoint2f dstPoints = new MatOfPoint2f()) using (MatOfByte inliersMask = new MatOfByte(new byte[srcPointsList.Count])) { srcPoints.fromList(srcPointsList); dstPoints.fromList(dstPointsList); //Debug.Log ("srcPoints " + srcPoints.ToString ()); //Debug.Log ("dstPoints " + dstPoints.ToString ()); Calib3d.findHomography(srcPoints, dstPoints, Calib3d.FM_RANSAC, reprojectionThreshold, inliersMask, 2000, 0.955).copyTo(homography); if (homography.rows() != 3 || homography.cols() != 3) { return(false); } //Debug.Log ("homography " + homography.ToString ()); //Debug.Log ("inliersMask " + inliersMask.dump ()); List <byte> inliersMaskList = inliersMask.toList(); List <DMatch> inliers = new List <DMatch>(); for (int i = 0; i < inliersMaskList.Count; i++) { if (inliersMaskList[i] == 1) { inliers.Add(matchesList[i]); } } matches.fromList(inliers); //Debug.Log ("matches " + matches.ToString ()); } return(matchesList.Count > minNumberMatchesAllowed); }
private IEnumerator init() { if (webCamTexture != null) { webCamTexture.Stop(); initDone = false; rgbaMat.Dispose(); matOpFlowThis.Dispose(); matOpFlowPrev.Dispose(); MOPcorners.Dispose(); mMOP2fptsThis.Dispose(); mMOP2fptsPrev.Dispose(); mMOP2fptsSafe.Dispose(); mMOBStatus.Dispose(); mMOFerr.Dispose(); } // Checks how many and which cameras are available on the device for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) { if (WebCamTexture.devices [cameraIndex].isFrontFacing == isFrontFacing) { Debug.Log(cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing); webCamDevice = WebCamTexture.devices [cameraIndex]; webCamTexture = new WebCamTexture(webCamDevice.name, width, height); break; } } if (webCamTexture == null) { webCamDevice = WebCamTexture.devices [0]; webCamTexture = new WebCamTexture(webCamDevice.name, width, height); } Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS); // Starts the camera webCamTexture.Play(); while (true) { //If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/) #if UNITY_IPHONE && !UNITY_EDITOR if (webCamTexture.width > 16 && webCamTexture.height > 16) { #else if (webCamTexture.didUpdateThisFrame) { #endif Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS); Debug.Log("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing); colors = new Color32[webCamTexture.width * webCamTexture.height]; rgbaMat = new Mat(webCamTexture.height, webCamTexture.width, CvType.CV_8UC4); matOpFlowThis = new Mat(); matOpFlowPrev = new Mat(); MOPcorners = new MatOfPoint(); mMOP2fptsThis = new MatOfPoint2f(); mMOP2fptsPrev = new MatOfPoint2f(); mMOP2fptsSafe = new MatOfPoint2f(); mMOBStatus = new MatOfByte(); mMOFerr = new MatOfFloat(); texture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false); gameObject.transform.eulerAngles = new Vector3(0, 0, 0); #if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR gameObject.transform.eulerAngles = new Vector3(0, 0, -90); #endif // gameObject.transform.rotation = gameObject.transform.rotation * Quaternion.AngleAxis (webCamTexture.videoRotationAngle, Vector3.back); gameObject.transform.localScale = new Vector3(webCamTexture.width, webCamTexture.height, 1); // bool videoVerticallyMirrored = webCamTexture.videoVerticallyMirrored; // float scaleX = 1; // float scaleY = videoVerticallyMirrored ? -1.0f : 1.0f; // if (webCamTexture.videoRotationAngle == 270) // scaleY = -1.0f; // gameObject.transform.localScale = new Vector3 (scaleX * gameObject.transform.localScale.x, scaleY * gameObject.transform.localScale.y, 1); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; #if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR Camera.main.orthographicSize = webCamTexture.width / 2; #else Camera.main.orthographicSize = webCamTexture.height / 2; #endif initDone = true; break; } else { yield return(0); } } } // Update is called once per frame void Update() { if (!initDone) { return; } #if UNITY_IPHONE && !UNITY_EDITOR if (webCamTexture.width > 16 && webCamTexture.height > 16) { #else if (webCamTexture.didUpdateThisFrame) { #endif Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors); if (webCamTexture.videoVerticallyMirrored) { if (webCamDevice.isFrontFacing) { if (webCamTexture.videoRotationAngle == 0) { Core.flip(rgbaMat, rgbaMat, 1); } else if (webCamTexture.videoRotationAngle == 90) { Core.flip(rgbaMat, rgbaMat, 0); } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, 1); } } else { if (webCamTexture.videoRotationAngle == 90) { } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, -1); } } } else { if (webCamDevice.isFrontFacing) { if (webCamTexture.videoRotationAngle == 0) { Core.flip(rgbaMat, rgbaMat, 1); } else if (webCamTexture.videoRotationAngle == 90) { Core.flip(rgbaMat, rgbaMat, 0); } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, 1); } } else { if (webCamTexture.videoRotationAngle == 90) { } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, -1); } } } if (mMOP2fptsPrev.rows() == 0) { // first time through the loop so we need prev and this mats // plus prev points // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // copy that to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get prev corners Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsPrev.fromArray(MOPcorners.toArray()); // get safe copy of this corners mMOP2fptsPrev.copyTo(mMOP2fptsSafe); } else { // we've been through before so // this mat is valid. Copy it to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // get the corners for this mat Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsThis.fromArray(MOPcorners.toArray()); // retrieve the corners from the prev mat // (saves calculating them again) mMOP2fptsSafe.copyTo(mMOP2fptsPrev); // and save this corners for next time through mMOP2fptsThis.copyTo(mMOP2fptsSafe); } /* * Parameters: * prevImg first 8-bit input image * nextImg second input image * prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers. * nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input. * status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0. * err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases). */ Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr); if (!mMOBStatus.empty()) { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus [x] == 1) { Point pt = cornersThis [x]; Point pt2 = cornersPrev [x]; Core.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1); Core.line(rgbaMat, pt, pt2, colorRed, iLineThickness); } } } Utils.matToTexture2D(rgbaMat, texture, colors); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; } } void OnDisable() { webCamTexture.Stop(); } void OnGUI() { float screenScale = Screen.width / 240.0f; Matrix4x4 scaledMatrix = Matrix4x4.Scale(new Vector3(screenScale, screenScale, screenScale)); GUI.matrix = scaledMatrix; GUILayout.BeginVertical(); if (GUILayout.Button("back")) { Application.LoadLevel("OpenCVForUnitySample"); } if (GUILayout.Button("change camera")) { isFrontFacing = !isFrontFacing; StartCoroutine(init()); } GUILayout.EndVertical(); } } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.isPlaying() && webCamTextureToMatHelper.didUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (mMOP2fptsPrev.rows() == 0) { // first time through the loop so we need prev and this mats // plus prev points // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // copy that to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get prev corners Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsPrev.fromArray(MOPcorners.toArray()); // get safe copy of this corners mMOP2fptsPrev.copyTo(mMOP2fptsSafe); } else { // we've been through before so // this mat is valid. Copy it to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // get the corners for this mat Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsThis.fromArray(MOPcorners.toArray()); // retrieve the corners from the prev mat // (saves calculating them again) mMOP2fptsSafe.copyTo(mMOP2fptsPrev); // and save this corners for next time through mMOP2fptsThis.copyTo(mMOP2fptsSafe); } /* * Parameters: * prevImg first 8-bit input image * nextImg second input image * prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers. * nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input. * status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0. * err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases). */ Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr); if (!mMOBStatus.empty()) { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus [x] == 1) { Point pt = cornersThis [x]; Point pt2 = cornersPrev [x]; Imgproc.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1); Imgproc.line(rgbaMat, pt, pt2, colorRed, iLineThickness); } } } // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (!initDone) { return; } if (screenOrientation != Screen.orientation) { screenOrientation = Screen.orientation; updateLayout(); } #if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1) if (webCamTexture.width > 16 && webCamTexture.height > 16) { #else if (webCamTexture.didUpdateThisFrame) { #endif Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors); //flip to correct direction. if (webCamDevice.isFrontFacing) { if (webCamTexture.videoRotationAngle == 0) { Core.flip(rgbaMat, rgbaMat, 1); } else if (webCamTexture.videoRotationAngle == 90) { Core.flip(rgbaMat, rgbaMat, 0); } if (webCamTexture.videoRotationAngle == 180) { Core.flip(rgbaMat, rgbaMat, 0); } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, 1); } } else { if (webCamTexture.videoRotationAngle == 180) { Core.flip(rgbaMat, rgbaMat, -1); } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, -1); } } if (mMOP2fptsPrev.rows() == 0) { // first time through the loop so we need prev and this mats // plus prev points // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // copy that to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get prev corners Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsPrev.fromArray(MOPcorners.toArray()); // get safe copy of this corners mMOP2fptsPrev.copyTo(mMOP2fptsSafe); } else { // we've been through before so // this mat is valid. Copy it to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // get the corners for this mat Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsThis.fromArray(MOPcorners.toArray()); // retrieve the corners from the prev mat // (saves calculating them again) mMOP2fptsSafe.copyTo(mMOP2fptsPrev); // and save this corners for next time through mMOP2fptsThis.copyTo(mMOP2fptsSafe); } /* * Parameters: * prevImg first 8-bit input image * nextImg second input image * prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers. * nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input. * status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0. * err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases). */ Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr); if (!mMOBStatus.empty()) { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus [x] == 1) { Point pt = cornersThis [x]; Point pt2 = cornersPrev [x]; Core.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1); Core.line(rgbaMat, pt, pt2, colorRed, iLineThickness); } } } Utils.matToTexture2D(rgbaMat, texture, colors); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; } } void OnDisable() { webCamTexture.Stop(); }
//Optical flow IEnumerator OpticalFlow() { Scalar tempHue; Scalar tempSpeed; int iCountTrackedPoints = 0; int vecCount = 0; if (mMOP2fptsPrev.rows() == 0) { // first time through the loop so we need prev and this mats Imgproc.cvtColor(openCVCreateMat.rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // copy that to prev mat matOpFlowThis.copyTo(matOpFlowPrev); //if (blurImage == true){ //Gaussian filter of the MOG2 images //Imgproc.GaussianBlur(matOpFlowPrev, matOpFlowPrev, kernelSize, sigmaX, sigmaY);//Gauss filter //} // get prev corners Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, qLevel, minDistCorners); //SLIDER input mMOP2fptsPrev.fromArray(MOPcorners.toArray()); // get safe copy of this corners mMOP2fptsPrev.copyTo(mMOP2fptsSafe); } else { // we've been through before so // this mat is valid. Copy it to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get this mat Imgproc.cvtColor(openCVCreateMat.rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); //if (blurImage == true){ //Gaussian filter of the MOG2 images //Imgproc.GaussianBlur(matOpFlowThis, matOpFlowThis, kernelSize, sigmaX, sigmaY);//Gauss filter //} // get the corners for this mat Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, qLevel, minDistCorners); // SLIDER input mMOP2fptsThis.fromArray(MOPcorners.toArray()); // retrieve the corners from the prev mat (saves calculating them again) mMOP2fptsSafe.copyTo(mMOP2fptsPrev); // and save this corners for next time through mMOP2fptsThis.copyTo(mMOP2fptsSafe); } Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr); if (mMOBStatus.rows() > 0) { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; double absX; double absY; //will use for calculation of polar coordiates for (x = 0; x < y; x++) { if (byteStatus [x] == 1) { Point pt = cornersThis [x]; Point pt2 = cornersPrev [x]; //if (pt != pt2) {//I think this IF statement should be removed as pt and pt2 should always be differnt float mySpeed = CalculateSpeedFloat(pt, pt2); absX = pt.x - pt2.x; absY = pt.y - pt2.y; float angle = Mathf.Atan2((float)absX, (float)absY) * Mathf.Rad2Deg; angle = Mathf.RoundToInt(angle); //Get Hue based on Angle tempHue = GetHueColor((int)angle); tempSpeed = GetSpeedColor((int)mySpeed); //Store so we can add tracers if (mySpeed > maxSpeed) //|| CalculateSpeedFloat (pt, pt2) <= 1 { yield return(null); } else { tracerPoints.AddTracersToStorage(pt, pt2, tempHue, tempSpeed, videoPlayer.frame, angle, mySpeed); speedVec = speedVec + mySpeed; angleVec = angleVec + angle; vecCount++; //tracerPoints2.AddTracersToStorage (pt, pt2, tempSpeed, videoPlayer.frame, angle, mySpeed); //ADD STORING SPEEDS TO VECTOR //CSVDataEmail.AddDataTrack (speed,angle.ToString ()); } iCountTrackedPoints++; } } } meanSpeed = (int)(speedVec / vecCount); meanAngle = (int)(angleVec / vecCount); //sTrackingLogger = "Video frame: " + videoPlayer.frame.ToString() + " Points: " + iCountTrackedPoints.ToString() + ""; sTrackingLogger = "Speed: " + meanSpeed.ToString() + " Angle: " + meanAngle.ToString() + ""; textTrackedPoints.text = sTrackingLogger; yield return(null); }
// Update is called once per frame void Update() { if (!IsStarted) { return; } Mat grayMat = webCamTextureToMat.GetMat(); Imgproc.cvtColor(grayMat, grayMat, Imgproc.COLOR_RGBA2GRAY); //Debug.Log("mMOP2fptsPrev.rows() : " + mMOP2fptsPrev.rows().ToString()); //Debug.Log("rgbaMat.rows() : " + rgbaMat.rows().ToString()); //Debug.Log("matOpFlowThis.rows() : " + matOpFlowThis.rows().ToString()); if (mMOP2fptsPrev.rows() == 0) { // first time through the loop so we need prev and this mats // plus prev points // get this mat //rgbaMat.copyTo(matOpFlowThis); grayMat.copyTo(matOpFlowThis); grayMat.copyTo(matOpFlowPrev); //matOpFlowThis = rgbaMat; //matOpFlowPrev = rgbaMat; matOpFlowPrev.empty(); //matOpFlowPrev = new Mat(rgbaMat.size(), rgbaMat.type()); //Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // copy that to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get prev corners Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.1, 100); mMOP2fptsPrev.fromArray(MOPcorners.toArray()); // get safe copy of this corners mMOP2fptsPrev.copyTo(mMOP2fptsSafe); //Debug.Log("opencv optical flow --- 1 "); } else { // we've been through before so // this mat is valid. Copy it to prev mat //rgbaMat.copyTo(matOpFlowThis); //matOpFlowPrev = new Mat(rgbaMat.size(), rgbaMat.type()); matOpFlowThis.copyTo(matOpFlowPrev); //matOpFlowThis = new Mat(rgbaMat.size(), rgbaMat.type()); // get this mat grayMat.copyTo(matOpFlowThis); //matOpFlowThis = rgbaMat; //Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // get the corners for this mat Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.1, 100); mMOP2fptsThis.fromArray(MOPcorners.toArray()); // retrieve the corners from the prev mat // (saves calculating them again) mMOP2fptsSafe.copyTo(mMOP2fptsPrev); // and save this corners for next time through mMOP2fptsThis.copyTo(mMOP2fptsSafe); //Debug.Log("opencv optical flow --- 2 "); } /* * Parameters: * prevImg first 8-bit input image * nextImg second input image * prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers. * nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input. * status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0. * err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases). */ Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr); if (!mMOBStatus.empty()) { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; int num_distance = 0; for (x = 0; x < y; x++) { if (byteStatus[x] == 1) { Point pt = cornersThis[x]; Point pt2 = cornersPrev[x]; Imgproc.circle(grayMat, pt, 5, colorRed, iLineThickness - 1); Imgproc.line(grayMat, pt, pt2, colorRed, iLineThickness); double distance = System.Math.Sqrt(System.Math.Pow((pt2.x - pt.x), 2.0) + System.Math.Pow((pt2.y - pt.y), 2.0)); if (distance > 20) { num_distance++; } //Utilities.Debug("Distance[" + x + "] : " + distance); //Debug.Log("Distance[" + x + "] : " + distance); } } Debug.Log("Num of Distance : " + num_distance); if (num_distance > 0) { Debug.Log("Movement Detected !!"); } } // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //this.GetComponent<CVCore>().Add(0, rgbaMat); Utils.matToTexture2D(grayMat, texture, colors); gameObject.GetComponent <Renderer>().material.mainTexture = texture; }