private void drawPoints()
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++)
{
if (byteStatus[x] == 1)
{
Point pt = cornersThis[x];
Point pt2 = cornersPrev[x];
Core.circle(rgbaMat, pt, 5, colorRed, lineThickness - 1);
Core.line(rgbaMat, pt, pt2, colorRed, lineThickness);
}
}
}
//public RawImage document;
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat mainMat = webCamTextureToMatHelper.GetMat();
if (!selectTarget) //find paper by contours
{
grayMat = new Mat();
// convert texture to matrix
mainMat.copyTo(grayMat);
mainMat = findPaper(mainMat);
// display matrix on the screen
Utils.fastMatToTexture2D(mainMat, texture);
}
else
{ // using optical flow
// set the currentGrayMat mat
currentGrayMat = new Mat(mainMat.rows(), mainMat.cols(), Imgproc.COLOR_RGB2GRAY);
Imgproc.cvtColor(mainMat, currentGrayMat, Imgproc.COLOR_RGBA2GRAY);
if (initOpticalFlow == true) // doing the init setting for optical flow
{
// create 40 points
Point[] points = new Point[40];
// set those points near the corner
// paperCornerMatOfPoint is the corner of the paper
for (int i = 0; i < 4; i++)
{
points[i * 10] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y);
points[i * 10 + 1] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y);
points[i * 10 + 2] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y + 1);
points[i * 10 + 3] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y + 1);
points[i * 10 + 4] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 1);
points[i * 10 + 5] = new Point(paperCornerMatOfPoint.toList()[i].x - 1, paperCornerMatOfPoint.toList()[i].y);
points[i * 10 + 6] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 1);
points[i * 10 + 7] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 2);
points[i * 10 + 8] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 2);
points[i * 10 + 9] = new Point(paperCornerMatOfPoint.toList()[i].x + 2, paperCornerMatOfPoint.toList()[i].y + 2);
}
// make the points closer to the corners (Harris Corner Detection )
//Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40, qualityLevel, minDistance, none, blockSize, false, 0.04);
//Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40,0.05,20);
corners.fromArray(points);
prevFeatures.fromList(corners.toList());
currentFeatures.fromList(corners.toList());
prevGrayMat = currentGrayMat.clone();
// won't go back t again
initOpticalFlow = false;
// not that useful lol
// create random color
// not working now
for (int i = 0; i < maxCorners; i++)
{
color.Add(new Scalar((int)(Random.value * 255), (int)(Random.value * 255),
(int)(Random.value * 255), 255));
}
}
else
{
// Don't want ball move
//currentFeatures.fromArray(prevFeatures.toArray());
// want ball move
prevFeatures.fromArray(currentFeatures.toArray());
// optical flow it will changes the valu of currentFeatures
Video.calcOpticalFlowPyrLK(prevGrayMat, currentGrayMat, prevFeatures, currentFeatures, mMOBStatus, err);
//Debug.Log(st.rows());
// change to points list
List <Point> prevList = prevFeatures.toList(),
nextList = currentFeatures.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++)
{
if (byteStatus[x] == 1)
{
Point pt = nextList[x];
Point pt2 = prevList[x];
Imgproc.circle(mainMat, pt, 10, new Scalar(0, 0, 255), -1);
Imgproc.line(mainMat, pt, pt2, new Scalar(0, 0, 255));
}
}
// draw the data
//for (int i = 0; i < prevList.Count; i++)
//{
// //Imgproc.circle(frame, prevList[i], 5, color[10]);
// Imgproc.circle(mainMat, nextList[i], 10, new Scalar(0, 0, 255), -1);
// Imgproc.line(mainMat, prevList[i], nextList[i], color[20]);
//}
List <List <Point> > cornersFeatures = new List <List <Point> >(40);
cornersFeatures.Add(new List <Point>(10));
// put the corners features data into the list
int tmp = 0;
bool last = true;
for (int i = 0; i < nextList.Count - 1; i++)
{
if (Mathf.Abs((float)(nextList[i].x - nextList[i + 1].x)) < 10 && Mathf.Abs((float)(nextList[i].y - nextList[i + 1].y)) < 10)
{
if (last == true)
{
cornersFeatures[tmp].Add(nextList[i]);
}
else
{
cornersFeatures.Add(new List <Point>(10));
tmp = tmp + 1;
cornersFeatures[tmp].Add(nextList[i]);
}
last = true;
}
else
{
last = false;
}
}
// count corners
int manyCornersFeatures = 0;
for (int i = 0; i < cornersFeatures.Count; i++)
{
Debug.Log(cornersFeatures[i].Count);
if (cornersFeatures[i].Count < 5)
{
cornersFeatures.RemoveAt(i);
}
else
{
manyCornersFeatures++;
}
}
//Debug.Log("Length" + manyCornersFeatures);
// if corners equal 4 then diplay virtual docunment into the frame
// doing the perspective transform
if (manyCornersFeatures == 4)
{
Mat documentMat = new Mat(document.height, document.width, CvType.CV_8UC3);
Utils.texture2DToMat(document, documentMat);
List <Point> srcPoints = new List <Point>();
srcPoints.Add(new Point(0, 0));
srcPoints.Add(new Point(documentMat.cols(), 0));
srcPoints.Add(new Point(documentMat.cols(), documentMat.rows()));
srcPoints.Add(new Point(0, documentMat.rows()));
Mat srcPointsMat = Converters.vector_Point_to_Mat(srcPoints, CvType.CV_32F);
List <Point> dstPoints = new List <Point>()
{
cornersFeatures[0][0], cornersFeatures[1][0], cornersFeatures[2][0], cornersFeatures[3][0]
};
Mat dstPointsMat = Converters.vector_Point_to_Mat(dstPoints, CvType.CV_32F);
//Make perspective transform
Mat m = Imgproc.getPerspectiveTransform(srcPointsMat, dstPointsMat);
Mat warpedMat = new Mat(new Size(), documentMat.type());
Debug.Log((cornersFeatures[1][0].x - cornersFeatures[0][0].x) + " " + (cornersFeatures[2][0].y - cornersFeatures[1][0].y));
Imgproc.warpPerspective(documentMat, warpedMat, m, mainMat.size(), Imgproc.INTER_LINEAR);
//warpedMat.convertTo(warpedMat, CvType.CV_32F);
//warpedMat.convertTo(warpedMat, CvType.CV_8UC3);
warpedMat.convertTo(warpedMat, CvType.CV_8UC3);
// same size as frame
Mat dst = new Mat(mainMat.size(), CvType.CV_8UC3);
//Mat dst = new Mat(frame.size(), CvType.CV_8UC3);
//Mat dst2 = new Mat();
Imgproc.cvtColor(mainMat, dst, Imgproc.COLOR_RGBA2RGB);
//dst.setTo(new Scalar(0, 255, 0));
//currentGrayMat.copyTo(dst);
//dst.convertTo(dst, CvType.CV_8UC3);
//Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA);
Mat img1 = new Mat();
Mat mask = new Mat(mainMat.size(), CvType.CV_8UC1, new Scalar(0));
Imgproc.cvtColor(warpedMat, img1, Imgproc.COLOR_RGB2GRAY);
Imgproc.Canny(img1, img1, 100, 200);
List <MatOfPoint> doc_contours = new List <MatOfPoint>();;
Imgproc.findContours(img1, doc_contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_NONE);
Imgproc.drawContours(mask, doc_contours, -1, new Scalar(255), Core.FILLED);
warpedMat.copyTo(dst, mask);
dst.convertTo(dst, CvType.CV_8UC3);
Debug.Log("dst" + dst.size());
Imgproc.cvtColor(dst, mainMat, Imgproc.COLOR_RGB2RGBA);
// display on the right
Texture2D finalTextue = new Texture2D(dst.width(), dst.height(), TextureFormat.RGB24, false);
Utils.matToTexture2D(dst, finalTextue);
targetRawImage.texture = finalTextue;
}
// current frame to old frame
prevGrayMat = currentGrayMat.clone();
//Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA);
// display matrix on the screen
Utils.fastMatToTexture2D(mainMat, texture);
}
}
}
}
/// <summary>
/// Refines the matches with homography.
/// </summary>
/// <returns><c>true</c>, if matches with homography was refined, <c>false</c> otherwise.</returns>
/// <param name="queryKeypoints">Query keypoints.</param>
/// <param name="trainKeypoints">Train keypoints.</param>
/// <param name="reprojectionThreshold">Reprojection threshold.</param>
/// <param name="matches">Matches.</param>
/// <param name="homography">Homography.</param>
static bool refineMatchesWithHomography
(
MatOfKeyPoint queryKeypoints,
MatOfKeyPoint trainKeypoints,
float reprojectionThreshold,
MatOfDMatch matches,
Mat homography
)
{
//Debug.Log ("matches " + matches.ToString ());
int minNumberMatchesAllowed = 8;
List <KeyPoint> queryKeypointsList = queryKeypoints.toList();
List <KeyPoint> trainKeypointsList = trainKeypoints.toList();
List <DMatch> matchesList = matches.toList();
if (matchesList.Count < minNumberMatchesAllowed)
{
return(false);
}
// Prepare data for cv::findHomography
List <Point> srcPointsList = new List <Point>(matchesList.Count);
List <Point> dstPointsList = new List <Point>(matchesList.Count);
for (int i = 0; i < matchesList.Count; i++)
{
srcPointsList.Add(trainKeypointsList[matchesList[i].trainIdx].pt);
dstPointsList.Add(queryKeypointsList[matchesList[i].queryIdx].pt);
}
// Find homography matrix and get inliers mask
using (MatOfPoint2f srcPoints = new MatOfPoint2f())
using (MatOfPoint2f dstPoints = new MatOfPoint2f())
using (MatOfByte inliersMask = new MatOfByte(new byte[srcPointsList.Count]))
{
srcPoints.fromList(srcPointsList);
dstPoints.fromList(dstPointsList);
//Debug.Log ("srcPoints " + srcPoints.ToString ());
//Debug.Log ("dstPoints " + dstPoints.ToString ());
Calib3d.findHomography(srcPoints,
dstPoints,
Calib3d.FM_RANSAC,
reprojectionThreshold,
inliersMask, 2000, 0.955).copyTo(homography);
if (homography.rows() != 3 || homography.cols() != 3)
{
return(false);
}
//Debug.Log ("homography " + homography.ToString ());
//Debug.Log ("inliersMask " + inliersMask.dump ());
List <byte> inliersMaskList = inliersMask.toList();
List <DMatch> inliers = new List <DMatch>();
for (int i = 0; i < inliersMaskList.Count; i++)
{
if (inliersMaskList[i] == 1)
{
inliers.Add(matchesList[i]);
}
}
matches.fromList(inliers);
//Debug.Log ("matches " + matches.ToString ());
}
return(matchesList.Count > minNumberMatchesAllowed);
}
private IEnumerator init()
{
if (webCamTexture != null)
{
webCamTexture.Stop();
initDone = false;
rgbaMat.Dispose();
matOpFlowThis.Dispose();
matOpFlowPrev.Dispose();
MOPcorners.Dispose();
mMOP2fptsThis.Dispose();
mMOP2fptsPrev.Dispose();
mMOP2fptsSafe.Dispose();
mMOBStatus.Dispose();
mMOFerr.Dispose();
}
// Checks how many and which cameras are available on the device
for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++)
{
if (WebCamTexture.devices [cameraIndex].isFrontFacing == isFrontFacing)
{
Debug.Log(cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing);
webCamDevice = WebCamTexture.devices [cameraIndex];
webCamTexture = new WebCamTexture(webCamDevice.name, width, height);
break;
}
}
if (webCamTexture == null)
{
webCamDevice = WebCamTexture.devices [0];
webCamTexture = new WebCamTexture(webCamDevice.name, width, height);
}
Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
// Starts the camera
webCamTexture.Play();
while (true)
{
//If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/)
#if UNITY_IPHONE && !UNITY_EDITOR
if (webCamTexture.width > 16 && webCamTexture.height > 16)
{
#else
if (webCamTexture.didUpdateThisFrame)
{
#endif
Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
Debug.Log("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing);
colors = new Color32[webCamTexture.width * webCamTexture.height];
rgbaMat = new Mat(webCamTexture.height, webCamTexture.width, CvType.CV_8UC4);
matOpFlowThis = new Mat();
matOpFlowPrev = new Mat();
MOPcorners = new MatOfPoint();
mMOP2fptsThis = new MatOfPoint2f();
mMOP2fptsPrev = new MatOfPoint2f();
mMOP2fptsSafe = new MatOfPoint2f();
mMOBStatus = new MatOfByte();
mMOFerr = new MatOfFloat();
texture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
gameObject.transform.eulerAngles = new Vector3(0, 0, 0);
#if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR
gameObject.transform.eulerAngles = new Vector3(0, 0, -90);
#endif
// gameObject.transform.rotation = gameObject.transform.rotation * Quaternion.AngleAxis (webCamTexture.videoRotationAngle, Vector3.back);
gameObject.transform.localScale = new Vector3(webCamTexture.width, webCamTexture.height, 1);
// bool videoVerticallyMirrored = webCamTexture.videoVerticallyMirrored;
// float scaleX = 1;
// float scaleY = videoVerticallyMirrored ? -1.0f : 1.0f;
// if (webCamTexture.videoRotationAngle == 270)
// scaleY = -1.0f;
// gameObject.transform.localScale = new Vector3 (scaleX * gameObject.transform.localScale.x, scaleY * gameObject.transform.localScale.y, 1);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
#if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR
Camera.main.orthographicSize = webCamTexture.width / 2;
#else
Camera.main.orthographicSize = webCamTexture.height / 2;
#endif
initDone = true;
break;
}
else
{
yield return(0);
}
}
}
// Update is called once per frame
void Update()
{
if (!initDone)
{
return;
}
#if UNITY_IPHONE && !UNITY_EDITOR
if (webCamTexture.width > 16 && webCamTexture.height > 16)
{
#else
if (webCamTexture.didUpdateThisFrame)
{
#endif
Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors);
if (webCamTexture.videoVerticallyMirrored)
{
if (webCamDevice.isFrontFacing)
{
if (webCamTexture.videoRotationAngle == 0)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
else if (webCamTexture.videoRotationAngle == 90)
{
Core.flip(rgbaMat, rgbaMat, 0);
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
}
else
{
if (webCamTexture.videoRotationAngle == 90)
{
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, -1);
}
}
}
else
{
if (webCamDevice.isFrontFacing)
{
if (webCamTexture.videoRotationAngle == 0)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
else if (webCamTexture.videoRotationAngle == 90)
{
Core.flip(rgbaMat, rgbaMat, 0);
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
}
else
{
if (webCamTexture.videoRotationAngle == 90)
{
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, -1);
}
}
}
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
}
/*
* Parameters:
* prevImg first 8-bit input image
* nextImg second input image
* prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
* nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
* status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
* err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (!mMOBStatus.empty())
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++)
{
if (byteStatus [x] == 1)
{
Point pt = cornersThis [x];
Point pt2 = cornersPrev [x];
Core.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1);
Core.line(rgbaMat, pt, pt2, colorRed, iLineThickness);
}
}
}
Utils.matToTexture2D(rgbaMat, texture, colors);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}
}
void OnDisable()
{
webCamTexture.Stop();
}
void OnGUI()
{
float screenScale = Screen.width / 240.0f;
Matrix4x4 scaledMatrix = Matrix4x4.Scale(new Vector3(screenScale, screenScale, screenScale));
GUI.matrix = scaledMatrix;
GUILayout.BeginVertical();
if (GUILayout.Button("back"))
{
Application.LoadLevel("OpenCVForUnitySample");
}
if (GUILayout.Button("change camera"))
{
isFrontFacing = !isFrontFacing;
StartCoroutine(init());
}
GUILayout.EndVertical();
}
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.isPlaying() && webCamTextureToMatHelper.didUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
}
/*
* Parameters:
* prevImg first 8-bit input image
* nextImg second input image
* prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
* nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
* status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
* err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (!mMOBStatus.empty())
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++)
{
if (byteStatus [x] == 1)
{
Point pt = cornersThis [x];
Point pt2 = cornersPrev [x];
Imgproc.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1);
Imgproc.line(rgbaMat, pt, pt2, colorRed, iLineThickness);
}
}
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (!initDone)
{
return;
}
if (screenOrientation != Screen.orientation)
{
screenOrientation = Screen.orientation;
updateLayout();
}
#if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1)
if (webCamTexture.width > 16 && webCamTexture.height > 16)
{
#else
if (webCamTexture.didUpdateThisFrame)
{
#endif
Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors);
//flip to correct direction.
if (webCamDevice.isFrontFacing)
{
if (webCamTexture.videoRotationAngle == 0)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
else if (webCamTexture.videoRotationAngle == 90)
{
Core.flip(rgbaMat, rgbaMat, 0);
}
if (webCamTexture.videoRotationAngle == 180)
{
Core.flip(rgbaMat, rgbaMat, 0);
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, 1);
}
}
else
{
if (webCamTexture.videoRotationAngle == 180)
{
Core.flip(rgbaMat, rgbaMat, -1);
}
else if (webCamTexture.videoRotationAngle == 270)
{
Core.flip(rgbaMat, rgbaMat, -1);
}
}
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
}
/*
* Parameters:
* prevImg first 8-bit input image
* nextImg second input image
* prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
* nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
* status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
* err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (!mMOBStatus.empty())
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++)
{
if (byteStatus [x] == 1)
{
Point pt = cornersThis [x];
Point pt2 = cornersPrev [x];
Core.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1);
Core.line(rgbaMat, pt, pt2, colorRed, iLineThickness);
}
}
}
Utils.matToTexture2D(rgbaMat, texture, colors);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}
}
void OnDisable()
{
webCamTexture.Stop();
}
//Optical flow
IEnumerator OpticalFlow()
{
Scalar tempHue;
Scalar tempSpeed;
int iCountTrackedPoints = 0;
int vecCount = 0;
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
Imgproc.cvtColor(openCVCreateMat.rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
//if (blurImage == true){
//Gaussian filter of the MOG2 images
//Imgproc.GaussianBlur(matOpFlowPrev, matOpFlowPrev, kernelSize, sigmaX, sigmaY);//Gauss filter
//}
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, qLevel, minDistCorners); //SLIDER input
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get this mat
Imgproc.cvtColor(openCVCreateMat.rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
//if (blurImage == true){
//Gaussian filter of the MOG2 images
//Imgproc.GaussianBlur(matOpFlowThis, matOpFlowThis, kernelSize, sigmaX, sigmaY);//Gauss filter
//}
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, qLevel, minDistCorners); // SLIDER input
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
}
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (mMOBStatus.rows() > 0)
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
double absX;
double absY; //will use for calculation of polar coordiates
for (x = 0; x < y; x++)
{
if (byteStatus [x] == 1)
{
Point pt = cornersThis [x];
Point pt2 = cornersPrev [x];
//if (pt != pt2) {//I think this IF statement should be removed as pt and pt2 should always be differnt
float mySpeed = CalculateSpeedFloat(pt, pt2);
absX = pt.x - pt2.x;
absY = pt.y - pt2.y;
float angle = Mathf.Atan2((float)absX, (float)absY) * Mathf.Rad2Deg;
angle = Mathf.RoundToInt(angle);
//Get Hue based on Angle
tempHue = GetHueColor((int)angle);
tempSpeed = GetSpeedColor((int)mySpeed);
//Store so we can add tracers
if (mySpeed > maxSpeed) //|| CalculateSpeedFloat (pt, pt2) <= 1
{
yield return(null);
}
else
{
tracerPoints.AddTracersToStorage(pt, pt2, tempHue, tempSpeed, videoPlayer.frame, angle, mySpeed);
speedVec = speedVec + mySpeed;
angleVec = angleVec + angle;
vecCount++;
//tracerPoints2.AddTracersToStorage (pt, pt2, tempSpeed, videoPlayer.frame, angle, mySpeed);
//ADD STORING SPEEDS TO VECTOR
//CSVDataEmail.AddDataTrack (speed,angle.ToString ());
}
iCountTrackedPoints++;
}
}
}
meanSpeed = (int)(speedVec / vecCount);
meanAngle = (int)(angleVec / vecCount);
//sTrackingLogger = "Video frame: " + videoPlayer.frame.ToString() + " Points: " + iCountTrackedPoints.ToString() + "";
sTrackingLogger = "Speed: " + meanSpeed.ToString() + " Angle: " + meanAngle.ToString() + "";
textTrackedPoints.text = sTrackingLogger;
yield return(null);
}
// Update is called once per frame
void Update()
{
if (!IsStarted)
{
return;
}
Mat grayMat = webCamTextureToMat.GetMat();
Imgproc.cvtColor(grayMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
//Debug.Log("mMOP2fptsPrev.rows() : " + mMOP2fptsPrev.rows().ToString());
//Debug.Log("rgbaMat.rows() : " + rgbaMat.rows().ToString());
//Debug.Log("matOpFlowThis.rows() : " + matOpFlowThis.rows().ToString());
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
//rgbaMat.copyTo(matOpFlowThis);
grayMat.copyTo(matOpFlowThis);
grayMat.copyTo(matOpFlowPrev);
//matOpFlowThis = rgbaMat;
//matOpFlowPrev = rgbaMat;
matOpFlowPrev.empty();
//matOpFlowPrev = new Mat(rgbaMat.size(), rgbaMat.type());
//Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.1, 100);
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
//Debug.Log("opencv optical flow --- 1 ");
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
//rgbaMat.copyTo(matOpFlowThis);
//matOpFlowPrev = new Mat(rgbaMat.size(), rgbaMat.type());
matOpFlowThis.copyTo(matOpFlowPrev);
//matOpFlowThis = new Mat(rgbaMat.size(), rgbaMat.type());
// get this mat
grayMat.copyTo(matOpFlowThis);
//matOpFlowThis = rgbaMat;
//Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.1, 100);
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
//Debug.Log("opencv optical flow --- 2 ");
}
/*
* Parameters:
* prevImg first 8-bit input image
* nextImg second input image
* prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
* nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
* status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
* err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (!mMOBStatus.empty())
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
int num_distance = 0;
for (x = 0; x < y; x++)
{
if (byteStatus[x] == 1)
{
Point pt = cornersThis[x];
Point pt2 = cornersPrev[x];
Imgproc.circle(grayMat, pt, 5, colorRed, iLineThickness - 1);
Imgproc.line(grayMat, pt, pt2, colorRed, iLineThickness);
double distance = System.Math.Sqrt(System.Math.Pow((pt2.x - pt.x), 2.0) + System.Math.Pow((pt2.y - pt.y), 2.0));
if (distance > 20)
{
num_distance++;
}
//Utilities.Debug("Distance[" + x + "] : " + distance);
//Debug.Log("Distance[" + x + "] : " + distance);
}
}
Debug.Log("Num of Distance : " + num_distance);
if (num_distance > 0)
{
Debug.Log("Movement Detected !!");
}
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//this.GetComponent<CVCore>().Add(0, rgbaMat);
Utils.matToTexture2D(grayMat, texture, colors);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
}