private void Update()
{
inputMat = webCamTextureToMatHelper.GetMat();
MatOfKeyPoint camKeyPoints = new MatOfKeyPoint();
Mat camDescriptors = new Mat();
Imgproc.cvtColor(inputMat, grayMat, Imgproc.COLOR_BGR2GRAY);
detector.detect(grayMat, camKeyPoints);
extractor.compute(grayMat, camKeyPoints, camDescriptors);
if (camKeyPoints.toList().Count < 1)
{
return;
}
List <MatOfDMatch> matches = new List <MatOfDMatch>();
matcher.knnMatch(makerDescriptors, camDescriptors, matches, 2);
//-- Filter matches using the Lowe's ratio test
float ratioThresh = 0.75f;
List <DMatch> listOfGoodMatches = new List <DMatch>();
for (int i = 0; i < matches.Count; i++)
{
if (matches[i].rows() > 1)
{
DMatch[] dMatches = matches[i].toArray();
if (dMatches[0].distance < ratioThresh * dMatches[1].distance)
{
listOfGoodMatches.Add(dMatches[0]);
}
}
}
MatOfDMatch goodMatches = new MatOfDMatch();
goodMatches.fromList(listOfGoodMatches);
//-- Draw matches
Mat resultImg = new Mat();
Features2d.drawMatches(makerMat, makerKeyPoints, grayMat, camKeyPoints, goodMatches, resultImg);
//listOfGoodMatches = goodMatches.toList();
////-- Localize the object
//List<Point> obj = new List<Point>();
//List<Point> scene = new List<Point>();
//List<KeyPoint> listOfKeypointsObject = makerKeyPoints.toList();
//List<KeyPoint> listOfKeypointsScene = camKeyPoints.toList();
//for (int i = 0; i < listOfGoodMatches.Count(); i++)
//{
// //-- Get the keypoints from the good matches
// obj.Add(listOfKeypointsObject[listOfGoodMatches[i].queryIdx].pt);
// scene.Add(listOfKeypointsScene[listOfGoodMatches[i].trainIdx].pt);
//}
//MatOfPoint2f objMat = new MatOfPoint2f();
//MatOfPoint2f sceneMat = new MatOfPoint2f();
//objMat.fromList(obj);
//sceneMat.fromList(scene);
//double ransacReprojThreshold = 3.0;
//Mat H = Calib3d.findHomography(objMat, sceneMat, Calib3d.RANSAC, ransacReprojThreshold);
////-- Get the corners from the image_1 ( the object to be "detected" )
//Mat objCorners = new Mat(4, 1, CvType.CV_32FC2);
//Mat sceneCorners = new Mat();
//float[] objCornersData = new float[(int)(objCorners.total() * objCorners.channels())];
//objCorners.get(0, 0, objCornersData);
//objCornersData[0] = 0;
//objCornersData[1] = 0;
//objCornersData[2] = makerMat.cols();
//objCornersData[3] = 0;
//objCornersData[4] = makerMat.cols();
//objCornersData[5] = makerMat.rows();
//objCornersData[6] = 0;
//objCornersData[7] = makerMat.rows();
//objCorners.put(0, 0, objCornersData);
//Core.perspectiveTransform(objCorners, sceneCorners, H);
//byte[] sceneCornersData = new byte[(int)(sceneCorners.total() * sceneCorners.channels())];
//sceneCorners.get(0, 0, sceneCornersData);
////-- Draw lines between the corners (the mapped object in the scene - image_2 )
//Imgproc.line(resultImg, new Point(sceneCornersData[0] + makerMat.cols(), sceneCornersData[1]),
// new Point(sceneCornersData[2] + makerMat.cols(), sceneCornersData[3]), new Scalar(0, 255, 0), 4);
//Imgproc.line(resultImg, new Point(sceneCornersData[2] + makerMat.cols(), sceneCornersData[3]),
// new Point(sceneCornersData[4] + makerMat.cols(), sceneCornersData[5]), new Scalar(0, 255, 0), 4);
//Imgproc.line(resultImg, new Point(sceneCornersData[4] + makerMat.cols(), sceneCornersData[5]),
// new Point(sceneCornersData[6] + makerMat.cols(), sceneCornersData[7]), new Scalar(0, 255, 0), 4);
//Imgproc.line(resultImg, new Point(sceneCornersData[6] + makerMat.cols(), sceneCornersData[7]),
// new Point(sceneCornersData[0] + makerMat.cols(), sceneCornersData[1]), new Scalar(0, 255, 0), 4);
if (!first)
{
texture = new Texture2D(resultImg.cols(), resultImg.rows(), TextureFormat.RGBA32, false);
dstQuad.GetComponent <Renderer>().material.mainTexture = texture;
first = true;
}
Utils.matToTexture2D(resultImg, texture);
}
/// <summary>
/// Raises the web cam texture to mat helper inited event.
/// </summary>
public void OnWebCamTextureToMatHelperInited()
{
Debug.Log("OnWebCamTextureToMatHelperInited");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
colors = new Color32[webCamTextureMat.cols() * webCamTextureMat.rows()];
texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false);
gameObject.transform.localScale = new Vector3(webCamTextureMat.cols(), webCamTextureMat.rows(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float imageScale = 1.0f;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = height / 2;
}
else
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
imageScale = (float)Screen.height / (float)Screen.width;
}
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
//set cameraparam
int max_d = Mathf.Max(webCamTextureMat.rows(), webCamTextureMat.cols());
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, max_d);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, webCamTextureMat.cols() / 2.0f);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, max_d);
camMatrix.put(1, 2, webCamTextureMat.rows() / 2.0f);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(0, 0, 0, 0);
Debug.Log("distCoeffs " + distCoeffs.dump());
//calibration camera
Size imageSize = new Size(webCamTextureMat.cols() * imageScale, webCamTextureMat.rows() * imageScale);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point();
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx [0]);
Debug.Log("fovy " + fovy [0]);
Debug.Log("focalLength " + focalLength [0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio [0]);
//Adjust Unity Camera FOV
if (widthScale < heightScale)
{
ARCamera.fieldOfView = (float)fovy[0];
}
else
{
ARCamera.fieldOfView = (float)fovx[0];
}
MarkerDesign[] markerDesigns = new MarkerDesign[markerSettings.Length];
for (int i = 0; i < markerDesigns.Length; i++)
{
markerDesigns [i] = markerSettings [i].markerDesign;
}
markerDetector = new MarkerDetector(camMatrix, distCoeffs, markerDesigns);
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
}
public Mat GetMat()
{
return(webCamTextureToMatHelper.GetMat());
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
frame = webCamTextureToMatHelper.GetMat();
frame.copyTo(img_orig);
drawing = img_orig.clone();
int lowThreshold = 50;// (int)200;// slider.value;
const int ratio = 1;
const int kernel_size = 3;
Imgproc.cvtColor(img_orig, img_lab, Imgproc.COLOR_BGR2Lab);
double omrSize = img_orig.cols() * img_orig.rows();
Imgproc.cvtColor(img_orig, img_gray, Imgproc.COLOR_RGBA2GRAY);
Imgproc.GaussianBlur(img_gray, img_gray, new Size(15, 15), 1.5, 1.5); //Gaussian blur
Imgproc.erode(img_gray, img_gray, new Mat(), new Point(-1, -1), 1); //Erosion
// Imgproc.dilate(img_gray, img_gray, new Mat(), new Point(-1, -1), 10, 1, new Scalar(10)); //Dilation
Imgproc.Canny(img_gray, img_edges, lowThreshold, lowThreshold * ratio, kernel_size, false);
//Shape detection
List <MatOfPoint> contours = new List <MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(img_edges, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0));
//Texture2D tex = new Texture2D(img_edges.width(), img_edges.height(), TextureFormat.RGB24, false);
//Utils.matToTexture2D(img_edges, tex);
//byte[] bytes1 = tex.EncodeToJPG();
//File.WriteAllBytes("D:/2019/OMR/" + "test213123.png", bytes1);
List <MatOfPoint> hulls = new List <MatOfPoint>();
for (int i = 0; i < contours.Count; i++)
{
MatOfInt hull_temp = new MatOfInt();
Imgproc.convexHull(contours[i], hull_temp);
int[] arrIndex = hull_temp.toArray();
Point[] arrContour = contours[i].toArray();
Point[] arrPoints = new Point[arrIndex.Length];
for (int k = 0; k < arrIndex.Length; k++)
{
arrPoints[k] = arrContour[arrIndex[k]];
}
MatOfPoint temp = new MatOfPoint();
temp.fromArray(arrPoints);
//Filter outliers
if (Imgproc.contourArea(temp) > omrSize / 3 && Imgproc.contourArea(temp) < (omrSize * 4) / 5)
{
hulls.Add(temp);
}
}
List <MatOfPoint2f> hull2f = new List <MatOfPoint2f>();
for (int i = 0; i < hulls.Count; i++)
{
MatOfPoint2f newPoint = new MatOfPoint2f(hulls[i].toArray());
hull2f.Add(newPoint);
}
for (int i = 0; i < hulls.Count; i++)
{
//Approximate polygon
MatOfPoint2f approx = new MatOfPoint2f();
Imgproc.approxPolyDP(hull2f[i], approx, 0.01 * Imgproc.arcLength(hull2f[i], true), true);
List <Point> approx_polygon = approx.toList();
// approx_polygon = Scannerproc.filterPolygon(approx_polygon);
// Debug.Log(approx_polygon.Count);
if (!Scannerproc.isSquare(approx_polygon))
{
continue;
}
else
{
nowRectPoints.Clear();
nowRectPoints.AddRange(approx_polygon);
perspectiveAlign();
}
//Center of mass
int cx = 0,
cy = 0;
for (int k = 0; k < approx_polygon.Count; k++)
{
cx += (int)approx_polygon[k].x;
cy += (int)approx_polygon[k].y;
}
cx /= approx_polygon.Count;
cy /= approx_polygon.Count;
Scannerproc.drawShape(drawing, approx_polygon, new Scalar(0, 255, 0));
}
if (showTextureOnScreen)
{
showCurrentTextureOnScreen();
}
}
}
/// <summary>
/// Raises the web cam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false);
ScreenQuad.GetComponent <Renderer> ().material.mainTexture = texture;
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = webCamTextureMat.width();
float height = webCamTextureMat.height();
float imageSizeScale = 1.0f;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
}
else
{
imageSizeScale = (float)Screen.height / (float)Screen.width;
}
//set cameraparam
int max_d = (int)Mathf.Max(width, height);
double fx = max_d;
double fy = max_d;
double cx = width / 2.0f;
double cy = height / 2.0f;
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(0, 0, 0, 0);
Debug.Log("distCoeffs " + distCoeffs.dump());
//calibration camera
Size imageSize = new Size(width * imageSizeScale, height * imageSizeScale);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point(0, 0);
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx [0]);
Debug.Log("fovy " + fovy [0]);
Debug.Log("focalLength " + focalLength [0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio [0]);
//To convert the difference of the FOV value of the OpenCV and Unity.
double fovXScale = (2.0 * Mathf.Atan((float)(imageSize.width / (2.0 * fx)))) / (Mathf.Atan2((float)cx, (float)fx) + Mathf.Atan2((float)(imageSize.width - cx), (float)fx));
double fovYScale = (2.0 * Mathf.Atan((float)(imageSize.height / (2.0 * fy)))) / (Mathf.Atan2((float)cy, (float)fy) + Mathf.Atan2((float)(imageSize.height - cy), (float)fy));
Debug.Log("fovXScale " + fovXScale);
Debug.Log("fovYScale " + fovYScale);
//resize screen Quad
Matrix4x4 p = ARUtils.CalculateProjectionMatrixFromCameraMatrixValues((float)fx, (float)fy, (float)cx, (float)cy, width, height, 0.3f, 1000f);
Vector3 cameraSpacePos = UnProjectVector(p, new Vector3(1.0f, 1.0f, 1.0f));
if (widthScale > heightScale)
{
ScreenQuad.transform.localScale = new Vector3(cameraSpacePos.x * 2f, cameraSpacePos.x * height / width * 2f, 1);
}
else
{
ScreenQuad.transform.localScale = new Vector3(cameraSpacePos.y * width / height * 2f, cameraSpacePos.y * 2f, 1);
}
//create markerDetector
MarkerDesign[] markerDesigns = new MarkerDesign[markerSettings.Length];
for (int i = 0; i < markerDesigns.Length; i++)
{
markerDesigns [i] = markerSettings [i].markerDesign;
}
markerDetector = new MarkerDetector(camMatrix, distCoeffs, markerDesigns);
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
//if WebCamera is frontFaceing,flip Mat.
if (webCamTextureToMatHelper.GetWebCamDevice().isFrontFacing)
{
webCamTextureToMatHelper.flipHorizontal = true;
}
}
/// <summary>
/// Update is called once per frame
/// </summary>
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
// Detect faces on resize image 在调整图像时察觉脸
//detect face rects 发现脸的矩形
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
;
if (detectResult.Count > 0)
{
//detect landmark points 检测具有界标意义的点
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult[0]);
//将points绘制在mat上
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
imagePoints.fromArray(
new Point((points[38].x + points[41].x) / 2, (points[38].y + points[41].y) / 2), //l eye (Interpupillary breadth)
new Point((points[43].x + points[46].x) / 2, (points[43].y + points[46].y) / 2), //r eye (Interpupillary breadth)
new Point(points[30].x, points[30].y), //nose (Nose top)
new Point(points[48].x, points[48].y), //l mouth (Mouth breadth)
new Point(points[54].x, points[54].y), //r mouth (Mouth breadth)
new Point(points[0].x, points[0].y), //l ear (Bitragion breadth)
new Point(points[16].x, points[16].y) //r ear (Bitragion breadth)
);
// Estimate head pose. 估计头部姿势
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
//从3D到2D点的pose中找到一个物体的姿势 rvec是旋转 tvec是平移向量
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_z = tvec.get(2, 0)[0];
if (double.IsNaN(tvec_z) || tvec_z < 0)
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
if (!double.IsNaN(tvec_z) && points.Count == 68)
{
cubismParameterDictionary.Clear();
Calib3d.Rodrigues(rvec, rotMat);
transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0)[0], (float)rotMat.get(0, 1)[0], (float)rotMat.get(0, 2)[0], (float)tvec.get(0, 0)[0]));
transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0)[0], (float)rotMat.get(1, 1)[0], (float)rotMat.get(1, 2)[0], (float)tvec.get(1, 0)[0]));
transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0)[0], (float)rotMat.get(2, 1)[0], (float)rotMat.get(2, 2)[0], (float)tvec.get(2, 0)[0]));
transformationM.SetRow(3, new Vector4(0, 0, 0, 1));
ARM = invertYM * transformationM * invertZM;
Vector3 forward;
forward.x = ARM.m02;
forward.y = ARM.m12;
forward.z = ARM.m22;
Vector3 upwards;
upwards.x = ARM.m01;
upwards.y = ARM.m11;
upwards.z = ARM.m21;
Vector3 angles = Quaternion.LookRotation(forward, upwards).eulerAngles;
float rotateX = angles.x > 180 ? angles.x - 360 : angles.x;
cubismParameterDictionary.Add(ParamAngleY, (float)Math.Round(rotateX));
float rotateY = angles.y > 180 ? angles.y - 360 : angles.y;
cubismParameterDictionary.Add(ParamAngleX, (float)Math.Round(-rotateY) * 2);
float rotateZ = angles.z > 180 ? angles.z - 360 : angles.z;
cubismParameterDictionary.Add(ParamAngleZ, (float)Math.Round(-rotateZ) * 2);
//Debug.("X" + rotateX + "Y" + rotateY + "Z" + rotateZ);
//ParamAngleX.BlendToValue(BlendMode,(float)(Math.Round(-rotateY) * 2));
//ParamAngleY.BlendToValue(BlendMode, (float)Math.Round(rotateX));
//ParamAngleZ.BlendToValue(BlendMode, (float)Math.Round(-rotateZ) * 2);
float eyeOpen_L = Mathf.Clamp(Mathf.Abs(points[43].y - points[47].y) / (Mathf.Abs(points[43].x - points[44].x) * 0.75f), -0.1f, 2.0f);
if (eyeOpen_L >= 0.8f)
{
eyeOpen_L = 1f;
}
else
{
eyeOpen_L = 0;
}
float eyeOpen_R = Mathf.Clamp(Mathf.Abs(points[38].y - points[40].y) / (Mathf.Abs(points[37].x - points[38].x) * 0.75f), -0.1f, 2.0f);
if (eyeOpen_R >= 0.8f)
{
eyeOpen_R = 1f;
}
else
{
eyeOpen_R = 0;
}
// ParamEyeROpen.BlendToValue(BlendMode, eyeOpen_R);
cubismParameterDictionary.Add(ParamEyeROpen, eyeOpen_R);
// ParamEyeLOpen.BlendToValue(BlendMode, eyeOpen_L);
cubismParameterDictionary.Add(ParamEyeLOpen, eyeOpen_L);
// ParamEyeBallX.BlendToValue(BlendMode, (float)rotateY / 30f);
cubismParameterDictionary.Add(ParamEyeBallX, rotateY / 30f);
// ParamEyeBallX.BlendToValue(BlendMode, (float)-rotateX / 30f - 0.25f);
cubismParameterDictionary.Add(ParamEyeBallY, (float)-rotateX / 30f - 0.25f);
float RY = Mathf.Abs(points[19].y - points[27].y) / Mathf.Abs(points[27].y - points[29].y);
RY -= 1;
RY *= 4f;
float LY = Mathf.Abs(points[24].y - points[27].y) / Mathf.Abs(points[27].y - points[29].y);
LY -= 1;
LY *= 4f;
// ParamBrowRY.BlendToValue(BlendMode, RY);
cubismParameterDictionary.Add(ParamBrowRY, RY);
// ParamBrowLY.BlendToValue(BlendMode, LY);
cubismParameterDictionary.Add(ParamBrowLY, LY);
float mouthOpen = Mathf.Clamp01(Mathf.Abs(points[62].y - points[66].y) / (Mathf.Abs(points[51].y - points[62].y) + Mathf.Abs(points[66].y - points[57].y)));
if (mouthOpen < 0.6f)
{
mouthOpen = 0;
}
// ParamMouthOpenY.BlendToValue(BlendMode, mouthOpen);
cubismParameterDictionary.Add(ParamMouthOpenY, mouthOpen);
float mouthSize = Mathf.Abs(points[48].x - points[54].x) / (Mathf.Abs(points[31].x - points[35].x));
// ParamMouthForm.BlendToValue(BlendMode, Mathf.Clamp(mouthSize, -1.0f, 1.0f));
cubismParameterDictionary.Add(ParamMouthForm, Mathf.Clamp(mouthSize, -1.0f, 1.0f));
}
}
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
/// <summary>
/// Raises the web cam texture to mat helper inited event.
/// </summary>
public void OnWebCamTextureToMatHelperInited()
{
Debug.Log("OnWebCamTextureToMatHelperInited");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
gameObject.transform.localScale = new Vector3(webCamTextureMat.cols(), webCamTextureMat.rows(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = gameObject.transform.localScale.x;
float height = gameObject.transform.localScale.y;
float imageSizeScale = 1.0f;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = height / 2;
}
else
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
imageSizeScale = (float)Screen.height / (float)Screen.width;
}
//set cameraparam
int max_d = Mathf.Max(webCamTextureMat.cols(), webCamTextureMat.rows());
double fx = max_d;
double fy = max_d;
double cx = webCamTextureMat.cols() / 2.0f;
double cy = webCamTextureMat.rows() / 2.0f;
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(0, 0, 0, 0);
Debug.Log("distCoeffs " + distCoeffs.dump());
//calibration camera
Size imageSize = new Size(webCamTextureMat.cols() * imageSizeScale, webCamTextureMat.rows() * imageSizeScale);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point();
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx [0]);
Debug.Log("fovy " + fovy [0]);
Debug.Log("focalLength " + focalLength [0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio [0]);
//To convert the difference of the FOV value of the OpenCV and Unity.
double fovXScale = (2.0 * Mathf.Atan((float)(imageSize.width / (2.0 * fx)))) / (Mathf.Atan2((float)cx, (float)fx) + Mathf.Atan2((float)(imageSize.width - cx), (float)fx));
double fovYScale = (2.0 * Mathf.Atan((float)(imageSize.height / (2.0 * fy)))) / (Mathf.Atan2((float)cy, (float)fy) + Mathf.Atan2((float)(imageSize.height - cy), (float)fy));
Debug.Log("fovXScale " + fovXScale);
Debug.Log("fovYScale " + fovYScale);
//Adjust Unity Camera FOV https://github.com/opencv/opencv/commit/8ed1945ccd52501f5ab22bdec6aa1f91f1e2cfd4
if (widthScale > heightScale)
{
ARCamera.fieldOfView = (float)(fovx [0] * fovXScale);
}
else
{
ARCamera.fieldOfView = (float)(fovy [0] * fovYScale);
}
MarkerDesign[] markerDesigns = new MarkerDesign[markerSettings.Length];
for (int i = 0; i < markerDesigns.Length; i++)
{
markerDesigns [i] = markerSettings [i].markerDesign;
}
markerDetector = new MarkerDetector(camMatrix, distCoeffs, markerDesigns);
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
//if WebCamera is frontFaceing,flip Mat.
if (webCamTextureToMatHelper.GetWebCamDevice().isFrontFacing)
{
webCamTextureToMatHelper.flipHorizontal = true;
}
StartCoroutine(resetCameraStatus());
}
/// <summary>
/// Raises the webcam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
gameObject.transform.localScale = new Vector3(webCamTextureMat.cols(), webCamTextureMat.rows(), 1);
Utilities.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
Utilities.Log("Webcam width " + webCamTextureMat.cols() + ".Webcam height " + webCamTextureMat.rows());
float width = webCamTextureMat.width();
float height = webCamTextureMat.height();
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
var rat = (float)webCamTextureMat.cols() / (float)webCamTextureMat.rows();
nW_goc = webCamTextureMat.cols();
nW_goc = 200;
nH_goc = (int)((float)nW_goc / (float)rat);
nW = nW_goc;
nH = (int)(nH_goc * 0.815f);
rat = (float)nW / (float)nH;
if (rawmainDL != null && debug == true)
{
rawmainDL.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL != null && debug == true)
{
rawdebugDL.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL2 != null && debug == true)
{
rawdebugDL2.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL3 != null && debug == true)
{
rawdebugDL3.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL4 != null && debug == true)
{
rawdebugDL4.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL5 != null && debug == true)
{
rawdebugDL5.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL6 != null && debug == true)
{
rawdebugDL6.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL7 != null && debug == true)
{
rawdebugDL7.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawdebugDL8 != null && debug == true)
{
rawdebugDL8.GetComponent <AspectRatioFitter>().aspectRatio = rat;
}
if (rawmainDL != null && debug == true)
{
texture = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL != null && debug == true)
{
dbText1 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL2 != null && debug == true)
{
dbText2 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL3 != null && debug == true)
{
dbText3 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL4 != null && debug == true)
{
dbText4 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL5 != null && debug == true)
{
dbText5 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL6 != null && debug == true)
{
dbText6 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL7 != null && debug == true)
{
dbText7 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
if (rawdebugDL8 != null && debug == true)
{
dbText8 = new Texture2D(nW, nH, TextureFormat.RGBA32, false);
}
rgbMat = new Mat(nW, nH, CvType.CV_8UC3);
if (rawmainDL != null && debug == true)
{
rawmainDL.texture = texture;
}
if (rawdebugDL != null && debug == true)
{
rawdebugDL.texture = dbText1;
}
if (rawdebugDL2 != null && debug == true)
{
rawdebugDL2.texture = dbText2;
}
if (rawdebugDL3 != null && debug == true)
{
rawdebugDL3.texture = dbText3;
}
if (rawdebugDL4 != null && debug == true)
{
rawdebugDL4.texture = dbText4;
}
if (rawdebugDL5 != null && debug == true)
{
rawdebugDL5.texture = dbText5;
}
if (rawdebugDL6 != null && debug == true)
{
rawdebugDL6.texture = dbText6;
}
if (rawdebugDL7 != null && debug == true)
{
rawdebugDL7.texture = dbText7;
}
if (rawdebugDL8 != null && debug == true)
{
rawdebugDL8.texture = dbText8;
}
thresholdMat = new Mat();
thresholdMat2 = new Mat();
hsvMat = new Mat();
hsvMat2 = new Mat();
hsvMat3 = new Mat();
hsvMat4 = new Mat();
rgbaMat = new Mat(1, 1, CvType.CV_8UC3);
rgbMat2 = new Mat(1, 1, CvType.CV_8UC3);
rgbMat3 = new Mat(1, 1, CvType.CV_8UC3);
rgbMat4 = new Mat(1, 1, CvType.CV_8UC3);
rgbMat2copy = new Mat(1, 1, CvType.CV_8UC3);
thresholdMatArr = new Mat[7];
Observable.FromMicroCoroutine(worker).Subscribe();
}
