public static Mat perspectiveAlign(Mat src, Mat dst)
{
//Parameters for perspective wrapping
//MatOfPoint2f srcQuad = new MatOfPoint2f(new Point(-80, 150), new Point(570, 150), new Point(55, 440), new Point(425, 440));
//MatOfPoint2f dstQuad = new MatOfPoint2f(new Point(0, 0), new Point(480, 0), new Point(0, 370), new Point(480, 370));
// Texture2D tex1 = new Texture2D(src.width(), src.height(), TextureFormat.RGB24, false);
// Utils.matToTexture2D(src, tex1);
// byte[] bytes1 = tex1.EncodeToJPG();
// System.IO.File.WriteAllBytes("D:/Patchs/" + yy + ".jpg", bytes1);
// yy++;
// FOR THE NEW BASE
// MatOfPoint2f srcQuad = new MatOfPoint2f(new Point(-20, 185), new Point(460, 185), new Point(90, 500), new Point(360, 500));
// MatOfPoint2f dstQuad = new MatOfPoint2f(new Point(0, 0), new Point(480, 0), new Point(0, 640), new Point(480, 640));
MatOfPoint2f srcQuad = new MatOfPoint2f(new Point(-60, 80), new Point(540, 80), new Point(45, 370), new Point(435, 370));
MatOfPoint2f dstQuad = new MatOfPoint2f(new Point(0, 0), new Point(480, 0), new Point(0, 380), new Point(480, 380));
Mat M = Imgproc.getPerspectiveTransform(srcQuad, dstQuad);
Imgproc.warpPerspective(src, dst, M, new Size(src.width(), src.height()));
Imgproc.resize(dst, dst, new Size(src.width(), 550));
return(dst);
}
public void ProcessFinger(Mat rgbaImage)
{
Imgproc.pyrDown(rgbaImage, mPyrDownMat);
Imgproc.pyrDown(mPyrDownMat, mPyrDownMat);
Imgproc.cvtColor(mPyrDownMat, mHsvMat, Imgproc.COLOR_RGB2HSV_FULL);
Imgproc.cvtColor(mPyrDownMat, mRGBAMat, Imgproc.COLOR_RGB2RGBA);
Imgproc.cvtColor(mPyrDownMat, mYCrCbMat, Imgproc.COLOR_RGB2YCrCb);
Core.inRange(mHsvMat, fLowerBoundHSV, fUpperBoundHSV, fMaskHSV);
fMask = fMaskHSV;
Imgproc.dilate(fMask, fDilatedMask, new Mat());
List <MatOfPoint> contoursFinger = new List <MatOfPoint>();
Imgproc.findContours(fDilatedMask, contoursFinger, fHierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
if (contoursFinger.Count == 0)
{
FingerContour = null;
return;
}
// Find max contour area
double maxArea = 0;
MatOfPoint biggestContour = null;
foreach (MatOfPoint each in contoursFinger)
{
MatOfPoint wrapper = each;
double area = Imgproc.contourArea(wrapper);
if (area > maxArea)
{
maxArea = area;
biggestContour = each;
}
}
if (maxArea < 130)
{
FingerContour = null;
return;
}
//Debug.Log("Finger contour area" + maxArea.ToString());
MatOfPoint2f contours_res2f = new MatOfPoint2f();
MatOfPoint2f biggestContour2f = new MatOfPoint2f(biggestContour.toArray());
Imgproc.approxPolyDP(biggestContour2f, contours_res2f, 3, true);
FingerContour = new MatOfPoint(contours_res2f.toArray());
contours_res2f.Dispose();
biggestContour2f.Dispose();
if (Imgproc.contourArea(FingerContour) > mMinContourArea * maxArea)
{
Core.multiply(FingerContour, new Scalar(4, 4), FingerContour);
}
}
public static bool isHeart(List <Point> shape)
{
//Check number of vertices
if (shape.Count < 20)
{
return(false);
}
MatOfPoint shape_area = new MatOfPoint();
shape_area.fromList(shape);
MatOfPoint2f shape_area2f = new MatOfPoint2f(shape_area.toArray());
// if (Imgproc.contourArea(shape_area) > 6000)
// return false;
double area = Imgproc.contourArea(shape_area);
double perim = Imgproc.arcLength(shape_area2f, true);
double ratio = area / perim;
if (ratio < 18 || ratio > 23)
{
return(false);
}
for (int i = 1; i < shape.Count; i++)
{
if (distanceTwoPoints(shape[i - 1], shape[i]) > 20)
{
return(true);
}
}
return(false);
}
// Start is called before the first frame update
void Start()
{
var devices = WebCamTexture.devices;
webCamDevice = devices[0];
webCamTexture = new WebCamTexture(webCamDevice.name);
webCamTexture.Play();
while (true)
{
if (webCamTexture.didUpdateThisFrame)
{
OnInited();
break;
}
}
List <Point> points = new List <Point>();
points.Add(new Point(0, 260));
points.Add(new Point(880, 260));
points.Add(new Point(880, 0));
points.Add(new Point(0, 0));
MatOfPoint2f pts2 = new MatOfPoint2f();
pts2.fromList(points);
double[] p1_1 = pts2.get(0, 0);
print(string.Join(" ", p1_1));
double[] p3_1 = pts2.get(2, 0);
print(string.Join(" ", p3_1));
double[] p2_1 = pts2.get(1, 0);
print(string.Join(" ", p2_1));
}
// Use this for initialization
void Start()
{
displayCameraPreviewToggle.isOn = displayCameraPreview;
useSeparateDetectionToggle.isOn = useSeparateDetection;
displayAxesToggle.isOn = displayAxes;
displayHeadToggle.isOn = displayHead;
displayEffectsToggle.isOn = displayEffects;
imageOptimizationHelper = gameObject.GetComponent <ImageOptimizationHelper> ();
webCamTextureToMatHelper = gameObject.GetComponent <HololensCameraStreamToMatHelper> ();
#if NETFX_CORE
webCamTextureToMatHelper.frameMatAcquired += OnFrameMatAcquired;
#endif
webCamTextureToMatHelper.Initialize();
rectangleTracker = new RectangleTracker();
// faceLandmarkDetector = new FaceLandmarkDetector (DlibFaceLandmarkDetector.Utils.getFilePath ("sp_human_face_68.dat"));
faceLandmarkDetector = new FaceLandmarkDetector(DlibFaceLandmarkDetector.Utils.getFilePath("sp_human_face_68_for_mobile.dat"));
// The coordinates of the detection object on the real world space connected with the pixel coordinates.(mm)
objectPoints = new MatOfPoint3f(
new Point3(-34, 90, 83), //l eye (Interpupillary breadth)
new Point3(34, 90, 83), //r eye (Interpupillary breadth)
new Point3(0.0, 50, 120), //nose (Nose top)
new Point3(-26, 15, 83), //l mouse (Mouth breadth)
new Point3(26, 15, 83), //r mouse (Mouth breadth)
new Point3(-79, 90, 0.0), //l ear (Bitragion breadth)
new Point3(79, 90, 0.0) //r ear (Bitragion breadth)
);
imagePoints = new MatOfPoint2f();
rotMat = new Mat(3, 3, CvType.CV_64FC1);
}
// Use this for initialization
void Start()
{
//set 3d face object points.
objectPoints = new MatOfPoint3f(new Point3(-31, 72, 86), //l eye
new Point3(31, 72, 86), //r eye
new Point3(0, 40, 114), //nose
new Point3(-20, 15, 90), //l mouse
new Point3(20, 15, 90) //r mouse
// ,
// new Point3 (-70, 60, -9),//l ear
// new Point3 (70, 60, -9)//r ear
);
imagePoints = new MatOfPoint2f();
rvec = new Mat();
tvec = new Mat();
rotM = new Mat(3, 3, CvType.CV_64FC1);
//initialize FaceTracker
faceTracker = new FaceTracker(Utils.getFilePath("tracker_model.json"));
//initialize FaceTrackerParams
faceTrackerParams = new FaceTrackerParams();
webCamTextureToMatHelper = gameObject.GetComponent <WebCamTextureToMatHelper> ();
webCamTextureToMatHelper.Init();
autoResetModeToggle.isOn = autoResetMode;
}
public bool GetPosition(Mat frame, bool isKeyboardFound)
{
Mat frameProc = new Mat(); //frame.rows(), frame.cols(), CvType.CV_16UC3
Mat frameMask = new Mat();
Mat hierarchy = new Mat();
Imgproc.cvtColor(frame, frameProc, Imgproc.COLOR_BGR2HSV);
Scalar lowerB = new Scalar(HueLower, SatLower, ValLower);
Scalar upperB = new Scalar(HueUpper, SatUpper, ValUpper);
Core.inRange(frameProc, lowerB, upperB, frameMask);
Core.bitwise_and(frame, frame, frameProc, frameMask);
//Imgproc.bilateralFilter(frameProc, frameProc, 9, 50, 100);
Imgproc.morphologyEx(frameProc, frameProc, 2, Mat.ones(5, 5, CvType.CV_8U)); //
Imgproc.dilate(frameProc, frameProc, Mat.ones(5, 5, CvType.CV_8U)); //Mat.ones(5, 5, CvType.CV_8U), anchor: new Point(-1, -1), iteration:2
Imgproc.cvtColor(frameProc, frameProc, Imgproc.COLOR_BGR2GRAY);
List <MatOfPoint> contoursList = new List <MatOfPoint>();
Imgproc.findContours(frameProc, contoursList, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE);
int count = 0;
foreach (MatOfPoint contour in contoursList)
{
MatOfPoint2f approx = new MatOfPoint2f();
MatOfPoint2f contourf = new MatOfPoint2f(contour.toArray());
Imgproc.approxPolyDP(contourf, approx, 0.01 * Imgproc.arcLength(contourf, true), true);
//print(approx.dump());
if (approx.rows() == 4 && Imgproc.contourArea(contour) >= min_area)
{
count++;
if (count >= 2)
{
continue;
}
else
{
OpenCVForUnity.CoreModule.Rect track_win = Imgproc.boundingRect(approx);
TrackWindow = new int[] { track_win.x, track_win.y, track_win.width, track_win.height };
if (frame.height() - 5 < TrackWindow[0] + TrackWindow[2] &&
TrackWindow[0] + TrackWindow[2] <= frame.height() ||
0 <= TrackWindow[0] && TrackWindow[0] < 5 ||
frame.width() - 5 < TrackWindow[1] + TrackWindow[3] &&
TrackWindow[1] + TrackWindow[3] <= frame.width() ||
0 <= TrackWindow[1] && TrackWindow[1] < 5)
{
continue;
}
else
{
Approx = approx;
Contour = contour;
return(isKeyboardFound = true);
}
}
}
}
return(isKeyboardFound = false);
}
private Scalar colorRed = new Scalar(255, 0, 0, 125); // Red color
void Start()
{
for (int i = 0; i < WebCamTexture.devices.Length; i++)
{
Debug.Log(WebCamTexture.devices[i].name);
}
mCamera = new WebCamTexture();
matOpFlowThis = new Mat();
matOpFlowPrev = new Mat();
MOPcorners = new MatOfPoint();
mMOP2fptsThis = new MatOfPoint2f();
mMOP2fptsPrev = new MatOfPoint2f();
mMOP2fptsSafe = new MatOfPoint2f();
mMOBStatus = new MatOfByte();
mMOFerr = new MatOfFloat();
mCamera.Play();
rgbaMat = new Mat(mCamera.height, mCamera.width, CvType.CV_8UC4);
texture = new Texture2D(mCamera.width, mCamera.height, TextureFormat.RGBA32, false);
colors = new Color32[mCamera.width * mCamera.height];
GetComponent <Renderer>().material.mainTexture = texture;
}
public virtual void Draw(Mat src, Mat dst)
{
var points = new MatOfPoint2f();
var patternSize = new Size((int)SizeX, (int)SizeY);
var found = false;
switch (boardType)
{
case BoardType.ChessBoard:
found = Calib3d.findChessboardCorners(src, patternSize, points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE);
break;
case BoardType.CirclesGrid:
found = Calib3d.findCirclesGrid(src, patternSize, points, Calib3d.CALIB_CB_SYMMETRIC_GRID);
break;
case BoardType.AsymmetricCirclesGrid:
found = Calib3d.findCirclesGrid(src, patternSize, points, Calib3d.CALIB_CB_ASYMMETRIC_GRID);
break;
}
if (found)
{
if (boardType == BoardType.ChessBoard)
{
Imgproc.cornerSubPix(src, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1));
}
Calib3d.drawChessboardCorners(dst, patternSize, points, found);
}
}
public void FishEyeCalibrate()
{
var patternSize = new Size(10, 7);
using (var image = Image("calibration/00.jpg"))
using (var corners = new MatOfPoint2f())
{
Cv2.FindChessboardCorners(image, patternSize, corners);
var objectPointsArray = Create3DChessboardCorners(patternSize, 1.0f).ToArray();
var imagePointsArray = corners.ToArray();
using (var objectPoints = MatOfPoint3f.FromArray(objectPointsArray))
using (var imagePoints = MatOfPoint2f.FromArray(imagePointsArray))
using (var cameraMatrix = new MatOfDouble(Mat.Eye(3, 3, MatType.CV_64FC1)))
using (var distCoeffs = new MatOfDouble())
{
var rms = Cv2.FishEye.Calibrate(new[] { objectPoints }, new[] { imagePoints }, image.Size(), cameraMatrix,
distCoeffs, out var rotationVectors, out var translationVectors,
FishEyeCalibrationFlags.None);
var distCoeffValues = distCoeffs.ToArray();
Assert.Equal(55.15, rms, 2);
Assert.Contains(distCoeffValues, d => Math.Abs(d) > 1e-20);
Assert.NotEmpty(rotationVectors);
Assert.NotEmpty(translationVectors);
}
}
}
private void FindHomographyPoints(out MatOfPoint2f matDst, out MatOfPoint2f matObj)
{
var corner1ScreenPoint = Cam.WorldToScreenPoint(Corner1.position);
var corner2ScreenPoint = Cam.WorldToScreenPoint(Corner2.position);
var corner3ScreenPoint = Cam.WorldToScreenPoint(Corner3.position);
var corner4ScreenPoint = Cam.WorldToScreenPoint(Corner4.position);
corner1ScreenPoint.y = Cam.pixelHeight - corner1ScreenPoint.y;
corner2ScreenPoint.y = Cam.pixelHeight - corner2ScreenPoint.y;
corner3ScreenPoint.y = Cam.pixelHeight - corner3ScreenPoint.y;
corner4ScreenPoint.y = Cam.pixelHeight - corner4ScreenPoint.y;
var srcPoints = new List <Point>
{
new Point(corner2ScreenPoint.x, corner2ScreenPoint.y),
new Point(corner1ScreenPoint.x, corner1ScreenPoint.y),
new Point(corner4ScreenPoint.x, corner4ScreenPoint.y),
new Point(corner3ScreenPoint.x, corner3ScreenPoint.y),
};
var dstPoints = new List <Point>
{
new Point(0, _drawingPlaceMat.height()),
new Point(_drawingPlaceMat.width(), _drawingPlaceMat.height()),
new Point(_drawingPlaceMat.width(), 0),
new Point(0, 0),
};
matObj = new MatOfPoint2f(srcPoints.ToArray());
matDst = new MatOfPoint2f(dstPoints.ToArray());
}
private void Run()
{
//set 3d face object points.
objectPoints = new MatOfPoint3f(new Point3(-31, 72, 86), //l eye
new Point3(31, 72, 86), //r eye
new Point3(0, 40, 114), //nose
new Point3(-20, 15, 90), //l mouse
new Point3(20, 15, 90) //r mouse
// ,
// new Point3 (-70, 60, -9),//l ear
// new Point3 (70, 60, -9)//r ear
);
imagePoints = new MatOfPoint2f();
rvec = new Mat();
tvec = new Mat();
rotM = new Mat(3, 3, CvType.CV_64FC1);
//initialize FaceTracker
faceTracker = new FaceTracker(tracker_model_json_filepath);
//initialize FaceTrackerParams
faceTrackerParams = new FaceTrackerParams();
cascade = new CascadeClassifier();
cascade.load(haarcascade_frontalface_alt_xml_filepath);
// if (cascade.empty())
// {
// Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
webCamTextureToMatHelper.Initialize();
}
// Use this for initialization
void Start()
{
// AppControl.control.Load();
roiPointList = new List <Point> ();
termination = new TermCriteria(TermCriteria.EPS | TermCriteria.COUNT, 10, 1);
webCamTextureToMatHelper = gameObject.GetComponent <WebCamTextureToMatHelper> ();
webCamTextureToMatHelper.Init(OnWebCamTextureToMatHelperInited, OnWebCamTextureToMatHelperDisposed);
// Calibration variables
patternsize = new Size(9, 6); // (<param1>, <param2>) : new size with width = <param1> and height = <param2>
squaresize = 22f;
pointbuf = new MatOfPoint2f(); //this will be filled by the detected corners of findChessboardCorners
objectpointbuf = new MatOfPoint3f(); //this will be filled by the detected corners of findChessboardCorners
imagePoints = new List <Mat>();
objectPoints = new List <Mat>();
calibrationComplete = false;
imageSize = new Size(480, 640);
cameraMatrix = new Mat();
distCoeffs = new Mat();
rvecs = new List <Mat>();
tvecs = new List <Mat>();
// Set the "lastInterval" to the system time
lastInterval = Time.realtimeSinceStartup;
}
public void BuildPatternFromImage(Mat image, Pattern pattern)
{
// Store original image in pattern structure
pattern.size = new Size(image.Cols, image.Rows);
pattern.frame = image.Clone();
GetGray(image, pattern.grayImg);
// Build 2d and 3d contours (3d contour lie in XY plane since it's planar)
List <Point2f> points2dList = new List <Point2f>(4);
List <Point3f> points3dList = new List <Point3f>(4);
// Image dimensions
float w = image.Cols;
float h = image.Rows;
// Normalized dimensions:
points2dList.Add(new Point2f(0, 0));
points2dList.Add(new Point2f(w, 0));
points2dList.Add(new Point2f(w, h));
points2dList.Add(new Point2f(0, h));
pattern.points2d = MatOfPoint2f.FromArray(points2dList);
points3dList.Add(new Point3f(-0.5f, -0.5f, 0));
points3dList.Add(new Point3f(+0.5f, -0.5f, 0));
points3dList.Add(new Point3f(+0.5f, +0.5f, 0));
points3dList.Add(new Point3f(-0.5f, +0.5f, 0));
pattern.points3d = MatOfPoint3f.FromArray(points3dList);
ExtractFeatures(pattern.grayImg, ref pattern.keypoints, pattern.descriptors);
Train(pattern);
}
private void GetCubies(List <MatOfPoint> contours, Mat imgMat, int index, List <Cubies> cubies)
{
MatOfPoint2f matOfPoint2f = new MatOfPoint2f();
MatOfPoint2f approxCurve = new MatOfPoint2f();
MatOfPoint approx = new MatOfPoint();
foreach (var contour in contours)
{
matOfPoint2f.fromList(contour.toList());
Imgproc.approxPolyDP(matOfPoint2f, approxCurve, 0.1 * Imgproc.arcLength(matOfPoint2f, true), true);
try
{
approxCurve.convertTo(approx, CvType.CV_32S);
OpenCVForUnity.Rect rect = Imgproc.boundingRect(approx);
if (approx.total() == 4)
{
cubies.Add(new Cubies(rect.x, rect.y, colorsList[index]));
Imgproc.rectangle(imgMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 40, 150), 2);
}
}
catch (ArgumentOutOfRangeException e) { }
}
print("Number of cubies: " + cubies.Count);
}
public static void TransformTv()
{
Mat img = Cv2.ImRead(@"C:\prog\consoleHomography\ConsoleApp1\ConsoleApp1\images\input\in.jpg");
Mat imgOut = new Mat();
Point2f[] source =
{
new Point2f(62f, 59f),
new Point2f(420f, 112f),
new Point2f(425f, 300f),
new Point2f(50f, 265f)
};
MatOfPoint2f src = MatOfPoint2f.FromArray(source);
Point2f[] destination =
{
new Point2f(0f, 0f),
new Point2f(img.Width, 0f),
new Point2f(img.Width, img.Height),
new Point2f(0f, img.Height)
};
MatOfPoint2f dest = MatOfPoint2f.FromArray(destination);
Mat m = Cv2.FindHomography(src, dest);
Cv2.WarpPerspective(img, imgOut, m, img.Size());
string imgName = $"{DateTime.Now:dd_MM_yyyy_HH_mm_ss}_out.jpg";
Cv2.ImWrite(@"C:\prog\consoleHomography\ConsoleApp1\ConsoleApp1\images\output\" + imgName, imgOut);
}
void HomographyTransform(int i)
{
// Init homography result Mat
homoMat_array[i] = new Mat(480, 640, CvType.CV_8UC1);
// Init regular point array
reg_point_array[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
reg_point_array[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
reg_point_array[2] = new Point(0.0, 0.0);
reg_point_array[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
// Extract face_points corresponding with reg_points
Point[] out_point_array = new Point[4];
for (int j = 0; j < 4; j++) // j :: face point count
{
int src_i = face_index[i, j];
out_point_array[j] = proj_point_array[src_i];
}
MatOfPoint2f regPoints = new MatOfPoint2f(reg_point_array);
MatOfPoint2f outPoints = new MatOfPoint2f(out_point_array);
Mat Homo_Mat = Calib3d.findHomography(regPoints, outPoints);
Imgproc.warpPerspective(rectMat_array[i], homoMat_array[i], Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
}
private void Run()
{
//set 3d face object points.
objectPoints = new MatOfPoint3f(new Point3(-31, 72, 86), //l eye
new Point3(31, 72, 86), //r eye
new Point3(0, 40, 114), //nose
new Point3(-20, 15, 90), //l mouse
new Point3(20, 15, 90), //r mouse
new Point3(-70, 60, -9), //l ear
new Point3(70, 60, -9) //r ear
);
imagePoints = new MatOfPoint2f();
rvec = new Mat();
tvec = new Mat();
rotM = new Mat(3, 3, CvType.CV_64FC1);
//initialize FaceTracker
faceTracker = new FaceTracker(tracker_model_json_filepath);
//initialize FaceTrackerParams
faceTrackerParams = new FaceTrackerParams();
cascade = new CascadeClassifier();
cascade.load(haarcascade_frontalface_alt_xml_filepath);
//if (cascade.empty())
//{
// Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
//}
#if UNITY_ANDROID && !UNITY_EDITOR
// Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2).
webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true;
#endif
webCamTextureToMatHelper.Initialize();
}
// vector_vector_Point2f
public static void Mat_to_vector_vector_Point2f(Mat m, List <MatOfPoint2f> pts)
{
if (m != null)
{
m.ThrowIfDisposed();
}
if (pts == null)
{
throw new CvException("Output List can't be null");
}
if (m == null)
{
throw new CvException("Input Mat can't be null");
}
List <Mat> mats = new List <Mat>(m.rows());
Mat_to_vector_Mat(m, mats);
foreach (Mat mi in mats)
{
MatOfPoint2f pt = new MatOfPoint2f(mi);
pts.Add(pt);
mi.release();
}
mats.Clear();
}
private double computeReprojectionErrors(List <Mat> objectPoints,
List <Mat> rvecs, List <Mat> tvecs, Mat perViewErrors)
{
MatOfPoint2f cornersProjected = new MatOfPoint2f();
double totalError = 0;
double error;
float[] viewErrors = new float[objectPoints.Count];
MatOfDouble distortionCoefficients = new MatOfDouble(mDistortionCoefficients);
int totalPoints = 0;
for (int i = 0; i < objectPoints.Count; i++)
{
MatOfPoint3f points = new MatOfPoint3f(objectPoints[i]);
Calib3d.Calib3d.ProjectPoints(points, rvecs[i], tvecs[i],
mCameraMatrix, distortionCoefficients, cornersProjected);
error = Core.Core.Norm(mCornersBuffer[i], cornersProjected, Core.Core.NormL2);
int n = objectPoints[i].Rows();
viewErrors[i] = (float)Math.Sqrt(error * error / n);
totalError += error * error;
totalPoints += n;
}
perViewErrors.Create(objectPoints.Count, 1, CvType.Cv32fc1);
perViewErrors.Put(0, 0, viewErrors);
return(Math.Sqrt(totalError / totalPoints));
}
// Use this for initialization
void Start()
{
isUsingSeparateDetectionToggle.isOn = isUsingSeparateDetection;
isShowingAxesToggle.isOn = isShowingAxes;
isShowingHeadToggle.isOn = isShowingHead;
isShowingEffectsToggle.isOn = isShowingEffects;
webCamTextureToMatHelper = gameObject.GetComponent <OptimizationWebCamTextureToMatHelper> ();
webCamTextureToMatHelper.Init();
rectangleTracker = new RectangleTracker();
faceLandmarkDetector = new FaceLandmarkDetector(DlibFaceLandmarkDetector.Utils.getFilePath("shape_predictor_68_face_landmarks.dat"));
// The coordinates of the detection object on the real world space connected with the pixel coordinates.(mm)
objectPoints = new MatOfPoint3f(
new Point3(-31, 72, 86), //l eye (Interpupillary breadth)
new Point3(31, 72, 86), //r eye (Interpupillary breadth)
new Point3(0, 40, 114), //nose (Nose top)
new Point3(-20, 15, 90), //l mouse (Mouth breadth)
new Point3(20, 15, 90), //r mouse (Mouth breadth)
new Point3(-69, 76, -2), //l ear (Bitragion breadth)
new Point3(69, 76, -2) //r ear (Bitragion breadth)
);
imagePoints = new MatOfPoint2f();
rvec = new Mat();
tvec = new Mat();
rotMat = new Mat(3, 3, CvType.CV_64FC1);
}
// Taken from http://docs.opencv.org/3.0-beta/doc/tutorials/calib3d/camera_calibration/camera_calibration.html
float computeReprojectionErrors(List <Mat> objectPoints, List <Mat> imagePoints, List <Mat> rvecs, List <Mat> tvecs, Mat cameraMatrix, Mat distCoeffs, List <float> perViewErrors)
{
MatOfPoint2f imagePoints2 = new MatOfPoint2f();
int i, totalPoints = 0;
float totalErr = 0, err;
int numItems = objectPoints.Count;
for (i = 0; i < numItems; ++i)
{
MatOfPoint3f objectPointsi = new MatOfPoint3f(objectPoints[i]);
MatOfPoint2f imagePointsi = new MatOfPoint2f(imagePoints[i]);
MatOfDouble distCoeffsv2 = new MatOfDouble(distCoeffs);
Calib3d.projectPoints(objectPointsi, rvecs[i], tvecs[i], cameraMatrix, distCoeffsv2, imagePoints2);
err = norml2(imagePointsi, imagePoints2); // difference
Size temp = objectPoints[i].size();
int n = (int)temp.height;
perViewErrors.Add(Mathf.Sqrt(err * err / (n))); // save for this view
totalErr += err * err; // sum it up
totalPoints += (int)n;
}
return(Mathf.Sqrt(totalErr / totalPoints)); // calculate the arithmetical mean
}
void Rectify(ref Point[] face_point_array, int i)
{
Debug.Log("R: Starting");
homoMat_array[i] = new Mat(480, 640, CvType.CV_8UC1);
reg_point_array[0] = new Point(0.0, HOMOGRAPHY_HEIGHT);
reg_point_array[1] = new Point(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT);
reg_point_array[2] = new Point(0.0, 0.0);
reg_point_array[3] = new Point(HOMOGRAPHY_WIDTH, 0.0);
Debug.Log("R: reg_point_array populated");
MatOfPoint2f srcPoints = new MatOfPoint2f(face_point_array);
MatOfPoint2f regPoints = new MatOfPoint2f(reg_point_array);
// Debug.Log("R: src and reg points instantiated");
Debug.LogFormat("Rectify Face Points; {0} \n {1} \n {2} \n {3}",
face_point_array[0], face_point_array[1], face_point_array[2], face_point_array[3]);
// Creating the H Matrix
Mat Homo_Mat = Calib3d.findHomography(srcPoints, regPoints);
Debug.Log("R: H Matrix Instantiated");
Imgproc.warpPerspective(cached_initMat, homoMat_array[i], Homo_Mat, new Size(HOMOGRAPHY_WIDTH, HOMOGRAPHY_HEIGHT));
Debug.Log("R: image rectified");
}
// Use this for initialization
public override void Setup()
{
//set 3d face object points.
objectPoints68 = new MatOfPoint3f(
new Point3(-34, 90, 83), //l eye (Interpupillary breadth)
new Point3(34, 90, 83), //r eye (Interpupillary breadth)
new Point3(0.0, 50, 120), //nose (Nose top)
new Point3(-26, 15, 83), //l mouse (Mouth breadth)
new Point3(26, 15, 83), //r mouse (Mouth breadth)
new Point3(-79, 90, 0.0), //l ear (Bitragion breadth)
new Point3(79, 90, 0.0) //r ear (Bitragion breadth)
);
objectPoints5 = new MatOfPoint3f(
new Point3(-23, 90, 83), //l eye (Inner corner of the eye)
new Point3(23, 90, 83), //r eye (Inner corner of the eye)
new Point3(-50, 90, 80), //l eye (Tail of the eye)
new Point3(50, 90, 80), //r eye (Tail of the eye)
new Point3(0.0, 50, 120) //nose (Nose top)
);
imagePoints = new MatOfPoint2f();
float width = 640;
float height = 480;
//set cameraparam
int max_d = (int)Mathf.Max(width, height);
double fx = max_d;
double fy = max_d;
double cx = width / 2.0f;
double cy = height / 2.0f;
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(0, 0, 0, 0);
Debug.Log("distCoeffs " + distCoeffs.dump());
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
didUpdateHeadRotation = false;
}
void Start()
{
srcMat = Imgcodecs.imread(Application.dataPath + "/Resources/aragaki.jpg", 1);
Imgproc.cvtColor(srcMat, srcMat, Imgproc.COLOR_BGR2RGB);
dstMat = srcMat.clone();
///* 仿射变换
Point srcPoint0 = new Point(0, 0);
Point srcPoint1 = new Point(srcMat.width() - 1, 0);
Point srcPoint2 = new Point(0, srcMat.height() - 1);
MatOfPoint2f srcTri = new MatOfPoint2f(new Point[3] {
srcPoint0, srcPoint1, srcPoint2
});
Point dstPoint0 = new Point(0, srcMat.height() * 0.33d);
Point dstPoint1 = new Point(srcMat.width() * 0.85d, srcMat.height() * 0.25d);
Point dstPoint2 = new Point(srcMat.width() * 0.15d, srcMat.height() * 0.7d);
MatOfPoint2f dstTri = new MatOfPoint2f(new Point[3] {
dstPoint0, dstPoint1, dstPoint2
});
warpMat = Imgproc.getAffineTransform(srcTri, dstTri);
Imgproc.warpAffine(srcMat, dstMat, warpMat, new Size(dstMat.width(), dstMat.height()));
//*/
/* 旋转变换
* //拷贝整个画布
* dstMat.copyTo(srcMat);
* Point center = new Point(srcMat.width() / 2, srcMat.height() / 2);
* double angle = -50.0d;
* double scale = -0.6d;
* rotMat = Imgproc.getRotationMatrix2D(center, angle, scale);
* Imgproc.warpAffine(srcMat, dstMat, rotMat, new Size(dstMat.width(), dstMat.height()));
*/
/*
* //透视变换
* Point srcPoint0 = new Point(0, 0);
* Point srcPoint1 = new Point(srcMat.width() - 1, 0);
* Point srcPoint2 = new Point(0, srcMat.height() - 1);
* Point srcPoint3 = new Point(srcMat.width() - 1, srcMat.height() - 1);
* MatOfPoint2f srcTri = new MatOfPoint2f(new Point[4] { srcPoint0, srcPoint1, srcPoint2, srcPoint3 });
* Point dstPoint0 = new Point(srcMat.width() * 0.05d, srcMat.height() * 0.33d);
* Point dstPoint1 = new Point(srcMat.width() * 0.9d, srcMat.height() * 0.25d);
* Point dstPoint2 = new Point(srcMat.width() * 0.2d, srcMat.height() * 0.7d);
* Point dstPoint3 = new Point(srcMat.width() * 0.8d, srcMat.height() * 0.9d);
* MatOfPoint2f dstTri = new MatOfPoint2f(new Point[4] { dstPoint0, dstPoint1, dstPoint2, dstPoint3 });
* warpMat = Imgproc.getPerspectiveTransform(srcTri, dstTri);
* Imgproc.warpPerspective(srcMat, dstMat, warpMat, new Size(dstMat.width(), dstMat.height()));
*/
Texture2D t2d = new Texture2D(dstMat.width(), dstMat.height());
Utils.matToTexture2D(dstMat, t2d);
Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero);
dstImage.sprite = sp;
dstImage.preserveAspect = true;
}
/// <summary>
/// Add a pair of real space + image space points.
/// Beware that calibration can fail if pattern is not rotated to fade forward, so that z is zero.
/// Also ensure that the point order in the the two point sets are matching.
/// </summary>
/// <param name="patternRealModelSample">Must be measured in millimeters</param>
/// <param name="patternImageSample"></param>
public void AddSample(MatOfPoint3f patternRealModelSample, MatOfPoint2f patternImageSample)
{
//Debug.Log( "patternRealModelSample\n" + patternRealModelSample.dump() );
//Debug.Log( "patternImageSample\n" + patternImageSample.dump() );
_patternRealSamples.Add(patternRealModelSample.clone());
_patternImageSamples.Add(patternImageSample.clone());
}
public static void PutPoint2f(MatOfPoint2f target, int index, float x, float y)
{
if (index >= target.size().height)
{
throw new Exception("Your mat of point is not big enough. Use alloc(capacity) before setting elements.");
}
target.put(index, 0, x, y);
}
// Start is called before the first frame update
void Start()
{
//Anders magic number
_drawingPlaceMat = new Mat(100, 150, CvType.CV_8UC4);
_drawingPlaceMat.setTo(new Scalar(255, 255, 255));
_imagePoints = new MatOfPoint2f();
_imagePoints.alloc(4);
}
void applyAffineTransform(Mat warpImage, Mat src, MatOfPoint2f srcTri, MatOfPoint2f dstTri)
{
// Given a pair of triangles, find the affine transform.
Mat warpMat = Imgproc.getAffineTransform(srcTri, dstTri);
// Apply the Affine Transform just found to the src image
Imgproc.warpAffine(src, warpImage, warpMat, warpImage.size(), Imgproc.INTER_LINEAR, Core.BORDER_REFLECT_101, new Scalar(255, 0, 0, 255));
}
public Mat create(Mat cameraMat, List <Region> regions)
{
if (regions.Count == 0)
{
return(null);
}
var regionGroups = new Dictionary <string, List <Region> >();
for (var i = 0; i < regions.Count; i++)
{
var exist = isConcatenateGroup(regions[i], regionGroups);
if (!exist)
{
regionGroups[regions[i].id] = new List <Region> {
regions[i]
};
}
}
Mat resultTexture = Mat.zeros(regions[0].parentSize, cameraMat.type());
foreach (KeyValuePair <string, List <Region> > pair in regionGroups)
{
// 領域グループ内の領域が一つの場合
if (pair.Value.Count == 1)
{
var tex = createForOne(cameraMat, pair.Value[0]);
Core.add(resultTexture, tex, resultTexture);
continue;
}
// 領域グループ内の領域の輪郭点を全て一つのリストにまとめる
List <Point> points = new List <Point>();
foreach (var region in pair.Value)
{
points.AddRange(region.candidate.contour2f.toList());
}
// グループの輪郭作成
MatOfPoint2f contours = new MatOfPoint2f();
contours.fromList(points);
// グループの輪郭のRotatedRectを得る
var ellipseRotatedRect = Imgproc.fitEllipse(contours);
// 事前用意してあるテクスチャ画像を食品領域のRotatedRectに射影変換
Mat originalTexture = selectOriginalTextureImage();
Mat texture = Mat.zeros(regions[0].parentSize, cameraMat.type());
ARUtil.affineTransform(originalTexture, texture, ellipseRotatedRect);
Core.add(resultTexture, texture, resultTexture);
}
return(coverBlackArea(resultTexture));
}
/// <summary>
/// projects points from the model coordinate space to the image coordinates.
/// Also computes derivatives of the image coordinates w.r.t the intrinsic
/// and extrinsic camera parameters
/// </summary>
/// <param name="objectPoints">Array of object points, 3xN/Nx3 1-channel or
/// 1xN/Nx1 3-channel, where N is the number of points in the view.</param>
/// <param name="rvec">Rotation vector (3x1).</param>
/// <param name="tvec">Translation vector (3x1).</param>
/// <param name="cameraMatrix">Camera matrix (3x3)</param>
/// <param name="distCoeffs">Input vector of distortion coefficients
/// (k_1, k_2, p_1, p_2[, k_3[, k_4, k_5, k_6]]) of 4, 5, or 8 elements.
/// If the vector is null, the zero distortion coefficients are assumed.</param>
/// <param name="imagePoints">Output array of image points, 2xN/Nx2 1-channel
/// or 1xN/Nx1 2-channel</param>
/// <param name="jacobian">Optional output 2Nx(10 + numDistCoeffs) jacobian matrix
/// of derivatives of image points with respect to components of the rotation vector,
/// translation vector, focal lengths, coordinates of the principal point and
/// the distortion coefficients. In the old interface different components of
/// the jacobian are returned via different output parameters.</param>
/// <param name="aspectRatio">Optional “fixed aspect ratio” parameter.
/// If the parameter is not 0, the function assumes that the aspect ratio (fx/fy)
/// is fixed and correspondingly adjusts the jacobian matrix.</param>
public static void ProjectPoints(IEnumerable<Point3f> objectPoints,
double[] rvec, double[] tvec,
double[,] cameraMatrix, double[] distCoeffs,
out Point2f[] imagePoints,
out double[,] jacobian,
double aspectRatio = 0)
{
if (objectPoints == null)
throw new ArgumentNullException(nameof(objectPoints));
if (rvec == null)
throw new ArgumentNullException(nameof(rvec));
if (rvec.Length != 3)
throw new ArgumentException("rvec.Length != 3");
if (tvec == null)
throw new ArgumentNullException(nameof(tvec));
if (tvec.Length != 3)
throw new ArgumentException("tvec.Length != 3");
if (cameraMatrix == null)
throw new ArgumentNullException(nameof(cameraMatrix));
if (cameraMatrix.GetLength(0) != 3 || cameraMatrix.GetLength(1) != 3)
throw new ArgumentException("cameraMatrix must be double[3,3]");
Point3f[] objectPointsArray = EnumerableEx.ToArray(objectPoints);
using (var objectPointsM = new Mat(objectPointsArray.Length, 1, MatType.CV_32FC3, objectPointsArray))
using (var rvecM = new Mat(3, 1, MatType.CV_64FC1, rvec))
using (var tvecM = new Mat(3, 1, MatType.CV_64FC1, tvec))
using (var cameraMatrixM = new Mat(3, 3, MatType.CV_64FC1, cameraMatrix))
using (var imagePointsM = new MatOfPoint2f())
{
var distCoeffsM = new Mat();
if (distCoeffs != null)
distCoeffsM = new Mat(distCoeffs.Length, 1, MatType.CV_64FC1, distCoeffs);
var jacobianM = new MatOfDouble();
NativeMethods.calib3d_projectPoints_Mat(objectPointsM.CvPtr,
rvecM.CvPtr, tvecM.CvPtr, cameraMatrixM.CvPtr, distCoeffsM.CvPtr,
imagePointsM.CvPtr, jacobianM.CvPtr, aspectRatio);
imagePoints = imagePointsM.ToArray();
jacobian = jacobianM.ToRectangularArray();
}
}
/// <summary>
/// Computes convex hull for a set of 2D points.
/// </summary>
/// <param name="points">The input 2D point set, represented by CV_32SC2 or CV_32FC2 matrix</param>
/// <param name="clockwise">If true, the output convex hull will be oriented clockwise,
/// otherwise it will be oriented counter-clockwise. Here, the usual screen coordinate
/// system is assumed - the origin is at the top-left corner, x axis is oriented to the right,
/// and y axis is oriented downwards.</param>
/// <returns>The output convex hull. It is a vector of points that form the
/// hull (must have the same type as the input points).</returns>
public Point2f[] ConvexHullFloatPoints(InputArray points, bool clockwise = false)
{
var dst = new MatOfPoint2f();
Cv2.ConvexHull(points, dst, clockwise, true);
return dst.ToArray();
}
/// <summary>
/// Raises the web cam texture to mat helper inited event.
/// </summary>
public void OnWebCamTextureToMatHelperInited()
{
Debug.Log ("OnWebCamTextureToMatHelperInited");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat ();
colors = new Color32[webCamTextureMat.cols () * webCamTextureMat.rows ()];
texture = new Texture2D (webCamTextureMat.cols (), webCamTextureMat.rows (), TextureFormat.RGBA32, false);
matOpFlowThis = new Mat ();
matOpFlowPrev = new Mat ();
MOPcorners = new MatOfPoint ();
mMOP2fptsThis = new MatOfPoint2f ();
mMOP2fptsPrev = new MatOfPoint2f ();
mMOP2fptsSafe = new MatOfPoint2f ();
mMOBStatus = new MatOfByte ();
mMOFerr = new MatOfFloat ();
gameObject.transform.localScale = new Vector3 (webCamTextureMat.cols (), webCamTextureMat.rows (), 1);
Debug.Log ("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale) {
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
} else {
Camera.main.orthographicSize = height / 2;
}
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
// webCamTextureToMatHelper.Play ();
}
private IEnumerator init()
{
if (webCamTexture != null) {
webCamTexture.Stop ();
initDone = false;
rgbaMat.Dispose ();
matOpFlowThis.Dispose ();
matOpFlowPrev.Dispose ();
MOPcorners.Dispose ();
mMOP2fptsThis.Dispose ();
mMOP2fptsPrev.Dispose ();
mMOP2fptsSafe.Dispose ();
mMOBStatus.Dispose ();
mMOFerr.Dispose ();
}
// Checks how many and which cameras are available on the device
for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) {
if (WebCamTexture.devices [cameraIndex].isFrontFacing == isFrontFacing) {
Debug.Log (cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing);
webCamDevice = WebCamTexture.devices [cameraIndex];
webCamTexture = new WebCamTexture (webCamDevice.name, width, height);
break;
}
}
if (webCamTexture == null) {
webCamDevice = WebCamTexture.devices [0];
webCamTexture = new WebCamTexture (webCamDevice.name, width, height);
}
Debug.Log ("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
// Starts the camera
webCamTexture.Play ();
while (true) {
//If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/)
#if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1)
if (webCamTexture.width > 16 && webCamTexture.height > 16) {
#else
if (webCamTexture.didUpdateThisFrame) {
#endif
Debug.Log ("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
Debug.Log ("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing);
colors = new Color32[webCamTexture.width * webCamTexture.height];
rgbaMat = new Mat (webCamTexture.height, webCamTexture.width, CvType.CV_8UC4);
matOpFlowThis = new Mat ();
matOpFlowPrev = new Mat ();
MOPcorners = new MatOfPoint ();
mMOP2fptsThis = new MatOfPoint2f ();
mMOP2fptsPrev = new MatOfPoint2f ();
mMOP2fptsSafe = new MatOfPoint2f ();
mMOBStatus = new MatOfByte ();
mMOFerr = new MatOfFloat ();
texture = new Texture2D (webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
gameObject.transform.eulerAngles = new Vector3 (0, 0, 0);
#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
gameObject.transform.eulerAngles = new Vector3 (0, 0, -90);
#endif
// gameObject.transform.rotation = gameObject.transform.rotation * Quaternion.AngleAxis (webCamTexture.videoRotationAngle, Vector3.back);
gameObject.transform.localScale = new Vector3 (webCamTexture.width, webCamTexture.height, 1);
// bool videoVerticallyMirrored = webCamTexture.videoVerticallyMirrored;
// float scaleX = 1;
// float scaleY = videoVerticallyMirrored ? -1.0f : 1.0f;
// if (webCamTexture.videoRotationAngle == 270)
// scaleY = -1.0f;
// gameObject.transform.localScale = new Vector3 (scaleX * gameObject.transform.localScale.x, scaleY * gameObject.transform.localScale.y, 1);
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
Camera.main.orthographicSize = webCamTexture.width / 2;
#else
Camera.main.orthographicSize = webCamTexture.height / 2;
#endif
initDone = true;
break;
} else {
yield return 0;
}
}
}
// Update is called once per frame
void Update ()
{
if (!initDone)
return;
#if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1)
if (webCamTexture.width > 16 && webCamTexture.height > 16) {
#else
if (webCamTexture.didUpdateThisFrame) {
#endif
Utils.webCamTextureToMat (webCamTexture, rgbaMat, colors);
if (webCamTexture.videoVerticallyMirrored) {
if (webCamDevice.isFrontFacing) {
if (webCamTexture.videoRotationAngle == 0) {
Core.flip (rgbaMat, rgbaMat, 1);
} else if (webCamTexture.videoRotationAngle == 90) {
Core.flip (rgbaMat, rgbaMat, 0);
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, 1);
}
} else {
if (webCamTexture.videoRotationAngle == 90) {
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, -1);
}
}
} else {
if (webCamDevice.isFrontFacing) {
if (webCamTexture.videoRotationAngle == 0) {
Core.flip (rgbaMat, rgbaMat, 1);
} else if (webCamTexture.videoRotationAngle == 90) {
Core.flip (rgbaMat, rgbaMat, 0);
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, 1);
}
} else {
if (webCamTexture.videoRotationAngle == 90) {
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, -1);
}
}
}
if (mMOP2fptsPrev.rows () == 0) {
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
Imgproc.cvtColor (rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo (matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack (matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsPrev.fromArray (MOPcorners.toArray ());
// get safe copy of this corners
mMOP2fptsPrev.copyTo (mMOP2fptsSafe);
} else {
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo (matOpFlowPrev);
// get this mat
Imgproc.cvtColor (rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack (matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsThis.fromArray (MOPcorners.toArray ());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo (mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo (mMOP2fptsSafe);
}
/*
Parameters:
prevImg first 8-bit input image
nextImg second input image
prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK (matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (!mMOBStatus.empty ()) {
List<Point> cornersPrev = mMOP2fptsPrev.toList ();
List<Point> cornersThis = mMOP2fptsThis.toList ();
List<byte> byteStatus = mMOBStatus.toList ();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++) {
if (byteStatus [x] == 1) {
Point pt = cornersThis [x];
Point pt2 = cornersPrev [x];
Core.circle (rgbaMat, pt, 5, colorRed, iLineThickness - 1);
Core.line (rgbaMat, pt, pt2, colorRed, iLineThickness);
}
}
}
Utils.matToTexture2D (rgbaMat, texture, colors);
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
}
}
void OnDisable ()
{
webCamTexture.Stop ();
}
void OnGUI ()
{
float screenScale = Screen.width / 240.0f;
Matrix4x4 scaledMatrix = Matrix4x4.Scale (new Vector3 (screenScale, screenScale, screenScale));
GUI.matrix = scaledMatrix;
GUILayout.BeginVertical ();
if (GUILayout.Button ("back")) {
Application.LoadLevel ("OpenCVForUnitySample");
}
if (GUILayout.Button ("change camera")) {
isFrontFacing = !isFrontFacing;
StartCoroutine (init ());
}
GUILayout.EndVertical ();
}
}
}
/// <summary>
/// Converts the screen point.
/// </summary>
/// <returns>The screen point.</returns>
/// <param name="screenPoint">Screen point.</param>
/// <param name="quad">Quad.</param>
/// <param name="cam">Cam.</param>
static Point convertScreenPoint(Point screenPoint, GameObject quad, Camera cam)
{
Vector2 tl;
Vector2 tr;
Vector2 br;
Vector2 bl;
tl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.x / 2, quad.transform.localPosition.y + quad.transform.localScale.y / 2, quad.transform.localPosition.z));
tr = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.x / 2, quad.transform.localPosition.y + quad.transform.localScale.y / 2, quad.transform.localPosition.z));
br = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.x / 2, quad.transform.localPosition.y - quad.transform.localScale.y / 2, quad.transform.localPosition.z));
bl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.x / 2, quad.transform.localPosition.y - quad.transform.localScale.y / 2, quad.transform.localPosition.z));
Mat srcRectMat = new Mat (4, 1, CvType.CV_32FC2);
Mat dstRectMat = new Mat (4, 1, CvType.CV_32FC2);
srcRectMat.put (0, 0, tl.x, tl.y, tr.x, tr.y, br.x, br.y, bl.x, bl.y);
dstRectMat.put (0, 0, 0.0, 0.0, quad.transform.localScale.x, 0.0, quad.transform.localScale.x, quad.transform.localScale.y, 0.0, quad.transform.localScale.y);
Mat perspectiveTransform = Imgproc.getPerspectiveTransform (srcRectMat, dstRectMat);
// Debug.Log ("srcRectMat " + srcRectMat.dump ());
// Debug.Log ("dstRectMat " + dstRectMat.dump ());
// Debug.Log ("perspectiveTransform " + perspectiveTransform.dump ());
MatOfPoint2f srcPointMat = new MatOfPoint2f (screenPoint);
MatOfPoint2f dstPointMat = new MatOfPoint2f ();
Core.perspectiveTransform (srcPointMat, dstPointMat, perspectiveTransform);
// Debug.Log ("srcPointMat " + srcPointMat.dump ());
// Debug.Log ("dstPointMat " + dstPointMat.dump ());
return dstPointMat.toArray () [0];
}
/// <summary>
/// Hands the pose estimation process.
/// </summary>
public void handPoseEstimationProcess(Mat rgbaMat)
{
//Imgproc.blur(mRgba, mRgba, new Size(5,5));
Imgproc.GaussianBlur (rgbaMat, rgbaMat, new OpenCVForUnity.Size (3, 3), 1, 1);
//Imgproc.medianBlur(mRgba, mRgba, 3);
if (!isColorSelected)
return;
List<MatOfPoint> contours = detector.getContours ();
detector.process (rgbaMat);
// Debug.Log ("Contours count: " + contours.Count);
if (contours.Count <= 0) {
return;
}
RotatedRect rect = Imgproc.minAreaRect (new MatOfPoint2f (contours [0].toArray ()));
double boundWidth = rect.size.width;
double boundHeight = rect.size.height;
int boundPos = 0;
for (int i = 1; i < contours.Count; i++) {
rect = Imgproc.minAreaRect (new MatOfPoint2f (contours [i].toArray ()));
if (rect.size.width * rect.size.height > boundWidth * boundHeight) {
boundWidth = rect.size.width;
boundHeight = rect.size.height;
boundPos = i;
}
}
OpenCVForUnity.Rect boundRect = Imgproc.boundingRect (new MatOfPoint (contours [boundPos].toArray ()));
Imgproc.rectangle (rgbaMat, boundRect.tl (), boundRect.br (), CONTOUR_COLOR_WHITE, 2, 8, 0);
// Debug.Log (
// " Row start [" +
// (int)boundRect.tl ().y + "] row end [" +
// (int)boundRect.br ().y + "] Col start [" +
// (int)boundRect.tl ().x + "] Col end [" +
// (int)boundRect.br ().x + "]");
double a = boundRect.br ().y - boundRect.tl ().y;
a = a * 0.7;
a = boundRect.tl ().y + a;
// Debug.Log (
// " A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]");
//Core.rectangle( mRgba, boundRect.tl(), boundRect.br(), CONTOUR_COLOR, 2, 8, 0 );
Imgproc.rectangle (rgbaMat, boundRect.tl (), new Point (boundRect.br ().x, a), CONTOUR_COLOR, 2, 8, 0);
MatOfPoint2f pointMat = new MatOfPoint2f ();
Imgproc.approxPolyDP (new MatOfPoint2f (contours [boundPos].toArray ()), pointMat, 3, true);
contours [boundPos] = new MatOfPoint (pointMat.toArray ());
MatOfInt hull = new MatOfInt ();
MatOfInt4 convexDefect = new MatOfInt4 ();
Imgproc.convexHull (new MatOfPoint (contours [boundPos].toArray ()), hull);
if (hull.toArray ().Length < 3)
return;
Imgproc.convexityDefects (new MatOfPoint (contours [boundPos] .toArray ()), hull, convexDefect);
List<MatOfPoint> hullPoints = new List<MatOfPoint> ();
List<Point> listPo = new List<Point> ();
for (int j = 0; j < hull.toList().Count; j++) {
listPo.Add (contours [boundPos].toList () [hull.toList () [j]]);
}
MatOfPoint e = new MatOfPoint ();
e.fromList (listPo);
hullPoints.Add (e);
List<MatOfPoint> defectPoints = new List<MatOfPoint> ();
List<Point> listPoDefect = new List<Point> ();
for (int j = 0; j < convexDefect.toList().Count; j = j+4) {
Point farPoint = contours [boundPos].toList () [convexDefect.toList () [j + 2]];
int depth = convexDefect.toList () [j + 3];
if (depth > threasholdSlider.value && farPoint.y < a) {
listPoDefect.Add (contours [boundPos].toList () [convexDefect.toList () [j + 2]]);
}
// Debug.Log ("defects [" + j + "] " + convexDefect.toList () [j + 3]);
}
MatOfPoint e2 = new MatOfPoint ();
e2.fromList (listPo);
defectPoints.Add (e2);
// Debug.Log ("hull: " + hull.toList ());
// Debug.Log ("defects: " + convexDefect.toList ());
Imgproc.drawContours (rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3);
// int defectsTotal = (int)convexDefect.total();
// Debug.Log ("Defect total " + defectsTotal);
this.numberOfFingers = listPoDefect.Count;
if (this.numberOfFingers > 5)
this.numberOfFingers = 5;
// Debug.Log ("numberOfFingers " + numberOfFingers);
// Core.putText (mRgba, "" + numberOfFingers, new Point (mRgba.cols () / 2, mRgba.rows () / 2), Core.FONT_HERSHEY_PLAIN, 4.0, new Scalar (255, 255, 255, 255), 6, Core.LINE_AA, false);
numberOfFingersText.text = numberOfFingers.ToString ();
foreach (Point p in listPoDefect) {
Imgproc.circle (rgbaMat, p, 6, new Scalar (255, 0, 255, 255), -1);
}
}
private IEnumerator init()
{
if (webCamTexture != null) {
webCamTexture.Stop ();
initDone = false;
rgbaMat.Dispose ();
matOpFlowThis.Dispose ();
matOpFlowPrev.Dispose ();
MOPcorners.Dispose ();
mMOP2fptsThis.Dispose ();
mMOP2fptsPrev.Dispose ();
mMOP2fptsSafe.Dispose ();
mMOBStatus.Dispose ();
mMOFerr.Dispose ();
}
// Checks how many and which cameras are available on the device
for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) {
if (WebCamTexture.devices [cameraIndex].isFrontFacing == shouldUseFrontFacing) {
Debug.Log (cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing);
webCamDevice = WebCamTexture.devices [cameraIndex];
webCamTexture = new WebCamTexture (webCamDevice.name, width, height);
break;
}
}
if (webCamTexture == null) {
webCamDevice = WebCamTexture.devices [0];
webCamTexture = new WebCamTexture (webCamDevice.name, width, height);
}
Debug.Log ("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
// Starts the camera
webCamTexture.Play ();
while (true) {
//If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/)
#if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1)
if (webCamTexture.width > 16 && webCamTexture.height > 16) {
#else
if (webCamTexture.didUpdateThisFrame) {
#if UNITY_IOS && !UNITY_EDITOR && UNITY_5_2
while (webCamTexture.width <= 16) {
webCamTexture.GetPixels32 ();
yield return new WaitForEndOfFrame ();
}
#endif
#endif
Debug.Log ("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
Debug.Log ("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing);
colors = new Color32[webCamTexture.width * webCamTexture.height];
rgbaMat = new Mat (webCamTexture.height, webCamTexture.width, CvType.CV_8UC4);
matOpFlowThis = new Mat ();
matOpFlowPrev = new Mat ();
MOPcorners = new MatOfPoint ();
mMOP2fptsThis = new MatOfPoint2f ();
mMOP2fptsPrev = new MatOfPoint2f ();
mMOP2fptsSafe = new MatOfPoint2f ();
mMOBStatus = new MatOfByte ();
mMOFerr = new MatOfFloat ();
texture = new Texture2D (webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
updateLayout ();
screenOrientation = Screen.orientation;
initDone = true;
break;
} else {
yield return 0;
}
}
}
private IEnumerator init()
{
if (webCamTexture != null) {
webCamTexture.Stop ();
initDone = false;
rgbaMat.Dispose ();
hsvMat.Dispose ();
if (roiHistMat != null)
roiHistMat.Dispose ();
roiPointList.Clear ();
}
// Checks how many and which cameras are available on the device
for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) {
if (WebCamTexture.devices [cameraIndex].isFrontFacing == isFrontFacing) {
Debug.Log (cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing);
webCamDevice = WebCamTexture.devices [cameraIndex];
webCamTexture = new WebCamTexture (webCamDevice.name, width, height);
break;
}
}
if (webCamTexture == null) {
webCamDevice = WebCamTexture.devices [0];
webCamTexture = new WebCamTexture (webCamDevice.name, width, height);
}
Debug.Log ("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
// Starts the camera
webCamTexture.Play ();
while (true) {
//If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/)
#if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1)
if (webCamTexture.width > 16 && webCamTexture.height > 16) {
#else
if (webCamTexture.didUpdateThisFrame) {
#endif
Debug.Log ("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS);
Debug.Log ("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing);
colors = new Color32[webCamTexture.width * webCamTexture.height];
rgbaMat = new Mat (webCamTexture.height, webCamTexture.width, CvType.CV_8UC4);
hsvMat = new Mat (webCamTexture.height, webCamTexture.width, CvType.CV_8UC3);
texture = new Texture2D (webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
gameObject.transform.eulerAngles = new Vector3 (0, 0, 0);
#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
gameObject.transform.eulerAngles = new Vector3 (0, 0, -90);
#endif
// gameObject.transform.rotation = gameObject.transform.rotation * Quaternion.AngleAxis (webCamTexture.videoRotationAngle, Vector3.back);
gameObject.transform.localScale = new Vector3 (webCamTexture.width, webCamTexture.height, 1);
// bool videoVerticallyMirrored = webCamTexture.videoVerticallyMirrored;
// float scaleX = 1;
// float scaleY = videoVerticallyMirrored ? -1.0f : 1.0f;
// if (webCamTexture.videoRotationAngle == 270)
// scaleY = -1.0f;
// gameObject.transform.localScale = new Vector3 (scaleX * gameObject.transform.localScale.x, scaleY * gameObject.transform.localScale.y, 1);
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
Camera.main.orthographicSize = (((float)Screen.height/(float)Screen.width) * (float)webCamTexture.height) / 2.0f;
#else
Camera.main.orthographicSize = webCamTexture.height / 2;
#endif
initDone = true;
break;
} else {
yield return 0;
}
}
}
// Update is called once per frame
void Update ()
{
if (!initDone)
return;
#if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1)
if (webCamTexture.width > 16 && webCamTexture.height > 16) {
#else
if (webCamTexture.didUpdateThisFrame) {
#endif
Utils.webCamTextureToMat (webCamTexture, rgbaMat, colors);
if (webCamTexture.videoVerticallyMirrored) {
if (webCamDevice.isFrontFacing) {
if (webCamTexture.videoRotationAngle == 0) {
Core.flip (rgbaMat, rgbaMat, 1);
} else if (webCamTexture.videoRotationAngle == 90) {
Core.flip (rgbaMat, rgbaMat, 0);
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, 1);
}
} else {
if (webCamTexture.videoRotationAngle == 90) {
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, -1);
}
}
} else {
if (webCamDevice.isFrontFacing) {
if (webCamTexture.videoRotationAngle == 0) {
Core.flip (rgbaMat, rgbaMat, 1);
} else if (webCamTexture.videoRotationAngle == 90) {
Core.flip (rgbaMat, rgbaMat, 0);
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, 1);
}
} else {
if (webCamTexture.videoRotationAngle == 90) {
} else if (webCamTexture.videoRotationAngle == 270) {
Core.flip (rgbaMat, rgbaMat, -1);
}
}
}
Imgproc.cvtColor (rgbaMat, hsvMat, Imgproc.COLOR_RGBA2RGB);
Imgproc.cvtColor (hsvMat, hsvMat, Imgproc.COLOR_RGB2HSV);
Point[] points = roiPointList.ToArray ();
if (roiPointList.Count == 4) {
using (Mat backProj = new Mat ()) {
Imgproc.calcBackProject (new List<Mat> (new Mat[]{hsvMat}), new MatOfInt (0), roiHistMat, backProj, new MatOfFloat (0, 180), 1.0);
RotatedRect r = Video.CamShift (backProj, roiRect, termination);
r.points (points);
}
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
//Touch
int touchCount = Input.touchCount;
if (touchCount == 1)
{
if(Input.GetTouch(0).phase == TouchPhase.Ended){
roiPointList.Clear ();
}
}
#else
if (Input.GetMouseButtonUp (0)) {
roiPointList.Clear ();
}
#endif
}
if (roiPointList.Count < 4) {
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
//Touch
int touchCount = Input.touchCount;
if (touchCount == 1)
{
Touch t = Input.GetTouch(0);
if(t.phase == TouchPhase.Ended){
roiPointList.Add (convertScreenPoint (new Point (t.position.x, t.position.y), gameObject, Camera.main));
// Debug.Log ("touch X " + t.position.x);
// Debug.Log ("touch Y " + t.position.y);
if (!(new OpenCVForUnity.Rect (0, 0, hsvMat.width (), hsvMat.height ()).contains (roiPointList [roiPointList.Count - 1]))) {
roiPointList.RemoveAt (roiPointList.Count - 1);
}
}
}
#else
//Mouse
if (Input.GetMouseButtonUp (0)) {
roiPointList.Add (convertScreenPoint (new Point (Input.mousePosition.x, Input.mousePosition.y), gameObject, Camera.main));
// Debug.Log ("mouse X " + Input.mousePosition.x);
// Debug.Log ("mouse Y " + Input.mousePosition.y);
if (!(new OpenCVForUnity.Rect (0, 0, hsvMat.width (), hsvMat.height ()).contains (roiPointList [roiPointList.Count - 1]))) {
roiPointList.RemoveAt (roiPointList.Count - 1);
}
}
#endif
if (roiPointList.Count == 4) {
using (MatOfPoint roiPointMat = new MatOfPoint (roiPointList.ToArray ())) {
roiRect = Imgproc.boundingRect (roiPointMat);
}
if (roiHistMat != null) {
roiHistMat.Dispose ();
roiHistMat = null;
}
roiHistMat = new Mat ();
using (Mat roiHSVMat = new Mat(hsvMat, roiRect))
using (Mat maskMat = new Mat ()) {
Imgproc.calcHist (new List<Mat> (new Mat[]{roiHSVMat}), new MatOfInt (0), maskMat, roiHistMat, new MatOfInt (16), new MatOfFloat (0, 180));
Core.normalize (roiHistMat, roiHistMat, 0, 255, Core.NORM_MINMAX);
// Debug.Log ("roiHist " + roiHistMat.ToString ());
}
}
}
if (points.Length < 4) {
for (int i = 0; i < points.Length; i++) {
Core.circle (rgbaMat, points [i], 6, new Scalar (0, 0, 255, 255), 2);
}
} else {
for (int i = 0; i < 4; i++) {
Core.line (rgbaMat, points [i], points [(i + 1) % 4], new Scalar (255, 0, 0, 255), 2);
}
Core.rectangle (rgbaMat, roiRect.tl (), roiRect.br (), new Scalar (0, 255, 0, 255), 2);
}
Core.putText (rgbaMat, "PLEASE TOUCH 4 POINTS", new Point (5, 25), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Core.LINE_AA, false);
Utils.matToTexture2D (rgbaMat, texture, colors);
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
}
}
void OnDisable ()
{
webCamTexture.Stop ();
}
void OnGUI ()
{
float screenScale = Screen.width / 240.0f;
Matrix4x4 scaledMatrix = Matrix4x4.Scale (new Vector3 (screenScale, screenScale, screenScale));
GUI.matrix = scaledMatrix;
GUILayout.BeginVertical ();
if (GUILayout.Button ("back")) {
Application.LoadLevel ("OpenCVForUnitySample");
}
if (GUILayout.Button ("change camera")) {
isFrontFacing = !isFrontFacing;
StartCoroutine (init ());
}
GUILayout.EndVertical ();
}
/// <summary>
/// Converts the screen point.
/// </summary>
/// <returns>The screen point.</returns>
/// <param name="screenPoint">Screen point.</param>
/// <param name="quad">Quad.</param>
/// <param name="cam">Cam.</param>
static Point convertScreenPoint (Point screenPoint, GameObject quad, Camera cam)
{
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
Vector2 tl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.y / 2, quad.transform.localPosition.y + quad.transform.localScale.x / 2, quad.transform.localPosition.z));
Vector2 tr = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.y / 2, quad.transform.localPosition.y - quad.transform.localScale.x / 2, quad.transform.localPosition.z));
Vector2 br = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.y / 2, quad.transform.localPosition.y - quad.transform.localScale.x / 2, quad.transform.localPosition.z));
Vector2 bl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.y / 2, quad.transform.localPosition.y + quad.transform.localScale.x / 2, quad.transform.localPosition.z));
#else
Vector2 tl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.x / 2, quad.transform.localPosition.y + quad.transform.localScale.y / 2, quad.transform.localPosition.z));
Vector2 tr = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.x / 2, quad.transform.localPosition.y + quad.transform.localScale.y / 2, quad.transform.localPosition.z));
Vector2 br = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.x / 2, quad.transform.localPosition.y - quad.transform.localScale.y / 2, quad.transform.localPosition.z));
Vector2 bl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.x / 2, quad.transform.localPosition.y - quad.transform.localScale.y / 2, quad.transform.localPosition.z));
#endif
Mat srcRectMat = new Mat (4, 1, CvType.CV_32FC2);
Mat dstRectMat = new Mat (4, 1, CvType.CV_32FC2);
srcRectMat.put (0, 0, tl.x, tl.y, tr.x, tr.y, br.x, br.y, bl.x, bl.y);
dstRectMat.put (0, 0, 0.0, 0.0, quad.transform.localScale.x, 0.0, quad.transform.localScale.x, quad.transform.localScale.y, 0.0, quad.transform.localScale.y);
Mat perspectiveTransform = Imgproc.getPerspectiveTransform (srcRectMat, dstRectMat);
// Debug.Log ("srcRectMat " + srcRectMat.dump ());
// Debug.Log ("dstRectMat " + dstRectMat.dump ());
// Debug.Log ("perspectiveTransform " + perspectiveTransform.dump ());
MatOfPoint2f srcPointMat = new MatOfPoint2f (screenPoint);
MatOfPoint2f dstPointMat = new MatOfPoint2f ();
Core.perspectiveTransform (srcPointMat, dstPointMat, perspectiveTransform);
// Debug.Log ("srcPointMat " + srcPointMat.dump ());
// Debug.Log ("dstPointMat " + dstPointMat.dump ());
return dstPointMat.toArray () [0];
}
}
/// <summary>
/// Finds the candidates.
/// </summary>
/// <param name="contours">Contours.</param>
/// <param name="detectedMarkers">Detected markers.</param>
void findCandidates (List<MatOfPoint> contours, List<Marker> detectedMarkers)
{
MatOfPoint2f approxCurve = new MatOfPoint2f ();
List<Marker> possibleMarkers = new List<Marker> ();
// For each contour, analyze if it is a parallelepiped likely to be the marker
for (int i=0; i<contours.Count; i++) {
// Approximate to a polygon
double eps = contours [i].toArray ().Length * 0.05;
Imgproc.approxPolyDP (new MatOfPoint2f (contours [i].toArray ()), approxCurve, eps, true);
Point[] approxCurveArray = approxCurve.toArray ();
// We interested only in polygons that contains only four points
if (approxCurveArray.Length != 4)
continue;
// And they have to be convex
if (!Imgproc.isContourConvex (new MatOfPoint (approxCurveArray)))
continue;
// Ensure that the distance between consecutive points is large enough
float minDist = float.MaxValue;
for (int p = 0; p < 4; p++) {
Point side = new Point (approxCurveArray [p].x - approxCurveArray [(p + 1) % 4].x, approxCurveArray [p].y - approxCurveArray [(p + 1) % 4].y);
float squaredSideLength = (float)side.dot (side);
minDist = Mathf.Min (minDist, squaredSideLength);
}
// Check that distance is not very small
if (minDist < m_minContourLengthAllowed)
continue;
// All tests are passed. Save marker candidate:
Marker m = new Marker ();
m.points = new MatOfPoint ();
List<Point> markerPointsList = new List<Point> ();
for (int p = 0; p<4; p++)
markerPointsList.Add (new Point (approxCurveArray [p].x, approxCurveArray [p].y));
// Sort the points in anti-clockwise order
// Trace a line between the first and second point.
// If the third point is at the right side, then the points are anti-clockwise
Point v1 = new Point (markerPointsList [1].x - markerPointsList [0].x, markerPointsList [1].y - markerPointsList [0].y);
Point v2 = new Point (markerPointsList [2].x - markerPointsList [0].x, markerPointsList [2].y - markerPointsList [0].y);
double o = (v1.x * v2.y) - (v1.y * v2.x);
if (o < 0.0) { //if the third point is in the left side, then sort in anti-clockwise order
Point tmp = markerPointsList [1];
markerPointsList [1] = markerPointsList [3];
markerPointsList [3] = tmp;
}
m.points.fromList (markerPointsList);
possibleMarkers.Add (m);
}
approxCurve.Dispose ();
//Debug.Log ("possibleMarkers " + possibleMarkers.Count);
// Remove these elements which corners are too close to each other.
// First detect candidates for removal:
List< Point > tooNearCandidates = new List<Point> ();
for (int i=0; i<possibleMarkers.Count; i++) {
Marker m1 = possibleMarkers [i];
Point[] m1PointsArray = m1.points.toArray ();
//calculate the average distance of each corner to the nearest corner of the other marker candidate
for (int j=i+1; j<possibleMarkers.Count; j++) {
Marker m2 = possibleMarkers [j];
Point[] m2PointsArray = m2.points.toArray ();
float distSquared = 0;
for (int c = 0; c < 4; c++) {
Point v = new Point (m1PointsArray [c].x - m2PointsArray [c].x, m1PointsArray [c].y - m2PointsArray [c].y);
distSquared += (float)v.dot (v);
}
distSquared /= 4;
if (distSquared < 100) {
tooNearCandidates.Add (new Point (i, j));
}
}
}
// Mark for removal the element of the pair with smaller perimeter
List<bool> removalMask = new List<bool> (possibleMarkers.Count);
for (int i = 0; i < possibleMarkers.Count; i++) {
removalMask.Add (false);
}
for (int i=0; i<tooNearCandidates.Count; i++) {
float p1 = perimeter (possibleMarkers [(int)tooNearCandidates [i].x].points);
float p2 = perimeter (possibleMarkers [(int)tooNearCandidates [i].y].points);
int removalIndex;
if (p1 > p2)
removalIndex = (int)tooNearCandidates [i].x;
else
removalIndex = (int)tooNearCandidates [i].y;
removalMask [removalIndex] = true;
}
// Return candidates
detectedMarkers.Clear ();
for (int i=0; i<possibleMarkers.Count; i++) {
if (!removalMask [i])
detectedMarkers.Add (possibleMarkers [i]);
}
}
/// <summary>
/// performs perspective transformation of each element of multi-channel input matrix
/// </summary>
/// <param name="src">The source two-channel or three-channel floating-point array;
/// each element is 2D/3D vector to be transformed</param>
/// <param name="m">3x3 or 4x4 transformation matrix</param>
/// <returns>The destination array; it will have the same size and same type as src</returns>
public static Point2f[] PerspectiveTransform(IEnumerable<Point2f> src, Mat m)
{
if (src == null)
throw new ArgumentNullException("src");
if (m == null)
throw new ArgumentNullException("m");
using (var srcMat = MatOfPoint2f.FromArray(src))
using (var dstMat = new MatOfPoint2f())
{
NativeMethods.core_perspectiveTransform_Mat(srcMat.CvPtr, dstMat.CvPtr, m.CvPtr);
return dstMat.ToArray();
}
}
/// <summary>
/// Recognizes the markers.
/// </summary>
/// <param name="grayscale">Grayscale.</param>
/// <param name="detectedMarkers">Detected markers.</param>
void recognizeMarkers (Mat grayscale, List<Marker> detectedMarkers)
{
List<Marker> goodMarkers = new List<Marker> ();
// Identify the markers
for (int i=0; i<detectedMarkers.Count; i++) {
Marker marker = detectedMarkers [i];
// Find the perspective transformation that brings current marker to rectangular form
Mat markerTransform = Imgproc.getPerspectiveTransform (new MatOfPoint2f (marker.points.toArray ()), m_markerCorners2d);
// Transform image to get a canonical marker image
Imgproc.warpPerspective (grayscale, canonicalMarkerImage, markerTransform, markerSize);
MatOfInt nRotations = new MatOfInt (0);
int id = Marker.getMarkerId (canonicalMarkerImage, nRotations, m_markerDesign);
if (id != - 1) {
marker.id = id;
// Debug.Log ("id " + id);
//sort the points so that they are always in the same order no matter the camera orientation
List<Point> MarkerPointsList = marker.points.toList ();
// std::rotate(marker.points.begin(), marker.points.begin() + 4 - nRotations, marker.points.end());
MarkerPointsList = MarkerPointsList.Skip (4 - nRotations.toArray () [0]).Concat (MarkerPointsList.Take (4 - nRotations.toArray () [0])).ToList ();
marker.points.fromList (MarkerPointsList);
goodMarkers.Add (marker);
}
nRotations.Dispose ();
}
// Debug.Log ("goodMarkers " + goodMarkers.Count);
// Refine marker corners using sub pixel accuracy
if (goodMarkers.Count > 0) {
List<Point> preciseCornersPoint = new List<Point> (4 * goodMarkers.Count);
for (int i = 0; i < preciseCornersPoint.Capacity; i++) {
preciseCornersPoint.Add (new Point (0, 0));
}
for (int i=0; i<goodMarkers.Count; i++) {
Marker marker = goodMarkers [i];
List<Point> markerPointsList = marker.points.toList ();
for (int c = 0; c <4; c++) {
preciseCornersPoint [i * 4 + c] = markerPointsList [c];
}
}
MatOfPoint2f preciseCorners = new MatOfPoint2f (preciseCornersPoint.ToArray ());
TermCriteria termCriteria = new TermCriteria (TermCriteria.MAX_ITER | TermCriteria.EPS, 30, 0.01);
Imgproc.cornerSubPix (grayscale, preciseCorners, new Size (5, 5), new Size (-1, -1), termCriteria);
preciseCornersPoint = preciseCorners.toList ();
// Copy refined corners position back to markers
for (int i=0; i<goodMarkers.Count; i++) {
Marker marker = goodMarkers [i];
List<Point> markerPointsList = marker.points.toList ();
for (int c=0; c<4; c++) {
markerPointsList [c] = preciseCornersPoint [i * 4 + c];
}
}
preciseCorners.Dispose ();
}
detectedMarkers.Clear ();
detectedMarkers.AddRange (goodMarkers);
}
/// <summary>
/// Ises the into.
/// </summary>
/// <returns><c>true</c>, if into was ised, <c>false</c> otherwise.</returns>
/// <param name="contour">Contour.</param>
/// <param name="b">The blue component.</param>
bool isInto (MatOfPoint2f contour, List<Point> b)
{
for (int i=0; i<b.Count; i++) {
if (Imgproc.pointPolygonTest (contour, b [i], false) > 0)
return true;
}
return false;
}
// fixed FromArray behavior
static Point2d[] MyPerspectiveTransform2(Point2f[] yourData, Mat transformationMatrix)
{
using (MatOfPoint2f s = MatOfPoint2f.FromArray(yourData))
using (MatOfPoint2f d = new MatOfPoint2f())
{
Cv2.PerspectiveTransform(s, d, transformationMatrix);
Point2f[] f = d.ToArray();
return f.Select(Point2fToPoint2d).ToArray();
}
}
