/// <summary>
/// Creates pose data dictionary.
/// </summary>
/// <param name="markerCount">Marker count.</param>
/// <param name="ids">ids.</param>
/// <param name="rvecs">Rvecs.</param>
/// <param name="tvecs">Tvecs.</param>
/// <returns>PoseData dictionary.</returns>
public static Dictionary <int, PoseData> CreatePoseDataDict(int markerCount, int[] ids, double[] rvecs, double[] tvecs)
{
Dictionary <int, PoseData> dict = new Dictionary <int, PoseData>();
if (markerCount == 0)
{
return(dict);
}
Vector3 rvec = new Vector3();
for (int i = 0; i < markerCount; i++)
{
PoseData data = new PoseData();
data.pos.Set((float)tvecs[i * 3], (float)tvecs[i * 3 + 1], (float)tvecs[i * 3 + 2]);
rvec.Set((float)rvecs[i * 3], (float)rvecs[i * 3 + 1], (float)rvecs[i * 3 + 2]);
float theta = rvec.magnitude;
rvec.Normalize();
data.rot = Quaternion.AngleAxis(theta * Mathf.Rad2Deg, rvec);
dict[ids[i]] = data;
}
return(dict);
}
/// <summary>
/// Performs a lowpass check on the position and rotation of each marker in newDict, comparing them to those in oldDict.
/// </summary>
/// <param name="oldDict">Old dictionary.</param>
/// <param name="newDict">New dictionary.</param>
/// <param name="posThreshold">Positon threshold.</param>
/// <param name="rotThreshold">Rotation threshold.</param>
public static void LowpassPoseDataDict(Dictionary <int, PoseData> oldDict, Dictionary <int, PoseData> newDict, float posThreshold, float rotThreshold)
{
posThreshold *= posThreshold;
List <int> keys = new List <int>(newDict.Keys);
foreach (int key in keys)
{
if (!oldDict.ContainsKey(key))
{
continue;
}
PoseData oldPose = oldDict[key];
PoseData newPose = newDict[key];
float posDiff = (newPose.pos - oldPose.pos).sqrMagnitude;
float rotDiff = Quaternion.Angle(newPose.rot, oldPose.rot);
if (posDiff < posThreshold)
{
newPose.pos = oldPose.pos;
}
if (rotDiff < rotThreshold)
{
newPose.rot = oldPose.rot;
}
newDict[key] = newPose;
}
}
/// <summary>
/// Convertes rvec and tvec value to PoseData.
/// </summary>
/// <param name="tvec">Rvec.</param>
/// <param name="tvec">Tvec.</param>
/// <returns>PoseData.</returns>
public static PoseData ConvertRvecTvecToPoseData(double[] rvec, double[] tvec)
{
PoseData data = new PoseData();
data.pos = ConvertTvecToPos(tvec);
data.rot = ConvertRvecToRot(rvec);
return(data);
}
/// <summary>
/// Performs a lowpass check on the position and rotation in newPose, comparing them to oldPose.
/// </summary>
/// <param name="oldPose">Old PoseData.</param>
/// <param name="newPose">New PoseData.</param>
/// <param name="posThreshold">Positon threshold.</param>
/// <param name="rotThreshold">Rotation threshold.</param>
public static void LowpassPoseData(ref PoseData oldPose, ref PoseData newPose, float posThreshold, float rotThreshold)
{
posThreshold *= posThreshold;
float posDiff = (newPose.pos - oldPose.pos).sqrMagnitude;
float rotDiff = Quaternion.Angle(newPose.rot, oldPose.rot);
if (posDiff < posThreshold)
{
newPose.pos = oldPose.pos;
}
if (rotDiff < rotThreshold)
{
newPose.rot = oldPose.rot;
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces on the downscale image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
Mat downScaleRgbaMat = null;
float DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(rgbaMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = rgbaMat;
DOWNSCALE_RATIO = 1.0f;
}
// set the downscale mat
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, downScaleRgbaMat);
//detect face rects
detectionResult = faceLandmarkDetector.Detect();
if (enableDownScale)
{
for (int i = 0; i < detectionResult.Count; ++i)
{
var rect = detectionResult[i];
detectionResult[i] = new UnityEngine.Rect(
rect.x * DOWNSCALE_RATIO,
rect.y * DOWNSCALE_RATIO,
rect.width * DOWNSCALE_RATIO,
rect.height * DOWNSCALE_RATIO);
}
}
}
List <Vector2> points = null;
if (detectionResult != null && detectionResult.Count > 0)
{
// set the original scale image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(detectionResult[0]);
}
if (points != null)
{
if (displayFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
MatOfPoint3f objectPoints = null;
bool isRightEyeOpen = false;
bool isLeftEyeOpen = false;
bool isMouthOpen = false;
if (points.Count == 68)
{
objectPoints = objectPoints68;
imagePoints.fromArray(
new Point((points[38].x + points[41].x) / 2, (points[38].y + points[41].y) / 2), //l eye (Interpupillary breadth)
new Point((points[43].x + points[46].x) / 2, (points[43].y + points[46].y) / 2), //r eye (Interpupillary breadth)
new Point(points[30].x, points[30].y), //nose (Tip)
new Point(points[33].x, points[33].y), //nose (Subnasale)
new Point(points[0].x, points[0].y), //l ear (Bitragion breadth)
new Point(points[16].x, points[16].y) //r ear (Bitragion breadth)
);
if (Mathf.Abs((float)(points[43].y - points[46].y)) > Mathf.Abs((float)(points[42].x - points[45].x)) / 5.0)
{
isRightEyeOpen = true;
}
if (Mathf.Abs((float)(points[38].y - points[41].y)) > Mathf.Abs((float)(points[39].x - points[36].x)) / 5.0)
{
isLeftEyeOpen = true;
}
float noseDistance = Mathf.Abs((float)(points[27].y - points[33].y));
float mouseDistance = Mathf.Abs((float)(points[62].y - points[66].y));
if (mouseDistance > noseDistance / 5.0)
{
isMouthOpen = true;
}
else
{
isMouthOpen = false;
}
}
else if (points.Count == 17)
{
objectPoints = objectPoints17;
imagePoints.fromArray(
new Point((points[2].x + points[3].x) / 2, (points[2].y + points[3].y) / 2), //l eye (Interpupillary breadth)
new Point((points[4].x + points[5].x) / 2, (points[4].y + points[5].y) / 2), //r eye (Interpupillary breadth)
new Point(points[0].x, points[0].y), //nose (Tip)
new Point(points[1].x, points[1].y), //nose (Subnasale)
new Point(points[6].x, points[6].y), //l ear (Bitragion breadth)
new Point(points[8].x, points[8].y) //r ear (Bitragion breadth)
);
if (Mathf.Abs((float)(points[11].y - points[12].y)) > Mathf.Abs((float)(points[4].x - points[5].x)) / 5.0)
{
isRightEyeOpen = true;
}
if (Mathf.Abs((float)(points[9].y - points[10].y)) > Mathf.Abs((float)(points[2].x - points[3].x)) / 5.0)
{
isLeftEyeOpen = true;
}
float noseDistance = Mathf.Abs((float)(points[3].y - points[1].y));
float mouseDistance = Mathf.Abs((float)(points[14].y - points[16].y));
if (mouseDistance > noseDistance / 2.0)
{
isMouthOpen = true;
}
else
{
isMouthOpen = false;
}
}
else if (points.Count == 6)
{
objectPoints = objectPoints6;
imagePoints.fromArray(
new Point((points[2].x + points[3].x) / 2, (points[2].y + points[3].y) / 2), //l eye (Interpupillary breadth)
new Point((points[4].x + points[5].x) / 2, (points[4].y + points[5].y) / 2), //r eye (Interpupillary breadth)
new Point(points[0].x, points[0].y), //nose (Tip)
new Point(points[1].x, points[1].y) //nose (Subnasale)
);
}
else if (points.Count == 5)
{
objectPoints = objectPoints5;
imagePoints.fromArray(
new Point(points[3].x, points[3].y), //l eye (Inner corner of the eye)
new Point(points[1].x, points[1].y), //r eye (Inner corner of the eye)
new Point(points[2].x, points[2].y), //l eye (Tail of the eye)
new Point(points[0].x, points[0].y), //r eye (Tail of the eye)
new Point(points[4].x, points[4].y) //nose (Subnasale)
);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "This example supports mainly the face landmark points of 68/17/6 points.";
}
}
// estimate head pose
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_x = tvec.get(0, 0)[0], tvec_y = tvec.get(1, 0)[0], tvec_z = tvec.get(2, 0)[0];
bool isNotInViewport = false;
Vector4 pos = VP * new Vector4((float)tvec_x, (float)tvec_y, (float)tvec_z, 1.0f);
if (pos.w != 0)
{
float x = pos.x / pos.w, y = pos.y / pos.w, z = pos.z / pos.w;
if (x < -1.0f || x > 1.0f || y < -1.0f || y > 1.0f || z < -1.0f || z > 1.0f)
{
isNotInViewport = true;
}
}
if (double.IsNaN(tvec_z) || isNotInViewport)
{ // if tvec is wrong data, do not use extrinsic guesses. (the estimated object is not in the camera field of view)
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
//Debug.Log (tvec.dump());
if (!isNotInViewport)
{
if (displayHead)
{
head.SetActive(true);
}
if (displayAxes)
{
axes.SetActive(true);
}
if (displayEffects)
{
rightEye.SetActive(isRightEyeOpen);
leftEye.SetActive(isLeftEyeOpen);
if (isMouthOpen)
{
mouth.SetActive(true);
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = true;
#if UNITY_5_5_OR_NEWER
var main = ps.main;
main.startSizeMultiplier = 20;
#else
ps.startSize = 20;
#endif
}
}
else
{
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = false;
}
}
}
// Convert to unity pose data.
double[] rvecArr = new double[3];
rvec.get(0, 0, rvecArr);
double[] tvecArr = new double[3];
tvec.get(0, 0, tvecArr);
PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr);
// Changes in pos/rot below these thresholds are ignored.
if (enableLowPassFilter)
{
ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass);
}
oldPoseData = poseData;
// Create transform matrix.
transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one);
}
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
// https://stackoverflow.com/questions/30234945/change-handedness-of-a-row-major-4x4-transformation-matrix
ARM = invertYM * transformationM * invertYM;
// Apply Y-axis and Z-axis refletion matrix. (Adjust the posture of the AR object)
ARM = ARM * invertYM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (capture == null)
{
return;
}
//Loop play
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
if (detectResult.Count > 0)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]);
if (displayFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, points, new Scalar(0, 255, 0), 2);
}
MatOfPoint3f objectPoints = null;
bool isRightEyeOpen = false;
bool isLeftEyeOpen = false;
bool isMouthOpen = false;
if (points.Count == 68)
{
objectPoints = objectPoints68;
imagePoints.fromArray(
new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye (Interpupillary breadth)
new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye (Interpupillary breadth)
new Point(points [30].x, points [30].y), //nose (Nose top)
new Point(points [48].x, points [48].y), //l mouth (Mouth breadth)
new Point(points [54].x, points [54].y), //r mouth (Mouth breadth)
new Point(points [0].x, points [0].y), //l ear (Bitragion breadth)
new Point(points [16].x, points [16].y) //r ear (Bitragion breadth)
);
if (Mathf.Abs((float)(points [43].y - points [46].y)) > Mathf.Abs((float)(points [42].x - points [45].x)) / 5.0)
{
isRightEyeOpen = true;
}
if (Mathf.Abs((float)(points [38].y - points [41].y)) > Mathf.Abs((float)(points [39].x - points [36].x)) / 5.0)
{
isLeftEyeOpen = true;
}
float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y));
float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y));
if (mouseDistance > noseDistance / 5.0)
{
isMouthOpen = true;
}
else
{
isMouthOpen = false;
}
}
else if (points.Count == 5)
{
objectPoints = objectPoints5;
imagePoints.fromArray(
new Point(points [3].x, points [3].y), //l eye (Inner corner of the eye)
new Point(points [1].x, points [1].y), //r eye (Inner corner of the eye)
new Point(points [2].x, points [2].y), //l eye (Tail of the eye)
new Point(points [0].x, points [0].y), //r eye (Tail of the eye)
new Point(points [4].x, points [4].y) //nose (Nose top)
);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "This example supports mainly the face landmark points of 68 points.";
}
}
// Estimate head pose.
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_z = tvec.get(2, 0) [0];
if (double.IsNaN(tvec_z) || tvec_z < 0) // if tvec is wrong data, do not use extrinsic guesses.
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
//Debug.Log (tvec.dump());
if (!double.IsNaN(tvec_z))
{
// Display effects.
if (displayHead)
{
head.SetActive(true);
}
if (displayAxes)
{
axes.SetActive(true);
}
if (displayEffects)
{
rightEye.SetActive(isRightEyeOpen);
leftEye.SetActive(isLeftEyeOpen);
if (isMouthOpen)
{
mouth.SetActive(true);
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = true;
#if UNITY_5_5_OR_NEWER
var main = ps.main;
main.startSizeMultiplier = 20;
#else
ps.startSize = 20;
#endif
}
}
else
{
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = false;
}
}
}
// Convert to unity pose data.
double[] rvecArr = new double[3];
rvec.get(0, 0, rvecArr);
double[] tvecArr = new double[3];
tvec.get(0, 0, tvecArr);
PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr);
// Changes in pos/rot below these thresholds are ignored.
if (enableLowPassFilter)
{
ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass);
}
oldPoseData = poseData;
// Create transform matrix.
transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one);
}
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z-axis inverted matrix.
ARM = ARM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
}
else
{
rightEye.SetActive(false);
leftEye.SetActive(false);
head.SetActive(false);
mouth.SetActive(false);
axes.SetActive(false);
}
//Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbMat, texture);
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
if (detectResult.Count > 0)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]);
if (displayFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
MatOfPoint3f objectPoints = null;
if (points.Count == 68)
{
objectPoints = objectPoints68;
imagePoints.fromArray(
new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye (Interpupillary breadth)
new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye (Interpupillary breadth)
new Point(points [30].x, points [30].y), //nose (Tip)
new Point(points [33].x, points [33].y), //nose (Subnasale)
new Point(points [0].x, points [0].y), //l ear (Bitragion breadth)
new Point(points [16].x, points [16].y) //r ear (Bitragion breadth)
);
float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y));
float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y));
}
else if (points.Count == 17)
{
objectPoints = objectPoints17;
imagePoints.fromArray(
new Point((points [2].x + points [3].x) / 2, (points [2].y + points [3].y) / 2), //l eye (Interpupillary breadth)
new Point((points [4].x + points [5].x) / 2, (points [4].y + points [5].y) / 2), //r eye (Interpupillary breadth)
new Point(points [0].x, points [0].y), //nose (Tip)
new Point(points [1].x, points [1].y), //nose (Subnasale)
new Point(points [6].x, points [6].y), //l ear (Bitragion breadth)
new Point(points [8].x, points [8].y) //r ear (Bitragion breadth)
);
float noseDistance = Mathf.Abs((float)(points [3].y - points [1].y));
float mouseDistance = Mathf.Abs((float)(points [14].y - points [16].y));
}
else if (points.Count == 6)
{
objectPoints = objectPoints6;
imagePoints.fromArray(
new Point((points [2].x + points [3].x) / 2, (points [2].y + points [3].y) / 2), //l eye (Interpupillary breadth)
new Point((points [4].x + points [5].x) / 2, (points [4].y + points [5].y) / 2), //r eye (Interpupillary breadth)
new Point(points [0].x, points [0].y), //nose (Tip)
new Point(points [1].x, points [1].y) //nose (Subnasale)
);
}
else if (points.Count == 5)
{
objectPoints = objectPoints5;
imagePoints.fromArray(
new Point(points [3].x, points [3].y), //l eye (Inner corner of the eye)
new Point(points [1].x, points [1].y), //r eye (Inner corner of the eye)
new Point(points [2].x, points [2].y), //l eye (Tail of the eye)
new Point(points [0].x, points [0].y), //r eye (Tail of the eye)
new Point(points [4].x, points [4].y) //nose (Subnasale)
);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "This example supports mainly the face landmark points of 68/17/6 points.";
}
}
// estimate head pose
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_x = tvec.get(0, 0) [0], tvec_y = tvec.get(1, 0) [0], tvec_z = tvec.get(2, 0) [0];
bool isNotInViewport = false;
Vector4 pos = VP * new Vector4((float)tvec_x, (float)tvec_y, (float)tvec_z, 1.0f);
if (pos.w != 0)
{
float x = pos.x / pos.w, y = pos.y / pos.w, z = pos.z / pos.w;
if (x < -1.0f || x > 1.0f || y < -1.0f || y > 1.0f || z < -1.0f || z > 1.0f)
{
isNotInViewport = true;
}
}
if (double.IsNaN(tvec_z) || isNotInViewport) // if tvec is wrong data, do not use extrinsic guesses. (the estimated object is not in the camera field of view)
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
if (!isNotInViewport)
{
if (displayHead)
{
head.SetActive(true);
}
if (displayAxes)
{
axes.SetActive(true);
}
// Convert to unity pose data.
double[] rvecArr = new double[3];
rvec.get(0, 0, rvecArr);
double[] tvecArr = new double[3];
tvec.get(0, 0, tvecArr);
PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr);
// Changes in pos/rot below these thresholds are ignored.
if (enableLowPassFilter)
{
ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass);
}
oldPoseData = poseData;
// Create transform matrix.
transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one);
//move joystick
float t = 0.50f;
int xpos;
if ((poseData.rot.eulerAngles.x - oldPoseData.rot.eulerAngles.x) < 10 && (poseData.rot.eulerAngles.x - oldPoseData.rot.eulerAngles.x) > 0)
{
xpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.x, oldPoseData.rot.eulerAngles.x, t));
}
else
{
xpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.x, oldPoseData.rot.eulerAngles.x, 1));
}
int ypos;
if ((poseData.rot.eulerAngles.y - oldPoseData.rot.eulerAngles.y) < 10 && (poseData.rot.eulerAngles.y - oldPoseData.rot.eulerAngles.y) > 0)
{
ypos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.y, oldPoseData.rot.eulerAngles.y, t));
}
else
{
ypos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.y, oldPoseData.rot.eulerAngles.y, 1));
}
int rpos;
if ((poseData.rot.eulerAngles.z - oldPoseData.rot.eulerAngles.z) < 10 && (poseData.rot.eulerAngles.z - oldPoseData.rot.eulerAngles.z) > 0)
{
rpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.z, oldPoseData.rot.eulerAngles.z, t));
}
else
{
rpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.z, oldPoseData.rot.eulerAngles.z, 1));
}
//int ypos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.y, oldPoseData.rot.eulerAngles.y,t));
int zpos = Convert.ToInt16(poseData.pos.z);
//zoom
int scZ = Convert.ToInt32(Scale(zpos, 400, 800, 0, 32000));
//up down
int scX;
if (xpos >= 0 && xpos < 20)
{
scX = Convert.ToInt32(Scale(xpos, 0, 10, 29000, 32000));
}
else
{
scX = Convert.ToInt32(Scale(xpos, 340, 360, 0, 29000));
}
//left right
int scY = Convert.ToInt32(Scale(ypos, 160, 200, 0, 32000));
//roll
int scR = Convert.ToInt32(Scale(rpos, 160, 200, 0, 32000));
iReport.AxisX = scX;
iReport.AxisY = scY;
iReport.AxisZ = scZ;
iReport.Slider = scR;
bool upd = joystick.UpdateVJD(id, ref iReport);
string message = "x:" + scX + " y:" + scY + " z:" + scZ + " r:" + scR;
Debug.Log(message);
}
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z-axis inverted matrix.
ARM = ARM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
