/// <summary>
/// Estimates the depth of a point on a screen, based on nearest neighbors.
/// </summary>
/// <returns>
/// <c>true</c> if a successful depth estimate was obtained.
/// </returns>
/// <param name="cam">The Unity camera.</param>
/// <param name="pos">The point in pixel coordinates to perform depth estimation.</param>
/// <param name="colorCameraPoint">
/// The point (x, y, z), where (x, y) is the back-projection of the UV
/// coordinates to the color camera space and z is the z coordinate of
/// the point in the point cloud nearest to the user selection after
/// projection onto the image plane. If there is not a point cloud point
/// close to the user selection after projection onto the image plane,
/// then the point will be set to (0.0, 0.0, 0.0) and isValidPoint will
/// be set to false.
/// </param>
public bool EstimateDepthOnScreen(Camera cam, Vector2 pos, out Vector3 colorCameraPoint)
{
// Set up parameters
Matrix4x4 colorCameraTUnityWorld = m_colorCameraTUnityCamera.ToMatrix4x4() * cam.transform.worldToLocalMatrix;
Vector2 normalizedPos = cam.ScreenToViewportPoint(pos);
// If the camera has a TangoARScreen attached, it is not displaying the entire color camera image. Correct
// the normalized coordinates by taking the clipping into account.
TangoARScreen arScreen = cam.gameObject.GetComponent <TangoARScreen>();
if (arScreen != null)
{
normalizedPos = arScreen.ViewportPointToCameraImagePoint(normalizedPos);
}
bool isValidPoint;
int returnType = TangoSupport.ScreenCoordinateToWorldNearestNeighbor(
m_points,
m_pointsCount,
m_depthTimestamp,
m_colorCameraIntrinsics,
ref colorCameraTUnityWorld,
normalizedPos,
out colorCameraPoint,
out isValidPoint);
return((returnType == Common.ErrorType.TANGO_SUCCESS) && isValidPoint);
}
/// <summary>
/// Set controller's transformation based on received pose.
/// </summary>
/// <param name="pose">Received Tango pose data.</param>
private void _UpdateTransformationFromPose(TangoPoseData pose)
{
// Remember the previous position so you can do delta motion.
m_prevTangoPosition = m_tangoPosition;
m_prevTangoRotation = m_tangoRotation;
// The callback pose is for device with respect to start of service pose.
if (pose.status_code == TangoEnums.TangoPoseStatusType.TANGO_POSE_VALID)
{
DMatrix4x4 globalTLocal;
bool success = m_tangoApplication.GetGlobalTLocal(out globalTLocal);
if (!success)
{
Debug.LogError("Unable to obtain GlobalTLocal from Tango application.");
return;
}
DMatrix4x4 startOfServiceTDevice = globalTLocal.Inverse * DMatrix4x4.TR(pose.translation, pose.orientation);
m_uwTuc = TangoSupport.UNITY_WORLD_T_START_SERVICE * startOfServiceTDevice.ToMatrix4x4()
* TangoSupport.DEVICE_T_UNITY_CAMERA * TangoSupport.m_devicePoseRotation;
Matrix4x4 uwOffsetTuc = m_uwOffsetTuw * m_uwTuc;
m_tangoPosition = uwOffsetTuc.GetColumn(3);
// apply scale for Daydream
m_tangoPosition.x *= m_motionMappingScaleXZ;
m_tangoPosition.z *= m_motionMappingScaleXZ;
m_tangoPosition.y *= m_motionMappingScaleY;
m_tangoRotation = Quaternion.LookRotation(uwOffsetTuc.GetColumn(2), uwOffsetTuc.GetColumn(1));
// Other pose data: pose count gets reset if pose status just became valid.
if (pose.status_code != m_poseStatus)
{
m_poseCount = 0;
}
m_poseCount++;
// Other pose data: pose delta time.
m_poseDeltaTime = (float)pose.timestamp - m_poseTimestamp;
m_poseTimestamp = (float)pose.timestamp;
}
m_poseStatus = pose.status_code;
if (m_clutchActive)
{
// When clutching, preserve position.
m_tangoPosition = m_prevTangoPosition;
// When clutching, preserve yaw, keep changes in pitch, roll.
Vector3 rotationAngles = m_tangoRotation.eulerAngles;
rotationAngles.y = m_prevTangoRotation.eulerAngles.y;
m_tangoRotation.eulerAngles = rotationAngles;
}
// edit: following https://blogs.unity3d.com/jp/2017/10/18/mobile-inside-out-vr-tracking-now-readily-available-on-your-phone-with-unity/
// Calculate final position and rotation deltas and apply them.
Vector3 deltaPosition = m_tangoPosition - m_prevTangoPosition;
Quaternion deltaRotation = m_tangoRotation * Quaternion.Inverse(m_prevTangoRotation);
if (m_characterMotion && m_characterController != null)
{
m_characterController.Move(deltaPosition);
//transform.rotation = deltaRotation * transform.rotation;
}
else
{
transform.position = transform.position + deltaPosition;
//transform.rotation = deltaRotation * transform.rotation;
}
}
/// <summary>
/// Callback that gets called when depth is available from the Tango Service.
/// </summary>
/// <param name="pointCloud">Depth information from Tango.</param>
public void OnTangoPointCloudAvailable(TangoPointCloudData pointCloud)
{
m_mostRecentPointCloud = pointCloud;
// Calculate the time since the last successful depth data
// collection.
if (m_depthTimestamp != 0.0)
{
m_depthDeltaTime = (float)((pointCloud.m_timestamp - m_depthTimestamp) * 1000.0);
}
// Fill in the data to draw the point cloud.
m_pointsCount = pointCloud.m_numPoints;
if (m_pointsCount > 0)
{
_SetUpCameraData();
DMatrix4x4 globalTLocal;
bool globalTLocalSuccess = m_tangoApplication.GetGlobalTLocal(out globalTLocal);
if (!globalTLocalSuccess)
{
return;
}
DMatrix4x4 unityWorldTGlobal = DMatrix4x4.FromMatrix4x4(TangoSupport.UNITY_WORLD_T_START_SERVICE) * globalTLocal.Inverse;
TangoPoseData poseData;
// Query pose to transform point cloud to world coordinates, here we are using the timestamp that we get from depth.
bool poseSuccess = _GetDevicePose(pointCloud.m_timestamp, out poseData);
if (!poseSuccess)
{
return;
}
DMatrix4x4 unityWorldTDevice = unityWorldTGlobal * DMatrix4x4.TR(poseData.translation, poseData.orientation);
// The transformation matrix that represents the point cloud's pose.
// Explanation:
// The point cloud, which is in Depth camera's frame, is put in Unity world's
// coordinate system(wrt Unity world).
// Then we are extracting the position and rotation from uwTuc matrix and applying it to
// the point cloud's transform.
DMatrix4x4 unityWorldTDepthCamera = unityWorldTDevice * m_deviceTDepthCamera;
transform.position = Vector3.zero;
transform.rotation = Quaternion.identity;
// Add offset to the point cloud depending on the offset from TangoDeltaPoseController.
if (m_tangoDeltaPoseController != null)
{
m_mostRecentUnityWorldTDepthCamera = m_tangoDeltaPoseController.UnityWorldOffset * unityWorldTDepthCamera.ToMatrix4x4();
}
else
{
m_mostRecentUnityWorldTDepthCamera = unityWorldTDepthCamera.ToMatrix4x4();
}
// Converting points array to world space.
m_overallZ = 0;
for (int i = 0; i < m_pointsCount; ++i)
{
Vector3 point = pointCloud[i];
m_points[i] = m_mostRecentUnityWorldTDepthCamera.MultiplyPoint3x4(point);
m_overallZ += point.z;
}
m_overallZ = m_overallZ / m_pointsCount;
m_depthTimestamp = pointCloud.m_timestamp;
if (m_updatePointsMesh)
{
// Need to update indices too!
int[] indices = new int[m_pointsCount];
for (int i = 0; i < m_pointsCount; ++i)
{
indices[i] = i;
}
m_mesh.Clear();
m_mesh.vertices = m_points;
m_mesh.SetIndices(indices, MeshTopology.Points, 0);
}
// The color should be pose relative; we need to store enough info to go back to pose values.
m_renderer.material.SetMatrix("depthCameraTUnityWorld", m_mostRecentUnityWorldTDepthCamera.inverse);
// Try to find the floor using this set of depth points if requested.
if (m_findFloorWithDepth)
{
_FindFloorWithDepth();
}
}
else
{
m_overallZ = 0;
}
}
/// <summary>
/// Updates the transformation to the latest pose.
/// </summary>
private void _UpdatePose()
{
// Query a new pose.
TangoPoseData pose = new TangoPoseData();
double queryTimestamp = IsTargetingColorCamera ? m_tangoARScreen.m_screenUpdateTime : 0.0f;
PoseProvider.GetPoseAtTime(pose, queryTimestamp, _GetFramePair());
// Do not update with invalide poses.
if (pose.status_code != TangoEnums.TangoPoseStatusType.TANGO_POSE_VALID)
{
return;
}
// Do not update if the last update was for the same timestamp.
if (pose.timestamp == LastPoseTimestamp)
{
return;
}
LastPoseTimestamp = pose.timestamp;
DMatrix4x4 globalTLocal;
if (!m_tangoApplication.GetGlobalTLocal(out globalTLocal))
{
Debug.LogError("Unable to obtain GlobalTLocal from TangoApplication.");
return;
}
DMatrix4x4 unityWorld_T_device =
DMatrix4x4.FromMatrix4x4(TangoSupport.UNITY_WORLD_T_START_SERVICE) * globalTLocal.Inverse * DMatrix4x4.TR(pose.translation, pose.orientation);
// Calculate matrix for the camera in the Unity world
if (IsTargetingColorCamera)
{
m_unityWorld_T_unityCamera =
unityWorld_T_device.ToMatrix4x4() * TangoSupport.COLOR_CAMERA_T_UNITY_CAMERA * TangoSupport.m_colorCameraPoseRotation;
}
else
{
m_unityWorld_T_unityCamera =
unityWorld_T_device.ToMatrix4x4() * TangoSupport.DEVICE_T_UNITY_CAMERA * TangoSupport.m_devicePoseRotation;
}
// Extract final position and rotation from matrix.
Matrix4x4 unityWorldOffset_T_unityCamera = m_unityWorldTransformOffset_T_unityWorld * m_unityWorld_T_unityCamera;
Vector3 finalPosition = unityWorldOffset_T_unityCamera.GetColumn(3);
Quaternion finalRotation = Quaternion.LookRotation(unityWorldOffset_T_unityCamera.GetColumn(2), unityWorldOffset_T_unityCamera.GetColumn(1));
// Filter out yaw if the clutch is enabled.
if (m_clutchEnabled)
{
finalPosition = transform.position;
finalRotation = Quaternion.Euler(finalRotation.eulerAngles.x, transform.eulerAngles.y, finalRotation.eulerAngles.z);
}
// Apply the final position.
if (m_characterController)
{
m_characterController.Move(finalPosition - transform.position);
}
else
{
transform.position = finalPosition;
}
transform.rotation = finalRotation;
}
/// @endcond
/// <summary>
/// Updates the transformation to the pose for that timestamp.
/// </summary>
/// <param name="timestamp">Time in seconds to update the transformation to.</param>
private void _UpdateTransformation(double timestamp)
{
TangoPoseData pose = new TangoPoseData();
TangoCoordinateFramePair pair;
// Choose the proper pair according to the properties of this controller
if (m_useAreaDescriptionPose)
{
if (m_tangoApplication.m_enableCloudADF)
{
pair.baseFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_GLOBAL_WGS84;
pair.targetFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_DEVICE;
}
else
{
pair.baseFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_AREA_DESCRIPTION;
pair.targetFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_DEVICE;
}
}
else
{
pair.baseFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_START_OF_SERVICE;
pair.targetFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_DEVICE;
}
PoseProvider.GetPoseAtTime(pose, timestamp, pair);
// Update properties from pose
if (pose.status_code == TangoEnums.TangoPoseStatusType.TANGO_POSE_VALID)
{
DMatrix4x4 globalTLocal;
bool success = m_tangoApplication.GetGlobalTLocal(out globalTLocal);
if (!success)
{
Debug.LogError("Unable to obtain GlobalTLocal from TangoApplication.");
return;
}
DMatrix4x4 uwTDevice = DMatrix4x4.FromMatrix4x4(TangoSupport.UNITY_WORLD_T_START_SERVICE) *
globalTLocal.Inverse * DMatrix4x4.TR(pose.translation, pose.orientation);
// Calculate matrix for the camera in the Unity world, taking into account offsets.
Matrix4x4 uwTuc = uwTDevice.ToMatrix4x4() * m_dTuc * TangoSupport.m_colorCameraPoseRotation;
// Extract final position, rotation.
m_tangoPosition = uwTuc.GetColumn(3);
m_tangoRotation = Quaternion.LookRotation(uwTuc.GetColumn(2), uwTuc.GetColumn(1));
// Other pose data -- Pose count gets reset if pose status just became valid.
if (pose.status_code != m_poseStatus)
{
m_poseCount = 0;
}
m_poseCount++;
// Other pose data -- Pose time.
m_poseTimestamp = timestamp;
}
m_poseStatus = pose.status_code;
// Apply final position and rotation.
transform.position = m_tangoPosition;
transform.rotation = m_tangoRotation;
}