/// <summary>
        /// Create any resources needed for emulation.
        /// </summary>
        private static void _InitializeResourcesForEmulation()
        {
            // Create textures:

            m_emulatedColorRenderTexture = new RenderTexture(EMULATED_CAMERA_WIDTH,
                                                             EMULATED_CAMERA_HEIGHT,
                                                             24, RenderTextureFormat.ARGB32);

            m_emulationByteBufferCaptureTextures    = new Texture2D[2];
            m_emulationByteBufferCaptureTextures[0] = new Texture2D(EMULATED_CAMERA_PACKED_WIDTH,
                                                                    EMULATED_CAMERA_PACKED_Y_HEIGHT,
                                                                    TextureFormat.ARGB32, false);
            m_emulationByteBufferCaptureTextures[1] = new Texture2D(EMULATED_CAMERA_PACKED_WIDTH,
                                                                    EMULATED_CAMERA_PACKED_UV_HEIGHT,
                                                                    TextureFormat.ARGB32, false);

            m_emulatedExpId_Y = new RenderTexture(EMULATED_CAMERA_PACKED_WIDTH,
                                                  EMULATED_CAMERA_PACKED_Y_HEIGHT,
                                                  0, RenderTextureFormat.ARGB32);
            m_emulatedExpId_CbCr = new RenderTexture(EMULATED_CAMERA_PACKED_WIDTH,
                                                     EMULATED_CAMERA_PACKED_UV_HEIGHT,
                                                     0, RenderTextureFormat.ARGB32);
            m_emulatedExpId_Y.filterMode    = FilterMode.Point;
            m_emulatedExpId_CbCr.filterMode = FilterMode.Point;
            m_emulatedExpId_Y.Create();
            m_emulatedExpId_CbCr.Create();

            // Find shaders by searching for them:
            if (EmulatedEnvironmentRenderHelper.CreateMaterialFromShaderName(EMULATED_RGB2YUV_Y_SHADERNAME,
                                                                             out m_yuvFilterY))
            {
                m_yuvFilterY.SetFloat("_TexWidth", EMULATED_CAMERA_WIDTH);
            }
            else
            {
                Debug.LogError("Could not find shader "
                               + EMULATED_RGB2YUV_Y_SHADERNAME
                               + ". Tango color camera emulation will not work correctly.");
            }

            if (EmulatedEnvironmentRenderHelper.CreateMaterialFromShaderName(EMULATED_RGB2YUV_CBCR_SHADERNAME,
                                                                             out m_yuvFilterCbCr))
            {
                m_yuvFilterCbCr.SetFloat("_TexWidth", EMULATED_CAMERA_WIDTH);
            }
            else
            {
                Debug.LogError("Could not find shader "
                               + EMULATED_RGB2YUV_CBCR_SHADERNAME
                               + ". Tango color camera emulation will not work correctly.");
            }
        }
        /// <summary>
        /// INTERNAL USE: Update the Tango emulation state for color camera data.
        /// </summary>
        /// <param name="useByteBufferMethod">Whether to update emulation for byte-buffer method.</param>
        internal static void UpdateTangoEmulation(bool useByteBufferMethod)
        {
            // Get emulated position and rotation in Unity space.
            TangoPoseData            poseData = new TangoPoseData();
            TangoCoordinateFramePair pair;

            pair.baseFrame   = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_START_OF_SERVICE;
            pair.targetFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_DEVICE;

            if (!PoseProvider.GetTimestampForColorEmulation(out m_lastColorEmulationTime))
            {
                Debug.LogError("Couldn't get a valid timestamp with which to emulate color camera. "
                               + "Color camera emulation will be skipped this frame.");
                return;
            }

            PoseProvider.GetPoseAtTime(poseData, m_lastColorEmulationTime, pair);
            if (poseData.status_code != TangoEnums.TangoPoseStatusType.TANGO_POSE_VALID)
            {
                return;
            }

            Vector3    position;
            Quaternion rotation;

            TangoSupport.TangoPoseToWorldTransform(poseData, out position, out rotation);

            // Instantiate any resources that we haven't yet.
            if (!m_emulationIsInitialized)
            {
                _InitializeResourcesForEmulation();
                m_emulationIsInitialized = true;
            }

            // Render.
            EmulatedEnvironmentRenderHelper.RenderEmulatedEnvironment(m_emulatedColorRenderTexture,
                                                                      EmulatedEnvironmentRenderHelper.EmulatedDataType.COLOR_CAMERA,
                                                                      position, rotation);

            m_emulationIsDirty = true;
        }
Exemplo n.º 3
0
        /// <summary>
        /// INTERNAL USE: Update the Tango emulation state for depth data.
        /// </summary>
        internal static void UpdateTangoEmulation()
        {
            m_emulatedPointCloud.Clear();

            // Timestamp shall be something in the past, and we'll emulate the depth cloud based on it.
            if (!PoseProvider.GetTimestampForDepthEmulation(out m_lastDepthEmulationTime))
            {
                Debug.LogError("Couldn't get a valid timestamp with which to emulate depth data. "
                               + "Depth emulation will be skipped this frame.");
                return;
            }

            // Get emulated position and rotation in Unity space.
            TangoPoseData            poseData = new TangoPoseData();
            TangoCoordinateFramePair pair;

            pair.baseFrame   = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_START_OF_SERVICE;
            pair.targetFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_DEVICE;

            PoseProvider.GetPoseAtTime(poseData, m_lastDepthEmulationTime, pair);
            if (poseData.status_code != TangoEnums.TangoPoseStatusType.TANGO_POSE_VALID)
            {
                return;
            }

            Vector3    position;
            Quaternion rotation;

            TangoSupport.TangoPoseToWorldTransform(poseData, out position, out rotation);

            // Instantiate any resources that we haven't yet.
            _InternResourcesForEmulation();

            // Render emulated depth camera data.
            EmulatedEnvironmentRenderHelper.RenderEmulatedEnvironment(m_emulatedDepthTexture,
                                                                      EmulatedEnvironmentRenderHelper.EmulatedDataType.DEPTH,
                                                                      position, rotation);

            // Capture rendered depth points from texture.
            RenderTexture.active = m_emulatedDepthTexture;
            m_emulationCaptureTexture.ReadPixels(new Rect(0, 0, m_emulatedDepthTexture.width, m_emulatedDepthTexture.height), 0, 0);
            m_emulationCaptureTexture.Apply();

            // Exctract captured data.
            Color32[] depthDataAsColors = m_emulationCaptureTexture.GetPixels32();

            // Convert depth texture to positions in camera space.
            Matrix4x4 projectionMatrix        = GL.GetGPUProjectionMatrix(EmulatedEnvironmentRenderHelper.m_emulationCamera.projectionMatrix, false);
            Matrix4x4 reverseProjectionMatrix = projectionMatrix.inverse;

            float width  = m_emulationCaptureTexture.width;
            float height = m_emulationCaptureTexture.height;

            for (int yTexel = 0; yTexel < height; yTexel++)
            {
                for (int xTexel = 0; xTexel < width; xTexel++)
                {
                    Color32 depthAsColor = depthDataAsColors[xTexel + (yTexel * m_emulationCaptureTexture.width)];
                    float   clipSpaceZ   = (depthAsColor.r - 128f) + (depthAsColor.g / 255f);

                    float ndcSpaceZ          = (clipSpaceZ - projectionMatrix.m23) / projectionMatrix.m22;
                    float perspectiveDivisor = ndcSpaceZ * projectionMatrix.m32;

                    float ndcSpaceX = (((xTexel + 0.5f) / width) * 2f) - 1;
                    float ndcSpaceY = (((yTexel + 0.5f) / height) * 2f) - 1;

                    Vector4 clipSpacePos = new Vector4(ndcSpaceX * perspectiveDivisor, ndcSpaceY * perspectiveDivisor, clipSpaceZ, perspectiveDivisor);
                    Vector4 viewSpacePos = reverseProjectionMatrix * clipSpacePos;

                    Vector3 emulatedDepthPos = new Vector3(viewSpacePos.x, -viewSpacePos.y, -viewSpacePos.z);

                    if (emulatedDepthPos.z > MIN_POINT_DISTANCE && emulatedDepthPos.z < MAX_POINT_DISTANCE)
                    {
                        m_emulatedPointCloud.Add(emulatedDepthPos);
                    }
                }
            }

            m_emulationIsDirty = true;
        }
Exemplo n.º 4
0
        /// <summary>
        /// Create any resources needed for emulation.
        /// </summary>
        private static void _InternResourcesForEmulation()
        {
            if (m_emulationIsInitialized)
            {
                return;
            }

            // Create textures:

            m_emulatedColorRenderTexture = new RenderTexture(EMULATED_CAMERA_WIDTH, EMULATED_CAMERA_HEIGHT, 24,
                                                             RenderTextureFormat.ARGB32);
            m_emulatedARScreenTexture = new RenderTexture(EMULATED_CAMERA_WIDTH, EMULATED_CAMERA_HEIGHT, 0,
                                                          RenderTextureFormat.ARGB32);

            m_emulationByteBufferCaptureTextures    = new Texture2D[2];
            m_emulationByteBufferCaptureTextures[0] = new Texture2D(EMULATED_CAMERA_PACKED_WIDTH,
                                                                    EMULATED_CAMERA_PACKED_Y_HEIGHT,
                                                                    TextureFormat.ARGB32, false);
            m_emulationByteBufferCaptureTextures[1] = new Texture2D(EMULATED_CAMERA_PACKED_WIDTH,
                                                                    EMULATED_CAMERA_PACKED_UV_HEIGHT,
                                                                    TextureFormat.ARGB32, false);

            // Find shaders by searching for them:
            if (EmulatedEnvironmentRenderHelper.CreateMaterialFromShaderName(EMULATED_RGB2YUV_Y_SHADERNAME,
                                                                             out m_yuvFilterY))
            {
                m_yuvFilterY.SetFloat("_TexWidth", EMULATED_CAMERA_WIDTH);
            }
            else
            {
                Debug.LogError("Could not find shader "
                               + EMULATED_RGB2YUV_Y_SHADERNAME
                               + ". Tango color camera emulation will not work correctly.");
            }

            if (EmulatedEnvironmentRenderHelper.CreateMaterialFromShaderName(EMULATED_RGB2YUV_CBCR_SHADERNAME,
                                                                             out m_yuvFilterCbCr))
            {
                m_yuvFilterCbCr.SetFloat("_TexWidth", EMULATED_CAMERA_WIDTH);
            }
            else
            {
                Debug.LogError("Could not find shader "
                               + EMULATED_RGB2YUV_CBCR_SHADERNAME
                               + ". Tango color camera emulation will not work correctly.");
            }

            if (EmulatedEnvironmentRenderHelper.CreateMaterialFromShaderName(EMULATED_RGB_ARSCREEN_SHADERNAME,
                                                                             out m_emulationArScreenMaterial))
            {
                m_emulationArScreenMaterial.SetVector("_UVBottomLeft", new Vector4(0, 0, 0, 0));
                m_emulationArScreenMaterial.SetVector("_UVBottomRight", new Vector4(0, 1, 0, 0));
                m_emulationArScreenMaterial.SetVector("_UVTopLeft", new Vector4(1, 0, 0, 0));
                m_emulationArScreenMaterial.SetVector("_UVTopRight", new Vector4(1, 1, 0, 0));
            }
            else
            {
                Debug.LogError("Could not find shader "
                               + EMULATED_RGB_ARSCREEN_SHADERNAME
                               + ". Tango color camera emulation will not work correctly.");
            }

            m_emulationIsInitialized = true;
        }
        /// <summary>
        /// INTERNAL USE: Update the Tango emulation state for depth data.
        ///
        /// Make this this is only called once per frame.
        /// </summary>
        internal static void UpdateTangoEmulation()
        {
            m_emulatedPointCloud.Clear();

            // Timestamp shall be something in the past, and we'll emulate the depth cloud based on it.
            m_lastDepthEmulationTime = PoseProvider.GetTimestampForDepthEmulation();

            // Get emulated position and rotation in Unity space.
            TangoPoseData            poseData = new TangoPoseData();
            TangoCoordinateFramePair pair;

            pair.baseFrame   = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_START_OF_SERVICE;
            pair.targetFrame = TangoEnums.TangoCoordinateFrameType.TANGO_COORDINATE_FRAME_DEVICE;

            PoseProvider.GetPoseAtTime(poseData, m_lastDepthEmulationTime, pair);
            if (poseData.status_code != TangoEnums.TangoPoseStatusType.TANGO_POSE_VALID)
            {
                return;
            }

            Vector3    position;
            Quaternion rotation;

            TangoSupport.TangoPoseToWorldTransform(poseData, out position, out rotation);

            // Instantiate any resources that we haven't yet.
            if (m_emulatedDepthTexture == null)
            {
                m_emulatedDepthTexture = new RenderTexture(NUM_X_DEPTH_SAMPLES, NUM_Y_DEPTH_SAMPLES, 24, RenderTextureFormat.ARGB32);
            }

            if (m_emulationCaptureTexture == null)
            {
                m_emulationCaptureTexture = new Texture2D(NUM_X_DEPTH_SAMPLES, NUM_Y_DEPTH_SAMPLES, TextureFormat.ARGB32, false);
            }

            if (m_emulatedDepthShader == null)
            {
                // Find depth shader by searching for it in project.
                string[] foundAssetGuids = UnityEditor.AssetDatabase.FindAssets("DepthEmulation t:Shader");
                if (foundAssetGuids.Length > 0)
                {
                    string assetPath = UnityEditor.AssetDatabase.GUIDToAssetPath(foundAssetGuids[0]);
                    m_emulatedDepthShader = UnityEditor.AssetDatabase.LoadAssetAtPath(assetPath, typeof(Shader)) as Shader;
                }
            }

            // Render emulated depth camera data.
            EmulatedEnvironmentRenderHelper.RenderEmulatedEnvironment(m_emulatedDepthTexture, m_emulatedDepthShader,
                                                                      position, rotation);

            // Capture rendered depth points from texture.
            RenderTexture.active = m_emulatedDepthTexture;
            m_emulationCaptureTexture.ReadPixels(new Rect(0, 0, m_emulatedDepthTexture.width, m_emulatedDepthTexture.height), 0, 0);
            m_emulationCaptureTexture.Apply();

            // Exctract captured data.
            Color32[] depthDataAsColors = m_emulationCaptureTexture.GetPixels32();

            // Convert depth texture to positions in camera space.
            Matrix4x4 projectionMatrix        = GL.GetGPUProjectionMatrix(EmulatedEnvironmentRenderHelper.m_emulationCamera.projectionMatrix, false);
            Matrix4x4 reverseProjectionMatrix = projectionMatrix.inverse;

            float width  = m_emulationCaptureTexture.width;
            float height = m_emulationCaptureTexture.height;

            for (int yTexel = 0; yTexel < height; yTexel++)
            {
                for (int xTexel = 0; xTexel < width; xTexel++)
                {
                    Color32 depthAsColor = depthDataAsColors[xTexel + (yTexel * m_emulationCaptureTexture.width)];
                    float   clipSpaceZ   = (depthAsColor.r - 128f) + (depthAsColor.g / 255f);

                    float ndcSpaceZ          = (clipSpaceZ - projectionMatrix.m23) / projectionMatrix.m22;
                    float perspectiveDivisor = ndcSpaceZ * projectionMatrix.m32;

                    float ndcSpaceX = (((xTexel + 0.5f) / width) * 2f) - 1;
                    float ndcSpaceY = (((yTexel + 0.5f) / height) * 2f) - 1;

                    Vector4 clipSpacePos = new Vector4(ndcSpaceX * perspectiveDivisor, ndcSpaceY * perspectiveDivisor, clipSpaceZ, perspectiveDivisor);
                    Vector4 viewSpacePos = reverseProjectionMatrix * clipSpacePos;

                    Vector3 emulatedDepthPos = new Vector3(viewSpacePos.x, -viewSpacePos.y, -viewSpacePos.z);

                    if (emulatedDepthPos.z > MIN_POINT_DISTANCE && emulatedDepthPos.z < MAX_POINT_DISTANCE)
                    {
                        m_emulatedPointCloud.Add(emulatedDepthPos);
                    }
                }
            }
        }