Example #1
0
        void RenderLoop()
        {
            while (true)
            {
                lock (renderLock)
                {
                    var deviceContext = device.ImmediateContext;

                    // render user view
                    deviceContext.ClearRenderTargetView(userViewRenderTargetView, Color4.Black);
                    deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0);

                    SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f);  // may need to change this default

                    if (localHeadTrackingEnabled)
                    {
                        float distanceSquared = 0;
                        lock (headCameraSpacePointLock)
                        {
                            headPosition = new SharpDX.Vector3(headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z);

                            float dx = handLeftCameraSpacePoint.X - handRightCameraSpacePoint.X;
                            float dy = handLeftCameraSpacePoint.Y - handRightCameraSpacePoint.Y;
                            float dz = handLeftCameraSpacePoint.Z - handRightCameraSpacePoint.Z;
                            distanceSquared = dx * dx + dy * dy + dz * dz;
                        }
                        var transform = SharpDX.Matrix.RotationY((float)Math.PI) * SharpDX.Matrix.Translation(-0.25f, 0.45f, 0);
                        headPosition = SharpDX.Vector3.TransformCoordinate(headPosition, transform);

                        if (trackingValid && (distanceSquared < 0.02f) && (alpha > 1))
                        {
                            alpha = 0;
                        }
                        //Console.WriteLine(distanceSquared);
                    }

                    var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY);
                    userView.Transpose();


                    //Console.WriteLine("headPosition = " + headPosition);


                    float aspect         = (float)userViewTextureWidth / (float)userViewTextureHeight;
                    var   userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f);
                    userProjection.Transpose();

                    // smooth depth images
                    foreach (var camera in ensemble.cameras)
                    {
                        var cameraDeviceResource = cameraDeviceResources[camera];
                        if (cameraDeviceResource.depthImageChanged)
                        {
                            fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView);
                            for (int i = 0; i < 1; i++)
                            {
                                bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2);
                                bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView);
                            }
                            cameraDeviceResource.depthImageChanged = false;
                        }
                    }

                    // wobble effect
                    if (wobbleEffectEnabled)
                    {
                        foreach (var camera in ensemble.cameras)
                        {
                            var cameraDeviceResource = cameraDeviceResources[camera];

                            var world = new SharpDX.Matrix();
                            for (int i = 0; i < 4; i++)
                            {
                                for (int j = 0; j < 4; j++)
                                {
                                    world[i, j] = (float)camera.pose[i, j];
                                }
                            }
                            world.Transpose();

                            // view and projection matrix are post-multiply
                            var userWorldViewProjection = world * userView * userProjection;

                            depthAndColorShader.SetConstants(deviceContext, camera.calibration, userWorldViewProjection);
                            depthAndColorShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.colorImageTextureRV, cameraDeviceResource.vertexBuffer, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
                        }
                    }

                    // 3d object
                    if (threeDObjectEnabled)
                    {
                        var world = SharpDX.Matrix.Scaling(1.0f) * SharpDX.Matrix.RotationY(90.0f / 180.0f * (float)Math.PI) *
                                    SharpDX.Matrix.RotationX(-40.0f / 180.0f * (float)Math.PI) * SharpDX.Matrix.Translation(0, 0.7f, 0.0f);

                        var pointLight = new PointLight();
                        pointLight.position = new Vector3(0, 2, 0);
                        pointLight.Ia       = new Vector3(0.1f, 0.1f, 0.1f);
                        meshShader.SetVertexShaderConstants(deviceContext, world, userView * userProjection, pointLight.position);
                        meshShader.Render(deviceContext, meshDeviceResources, pointLight, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
                    }

                    // wobble effect
                    if (wobbleEffectEnabled)
                    {
                        alpha += 0.05f;
                        if (alpha > 1)
                        {
                            radialWobbleShader.SetConstants(deviceContext, 0);
                        }
                        else
                        {
                            radialWobbleShader.SetConstants(deviceContext, alpha);
                        }
                        radialWobbleShader.Render(deviceContext, userViewSRV, filteredUserViewRenderTargetView);
                    }


                    // desktop duplication
                    if (desktopDuplicationEnabled)
                    {
                        var outputDuplicateFrameInformation = default(OutputDuplicateFrameInformation);
                        SharpDX.DXGI.Resource resource      = null;
                        outputDuplication.AcquireNextFrame(1000, out outputDuplicateFrameInformation, out resource);
                        var texture = resource.QueryInterface <Texture2D>();

                        var rect = new RECT();
                        GetWindowRect(windowPtr, out rect);

                        var sourceRegion = new ResourceRegion()
                        {
                            Left   = rect.Left + leftNudge,
                            Right  = rect.Right + rightNudge,
                            Top    = rect.Top + topNudge,
                            Bottom = rect.Bottom + bottomNudge,
                            Front  = 0,
                            Back   = 1,
                        };
                        deviceContext.CopySubresourceRegion(texture, 0, sourceRegion, desktopTexture, 0);
                        texture.Dispose();
                    }


                    // render user view to seperate form
                    passThroughShader.viewport = new Viewport(0, 0, userViewForm.Width, userViewForm.Height);
                    // TODO: clean this up by simply using a pointer to the userViewSRV
                    if (threeDObjectEnabled)
                    {
                        passThroughShader.Render(deviceContext, userViewSRV, userViewForm.renderTargetView);
                    }
                    if (wobbleEffectEnabled)
                    {
                        passThroughShader.Render(deviceContext, filteredUserViewSRV, userViewForm.renderTargetView);
                    }
                    if (desktopDuplicationEnabled)
                    {
                        passThroughShader.Render(deviceContext, desktopTextureSRV, userViewForm.renderTargetView);
                    }
                    userViewForm.swapChain.Present(0, PresentFlags.None);

                    // projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1]
                    // TODO: put this in SetContants?
                    userProjection[0, 0] /= 2;
                    userProjection[1, 1] /= -2; // y points down
                    userProjection[2, 0] += 0.5f;
                    userProjection[2, 1] += 0.5f;

                    // projection mapping for each projector
                    foreach (var form in projectorForms)
                    {
                        deviceContext.ClearRenderTargetView(form.renderTargetView, Color4.Black);
                        deviceContext.ClearDepthStencilView(form.depthStencilView, DepthStencilClearFlags.Depth, 1, 0);

                        foreach (var camera in ensemble.cameras)
                        {
                            var cameraDeviceResource = cameraDeviceResources[camera];

                            var world = new SharpDX.Matrix();
                            for (int i = 0; i < 4; i++)
                            {
                                for (int j = 0; j < 4; j++)
                                {
                                    world[i, j] = (float)camera.pose[i, j];
                                }
                            }
                            world.Transpose();

                            var projectorWorldViewProjection = world * form.view * form.projection;
                            var userWorldViewProjection      = world * userView * userProjection;

                            projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection);

                            // TODO: clean this up by simply using a pointer to the userViewSRV
                            if (wobbleEffectEnabled)
                            {
                                projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, filteredUserViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
                            }
                            if (threeDObjectEnabled)
                            {
                                projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, userViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
                            }
                            if (desktopDuplicationEnabled)
                            {
                                projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, desktopTextureSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
                            }
                        }

                        form.swapChain.Present(1, PresentFlags.None);
                    }


                    if (desktopDuplicationEnabled)
                    {
                        outputDuplication.ReleaseFrame();
                    }


                    Console.WriteLine(stopwatch.ElapsedMilliseconds);
                    stopwatch.Restart();
                }
            }
        }
Example #2
0
        void RenderLoop()
        {
            while (true)
            {
                lock (renderLock)
                {
                    var deviceContext = device.ImmediateContext;

                    // render user view
                    deviceContext.ClearRenderTargetView(userViewRenderTargetView, Color4.Black);
                    deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0);

                    SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f);  // may need to change this default

                    if (localHeadTrackingEnabled)
                    {
                        float distanceSquared = 0;
                        lock (headCameraSpacePointLock)
                        {
                            headPosition = new SharpDX.Vector3(headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z);

                            float dx = handLeftCameraSpacePoint.X - handRightCameraSpacePoint.X;
                            float dy = handLeftCameraSpacePoint.Y - handRightCameraSpacePoint.Y;
                            float dz = handLeftCameraSpacePoint.Z - handRightCameraSpacePoint.Z;
                            distanceSquared = dx * dx + dy * dy + dz * dz;
                        }
                        var transform = SharpDX.Matrix.RotationY((float)Math.PI) * SharpDX.Matrix.Translation(0, 0.45f, 0);
                        headPosition = SharpDX.Vector3.TransformCoordinate(headPosition, transform);

                        if (trackingValid && (distanceSquared < 0.02f) && (alpha > 1))
                        {
                            alpha = 0;
                        }
                        //Console.WriteLine(distanceSquared);
                    }

                    var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY);
                    userView.Transpose();


                    //Console.WriteLine("headPosition = " + headPosition);


                    float aspect         = (float)userViewTextureWidth / (float)userViewTextureHeight;
                    var   userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f);
                    userProjection.Transpose();

                    // smooth depth images
                    foreach (var camera in ensemble.cameras)
                    {
                        var cameraDeviceResource = cameraDeviceResources[camera];
                        if (cameraDeviceResource.depthImageChanged)
                        {
                            fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView);
                            for (int i = 0; i < 1; i++)
                            {
                                bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2);
                                bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView);
                            }
                            cameraDeviceResource.depthImageChanged = false;
                        }
                    }

                    // wobble effect
                    if (wobbleEffectEnabled)
                    {
                        foreach (var camera in ensemble.cameras)
                        {
                            var cameraDeviceResource = cameraDeviceResources[camera];

                            var world = new SharpDX.Matrix();
                            for (int i = 0; i < 4; i++)
                            {
                                for (int j = 0; j < 4; j++)
                                {
                                    world[i, j] = (float)camera.pose[i, j];
                                }
                            }
                            world.Transpose();

                            // view and projection matrix are post-multiply
                            var userWorldViewProjection = world * userView * userProjection;

                            depthAndColorShader.SetConstants(deviceContext, camera.calibration, userWorldViewProjection);
                            depthAndColorShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.colorImageTextureRV, cameraDeviceResource.vertexBuffer, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
                        }
                    }

                    // 3d object
                    if (threeDObjectEnabled)
                    {
                        var world = SharpDX.Matrix.Scaling(1.0f) * SharpDX.Matrix.RotationY(90.0f / 180.0f * (float)Math.PI) *
                                    SharpDX.Matrix.RotationX(-40.0f / 180.0f * (float)Math.PI) * SharpDX.Matrix.Translation(0, 0.7f, 0.0f);

                        var pointLight = new PointLight();
                        pointLight.position = new Vector3(0, 2, 0);
                        pointLight.Ia       = new Vector3(0.1f, 0.1f, 0.1f);
                        meshShader.SetVertexShaderConstants(deviceContext, world, userView * userProjection, pointLight.position);
                        meshShader.Render(deviceContext, meshDeviceResources, pointLight, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
                    }

                    // wobble effect
                    if (wobbleEffectEnabled)
                    {
                        alpha += 0.05f;
                        if (alpha > 1)
                        {
                            radialWobbleShader.SetConstants(deviceContext, 0);
                        }
                        else
                        {
                            radialWobbleShader.SetConstants(deviceContext, alpha);
                        }
                        radialWobbleShader.Render(deviceContext, userViewSRV, filteredUserViewRenderTargetView);
                    }

                    // desktop duplication
                    if (desktopDuplicationEnabled)
                    {
                        // update the desktop texture; this will block until there is some change
                        var outputDuplicateFrameInformation = default(OutputDuplicateFrameInformation);
                        SharpDX.DXGI.Resource resource      = null;
                        outputDuplication.AcquireNextFrame(1000, out outputDuplicateFrameInformation, out resource);
                        var texture = resource.QueryInterface <Texture2D>();

                        // pick up the window under the cursor
                        var cursorPos = new POINT();
                        GetCursorPos(out cursorPos);
                        var hwnd = WindowFromPoint(cursorPos);
                        var rect = new RECT();
                        GetWindowRect(hwnd, out rect);

                        // adjust bounds so falls within source texture
                        if (rect.Left < 0)
                        {
                            rect.Left = 0;
                        }
                        if (rect.Top < 0)
                        {
                            rect.Top = 0;
                        }
                        if (rect.Right > texture.Description.Width - 1)
                        {
                            rect.Right = texture.Description.Width;
                        }
                        if (rect.Bottom > texture.Description.Height - 1)
                        {
                            rect.Bottom = texture.Description.Height;
                        }

                        int width  = rect.Right - rect.Left;
                        int height = rect.Bottom - rect.Top;

                        // resize our texture if necessary
                        if ((desktopTexture == null) || (desktopTexture.Description.Width != width) || (desktopTexture.Description.Height != height))
                        {
                            if (desktopTexture != null)
                            {
                                desktopTextureSRV.Dispose();
                                desktopTexture.Dispose();
                            }
                            var desktopTextureDesc = new Texture2DDescription()
                            {
                                Width             = width,
                                Height            = height,
                                MipLevels         = 1, // revisit this; we may benefit from mipmapping?
                                ArraySize         = 1,
                                Format            = SharpDX.DXGI.Format.B8G8R8A8_UNorm,
                                SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0),
                                Usage             = ResourceUsage.Default,
                                BindFlags         = BindFlags.ShaderResource,
                                CpuAccessFlags    = CpuAccessFlags.None,
                            };
                            desktopTexture    = new Texture2D(device, desktopTextureDesc);
                            desktopTextureSRV = new ShaderResourceView(device, desktopTexture);
                        }

                        // copy the winodw region into our texture
                        var sourceRegion = new ResourceRegion()
                        {
                            Left   = rect.Left,
                            Right  = rect.Right,
                            Top    = rect.Top,
                            Bottom = rect.Bottom,
                            Front  = 0,
                            Back   = 1,
                        };
                        deviceContext.CopySubresourceRegion(texture, 0, sourceRegion, desktopTexture, 0);
                        texture.Dispose();
                    }

                    // render user view to seperate form
                    passThroughShader.viewport = new Viewport(0, 0, userViewForm.videoPanel1.Width, userViewForm.videoPanel1.Height);

                    // TODO: clean this up by simply using a pointer to the userViewSRV
                    if (threeDObjectEnabled)
                    {
                        passThroughShader.Render(deviceContext, userViewSRV, userViewForm.renderTargetView);
                    }
                    if (wobbleEffectEnabled)
                    {
                        passThroughShader.Render(deviceContext, filteredUserViewSRV, userViewForm.renderTargetView);
                    }
                    if (desktopDuplicationEnabled)
                    {
                        passThroughShader.Render(deviceContext, desktopTextureSRV, userViewForm.renderTargetView);
                    }
                    userViewForm.swapChain.Present(0, PresentFlags.None);

                    // projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1]
                    // TODO: put this in SetContants?
                    userProjection[0, 0] /= 2;
                    userProjection[1, 1] /= -2; // y points down
                    userProjection[2, 0] += 0.5f;
                    userProjection[2, 1] += 0.5f;

                    // projection mapping for each projector
                    foreach (var form in projectorForms)
                    {
                        deviceContext.ClearRenderTargetView(form.renderTargetView, Color4.Black);
                        deviceContext.ClearDepthStencilView(form.depthStencilView, DepthStencilClearFlags.Depth, 1, 0);

                        foreach (var camera in ensemble.cameras)
                        {
                            var cameraDeviceResource = cameraDeviceResources[camera];

                            var world = new SharpDX.Matrix();
                            for (int i = 0; i < 4; i++)
                            {
                                for (int j = 0; j < 4; j++)
                                {
                                    world[i, j] = (float)camera.pose[i, j];
                                }
                            }
                            world.Transpose();

                            var projectorWorldViewProjection = world * form.view * form.projection;
                            var userWorldViewProjection      = world * userView * userProjection;

                            projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection);

                            // TODO: clean this up by simply using a pointer to the userViewSRV
                            if (wobbleEffectEnabled)
                            {
                                projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, filteredUserViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
                            }
                            if (threeDObjectEnabled)
                            {
                                projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, userViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
                            }
                            if (desktopDuplicationEnabled)
                            {
                                projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, desktopTextureSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
                            }
                        }

                        form.swapChain.Present(1, PresentFlags.None);
                    }


                    if (desktopDuplicationEnabled)
                    {
                        outputDuplication.ReleaseFrame();
                    }


                    //Console.WriteLine(stopwatch.ElapsedMilliseconds);
                    stopwatch.Restart();
                }
            }
        }
        void RenderLoop()
        {
            while (true)
            {
                lock (renderLock)
                {
                    var deviceContext = device.ImmediateContext;

                    // render user view
                    deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0);

                    SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f);  // may need to change this default

                    var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY);
                    userView.Transpose();

                    float aspect         = (float)userViewTextureWidth / (float)userViewTextureHeight;
                    var   userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f);
                    userProjection.Transpose();

                    // smooth depth images
                    foreach (var camera in ensemble.cameras)
                    {
                        var cameraDeviceResource = cameraDeviceResources[camera];
                        if (cameraDeviceResource.depthImageChanged)
                        {
                            fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView);
                            bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2);
                            bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView);
                            cameraDeviceResource.depthImageChanged = false;
                        }
                    }

                    // projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1]
                    // TODO: put this in SetContants?
                    userProjection[0, 0] /= 2;
                    userProjection[1, 1] /= -2; // y points down
                    userProjection[2, 0] += 0.5f;
                    userProjection[2, 1] += 0.5f;

                    // projection mapping for each projector
                    foreach (var form in projectorForms)
                    {
                        deviceContext.ClearRenderTargetView(form.renderTargetView, Color4.Black);
                        deviceContext.ClearDepthStencilView(form.depthStencilView, DepthStencilClearFlags.Depth, 1, 0);

                        foreach (var camera in ensemble.cameras)
                        {
                            var cameraDeviceResource = cameraDeviceResources[camera];

                            // Copy camera pose
                            var world = new SharpDX.Matrix();
                            for (int i = 0; i < 4; i++)
                            {
                                for (int j = 0; j < 4; j++)
                                {
                                    world[i, j] = (float)camera.pose[i, j];
                                }
                            }
                            world.Transpose();

                            var projectorWorldViewProjection = world * form.view * form.projection;
                            var userWorldViewProjection      = world * userView * userProjection;

                            projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection, clock.Elapsed);
                            projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
                        }

                        form.swapChain.Present(1, PresentFlags.None);
                    }

                    //Console.WriteLine(stopwatch.ElapsedMilliseconds);
                    stopwatch.Restart();
                }
            }
        }