protected override void OnLoad(EventArgs e)
{
base.OnLoad(e);
// cache location etc for when we come out of fullscreen
windowedLocation = Location;
windowedSize = Size;
// pick up view and projection for projector
view = new SharpDX.Matrix();
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 4; j++)
{
view[i, j] = (float)projector.pose[i, j];
}
}
view.Invert();
view.Transpose();
var cameraMatrix = projector.cameraMatrix;
float fx = (float)cameraMatrix[0, 0];
float fy = (float)cameraMatrix[1, 1];
float cx = (float)cameraMatrix[0, 2];
float cy = (float)cameraMatrix[1, 2];
float near = 0.1f;
float far = 100.0f;
float w = projector.width;
float h = projector.height;
projection = GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far);
projection.Transpose();
}
void RenderLoop()
{
while (true)
{
lock (renderLock)
{
var deviceContext = device.ImmediateContext;
// render user view
deviceContext.ClearRenderTargetView(userViewRenderTargetView, Color4.Black);
deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0);
SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f); // may need to change this default
if (localHeadTrackingEnabled)
{
float distanceSquared = 0;
lock (headCameraSpacePointLock)
{
headPosition = new SharpDX.Vector3(headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z);
float dx = handLeftCameraSpacePoint.X - handRightCameraSpacePoint.X;
float dy = handLeftCameraSpacePoint.Y - handRightCameraSpacePoint.Y;
float dz = handLeftCameraSpacePoint.Z - handRightCameraSpacePoint.Z;
distanceSquared = dx * dx + dy * dy + dz * dz;
}
var transform = SharpDX.Matrix.RotationY((float)Math.PI) * SharpDX.Matrix.Translation(0, 0.45f, 0);
headPosition = SharpDX.Vector3.TransformCoordinate(headPosition, transform);
if (trackingValid && (distanceSquared < 0.02f) && (alpha > 1))
{
alpha = 0;
}
//Console.WriteLine(distanceSquared);
}
var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY);
userView.Transpose();
//Console.WriteLine("headPosition = " + headPosition);
float aspect = (float)userViewTextureWidth / (float)userViewTextureHeight;
var userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f);
userProjection.Transpose();
// smooth depth images
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
if (cameraDeviceResource.depthImageChanged)
{
fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView);
for (int i = 0; i < 1; i++)
{
bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2);
bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView);
}
cameraDeviceResource.depthImageChanged = false;
}
}
// wobble effect
if (wobbleEffectEnabled)
{
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
var world = new SharpDX.Matrix();
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 4; j++)
{
world[i, j] = (float)camera.pose[i, j];
}
}
world.Transpose();
// view and projection matrix are post-multiply
var userWorldViewProjection = world * userView * userProjection;
depthAndColorShader.SetConstants(deviceContext, camera.calibration, userWorldViewProjection);
depthAndColorShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.colorImageTextureRV, cameraDeviceResource.vertexBuffer, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
}
}
// 3d object
if (threeDObjectEnabled)
{
var world = SharpDX.Matrix.Scaling(1.0f) * SharpDX.Matrix.RotationY(90.0f / 180.0f * (float)Math.PI) *
SharpDX.Matrix.RotationX(-40.0f / 180.0f * (float)Math.PI) * SharpDX.Matrix.Translation(0, 0.7f, 0.0f);
var pointLight = new PointLight();
pointLight.position = new Vector3(0, 2, 0);
pointLight.Ia = new Vector3(0.1f, 0.1f, 0.1f);
meshShader.SetVertexShaderConstants(deviceContext, world, userView * userProjection, pointLight.position);
meshShader.Render(deviceContext, meshDeviceResources, pointLight, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
}
// wobble effect
if (wobbleEffectEnabled)
{
alpha += 0.05f;
if (alpha > 1)
{
radialWobbleShader.SetConstants(deviceContext, 0);
}
else
{
radialWobbleShader.SetConstants(deviceContext, alpha);
}
radialWobbleShader.Render(deviceContext, userViewSRV, filteredUserViewRenderTargetView);
}
// desktop duplication
if (desktopDuplicationEnabled)
{
// update the desktop texture; this will block until there is some change
var outputDuplicateFrameInformation = default(OutputDuplicateFrameInformation);
SharpDX.DXGI.Resource resource = null;
outputDuplication.AcquireNextFrame(1000, out outputDuplicateFrameInformation, out resource);
var texture = resource.QueryInterface <Texture2D>();
// pick up the window under the cursor
var cursorPos = new POINT();
GetCursorPos(out cursorPos);
var hwnd = WindowFromPoint(cursorPos);
var rect = new RECT();
GetWindowRect(hwnd, out rect);
// adjust bounds so falls within source texture
if (rect.Left < 0)
{
rect.Left = 0;
}
if (rect.Top < 0)
{
rect.Top = 0;
}
if (rect.Right > texture.Description.Width - 1)
{
rect.Right = texture.Description.Width;
}
if (rect.Bottom > texture.Description.Height - 1)
{
rect.Bottom = texture.Description.Height;
}
int width = rect.Right - rect.Left;
int height = rect.Bottom - rect.Top;
// resize our texture if necessary
if ((desktopTexture == null) || (desktopTexture.Description.Width != width) || (desktopTexture.Description.Height != height))
{
if (desktopTexture != null)
{
desktopTextureSRV.Dispose();
desktopTexture.Dispose();
}
var desktopTextureDesc = new Texture2DDescription()
{
Width = width,
Height = height,
MipLevels = 1, // revisit this; we may benefit from mipmapping?
ArraySize = 1,
Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm,
SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0),
Usage = ResourceUsage.Default,
BindFlags = BindFlags.ShaderResource,
CpuAccessFlags = CpuAccessFlags.None,
};
desktopTexture = new Texture2D(device, desktopTextureDesc);
desktopTextureSRV = new ShaderResourceView(device, desktopTexture);
}
// copy the winodw region into our texture
var sourceRegion = new ResourceRegion()
{
Left = rect.Left,
Right = rect.Right,
Top = rect.Top,
Bottom = rect.Bottom,
Front = 0,
Back = 1,
};
deviceContext.CopySubresourceRegion(texture, 0, sourceRegion, desktopTexture, 0);
texture.Dispose();
}
// render user view to seperate form
passThroughShader.viewport = new Viewport(0, 0, userViewForm.videoPanel1.Width, userViewForm.videoPanel1.Height);
// TODO: clean this up by simply using a pointer to the userViewSRV
if (threeDObjectEnabled)
{
passThroughShader.Render(deviceContext, userViewSRV, userViewForm.renderTargetView);
}
if (wobbleEffectEnabled)
{
passThroughShader.Render(deviceContext, filteredUserViewSRV, userViewForm.renderTargetView);
}
if (desktopDuplicationEnabled)
{
passThroughShader.Render(deviceContext, desktopTextureSRV, userViewForm.renderTargetView);
}
userViewForm.swapChain.Present(0, PresentFlags.None);
// projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1]
// TODO: put this in SetContants?
userProjection[0, 0] /= 2;
userProjection[1, 1] /= -2; // y points down
userProjection[2, 0] += 0.5f;
userProjection[2, 1] += 0.5f;
// projection mapping for each projector
foreach (var form in projectorForms)
{
deviceContext.ClearRenderTargetView(form.renderTargetView, Color4.Black);
deviceContext.ClearDepthStencilView(form.depthStencilView, DepthStencilClearFlags.Depth, 1, 0);
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
var world = new SharpDX.Matrix();
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 4; j++)
{
world[i, j] = (float)camera.pose[i, j];
}
}
world.Transpose();
var projectorWorldViewProjection = world * form.view * form.projection;
var userWorldViewProjection = world * userView * userProjection;
projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection);
// TODO: clean this up by simply using a pointer to the userViewSRV
if (wobbleEffectEnabled)
{
projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, filteredUserViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
}
if (threeDObjectEnabled)
{
projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, userViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
}
if (desktopDuplicationEnabled)
{
projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, desktopTextureSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
}
}
form.swapChain.Present(1, PresentFlags.None);
}
if (desktopDuplicationEnabled)
{
outputDuplication.ReleaseFrame();
}
//Console.WriteLine(stopwatch.ElapsedMilliseconds);
stopwatch.Restart();
}
}
}
void RenderLoop()
{
while (true)
{
lock (renderLock)
{
var deviceContext = device.ImmediateContext;
// render user view
deviceContext.ClearRenderTargetView(userViewRenderTargetView, Color4.Black);
deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0);
SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f); // may need to change this default
if (localHeadTrackingEnabled)
{
float distanceSquared = 0;
lock (headCameraSpacePointLock)
{
headPosition = new SharpDX.Vector3(headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z);
float dx = handLeftCameraSpacePoint.X - handRightCameraSpacePoint.X;
float dy = handLeftCameraSpacePoint.Y - handRightCameraSpacePoint.Y;
float dz = handLeftCameraSpacePoint.Z - handRightCameraSpacePoint.Z;
distanceSquared = dx * dx + dy * dy + dz * dz;
}
var transform = SharpDX.Matrix.RotationY((float)Math.PI) * SharpDX.Matrix.Translation(-0.25f, 0.45f, 0);
headPosition = SharpDX.Vector3.TransformCoordinate(headPosition, transform);
if (trackingValid && (distanceSquared < 0.02f) && (alpha > 1))
{
alpha = 0;
}
//Console.WriteLine(distanceSquared);
}
var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY);
userView.Transpose();
//Console.WriteLine("headPosition = " + headPosition);
float aspect = (float)userViewTextureWidth / (float)userViewTextureHeight;
var userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f);
userProjection.Transpose();
// smooth depth images
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
if (cameraDeviceResource.depthImageChanged)
{
fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView);
for (int i = 0; i < 1; i++)
{
bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2);
bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView);
}
cameraDeviceResource.depthImageChanged = false;
}
}
// wobble effect
if (wobbleEffectEnabled)
{
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
var world = new SharpDX.Matrix();
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 4; j++)
{
world[i, j] = (float)camera.pose[i, j];
}
}
world.Transpose();
// view and projection matrix are post-multiply
var userWorldViewProjection = world * userView * userProjection;
depthAndColorShader.SetConstants(deviceContext, camera.calibration, userWorldViewProjection);
depthAndColorShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.colorImageTextureRV, cameraDeviceResource.vertexBuffer, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
}
}
// 3d object
if (threeDObjectEnabled)
{
var world = SharpDX.Matrix.Scaling(1.0f) * SharpDX.Matrix.RotationY(90.0f / 180.0f * (float)Math.PI) *
SharpDX.Matrix.RotationX(-40.0f / 180.0f * (float)Math.PI) * SharpDX.Matrix.Translation(0, 0.7f, 0.0f);
var pointLight = new PointLight();
pointLight.position = new Vector3(0, 2, 0);
pointLight.Ia = new Vector3(0.1f, 0.1f, 0.1f);
meshShader.SetVertexShaderConstants(deviceContext, world, userView * userProjection, pointLight.position);
meshShader.Render(deviceContext, meshDeviceResources, pointLight, userViewRenderTargetView, userViewDepthStencilView, userViewViewport);
}
// wobble effect
if (wobbleEffectEnabled)
{
alpha += 0.05f;
if (alpha > 1)
{
radialWobbleShader.SetConstants(deviceContext, 0);
}
else
{
radialWobbleShader.SetConstants(deviceContext, alpha);
}
radialWobbleShader.Render(deviceContext, userViewSRV, filteredUserViewRenderTargetView);
}
// desktop duplication
if (desktopDuplicationEnabled)
{
var outputDuplicateFrameInformation = default(OutputDuplicateFrameInformation);
SharpDX.DXGI.Resource resource = null;
outputDuplication.AcquireNextFrame(1000, out outputDuplicateFrameInformation, out resource);
var texture = resource.QueryInterface <Texture2D>();
var rect = new RECT();
GetWindowRect(windowPtr, out rect);
var sourceRegion = new ResourceRegion()
{
Left = rect.Left + leftNudge,
Right = rect.Right + rightNudge,
Top = rect.Top + topNudge,
Bottom = rect.Bottom + bottomNudge,
Front = 0,
Back = 1,
};
deviceContext.CopySubresourceRegion(texture, 0, sourceRegion, desktopTexture, 0);
texture.Dispose();
}
// render user view to seperate form
passThroughShader.viewport = new Viewport(0, 0, userViewForm.Width, userViewForm.Height);
// TODO: clean this up by simply using a pointer to the userViewSRV
if (threeDObjectEnabled)
{
passThroughShader.Render(deviceContext, userViewSRV, userViewForm.renderTargetView);
}
if (wobbleEffectEnabled)
{
passThroughShader.Render(deviceContext, filteredUserViewSRV, userViewForm.renderTargetView);
}
if (desktopDuplicationEnabled)
{
passThroughShader.Render(deviceContext, desktopTextureSRV, userViewForm.renderTargetView);
}
userViewForm.swapChain.Present(0, PresentFlags.None);
// projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1]
// TODO: put this in SetContants?
userProjection[0, 0] /= 2;
userProjection[1, 1] /= -2; // y points down
userProjection[2, 0] += 0.5f;
userProjection[2, 1] += 0.5f;
// projection mapping for each projector
foreach (var form in projectorForms)
{
deviceContext.ClearRenderTargetView(form.renderTargetView, Color4.Black);
deviceContext.ClearDepthStencilView(form.depthStencilView, DepthStencilClearFlags.Depth, 1, 0);
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
var world = new SharpDX.Matrix();
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 4; j++)
{
world[i, j] = (float)camera.pose[i, j];
}
}
world.Transpose();
var projectorWorldViewProjection = world * form.view * form.projection;
var userWorldViewProjection = world * userView * userProjection;
projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection);
// TODO: clean this up by simply using a pointer to the userViewSRV
if (wobbleEffectEnabled)
{
projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, filteredUserViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
}
if (threeDObjectEnabled)
{
projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, userViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
}
if (desktopDuplicationEnabled)
{
projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, desktopTextureSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
}
}
form.swapChain.Present(1, PresentFlags.None);
}
if (desktopDuplicationEnabled)
{
outputDuplication.ReleaseFrame();
}
Console.WriteLine(stopwatch.ElapsedMilliseconds);
stopwatch.Restart();
}
}
}
void RenderLoop()
{
while (true)
{
lock (renderLock)
{
var deviceContext = device.ImmediateContext;
// render user view
deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0);
SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f); // may need to change this default
var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY);
userView.Transpose();
float aspect = (float)userViewTextureWidth / (float)userViewTextureHeight;
var userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f);
userProjection.Transpose();
// smooth depth images
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
if (cameraDeviceResource.depthImageChanged)
{
fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView);
bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2);
bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView);
cameraDeviceResource.depthImageChanged = false;
}
}
// projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1]
// TODO: put this in SetContants?
userProjection[0, 0] /= 2;
userProjection[1, 1] /= -2; // y points down
userProjection[2, 0] += 0.5f;
userProjection[2, 1] += 0.5f;
// projection mapping for each projector
foreach (var form in projectorForms)
{
deviceContext.ClearRenderTargetView(form.renderTargetView, Color4.Black);
deviceContext.ClearDepthStencilView(form.depthStencilView, DepthStencilClearFlags.Depth, 1, 0);
foreach (var camera in ensemble.cameras)
{
var cameraDeviceResource = cameraDeviceResources[camera];
// Copy camera pose
var world = new SharpDX.Matrix();
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 4; j++)
{
world[i, j] = (float)camera.pose[i, j];
}
}
world.Transpose();
var projectorWorldViewProjection = world * form.view * form.projection;
var userWorldViewProjection = world * userView * userProjection;
projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection, clock.Elapsed);
projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport);
}
form.swapChain.Present(1, PresentFlags.None);
}
//Console.WriteLine(stopwatch.ElapsedMilliseconds);
stopwatch.Restart();
}
}
}