// Zero based device index and device params and output window
public CameraDevice(Canvas iCanvas, int iDeviceNum, int iWidth, int iHeight, short iBPP)
{
DsDevice[] capDevices;
// Get the collection of video devices
capDevices = DsDevice.GetDevicesOfCat(FilterCategory.VideoInputDevice);
if (iDeviceNum + 1 > capDevices.Length)
{
throw new Exception("No video capture devices found at that index!");
}
try
{
deviceNumber = iDeviceNum;
InitDevice(capDevices[iDeviceNum], iWidth, iHeight);
camTextureA = new Texture2D(iCanvas.GetDevice(), Width, Height, 1, TextureUsage.None, SurfaceFormat.Color);
camTextureB = new Texture2D(iCanvas.GetDevice(), Width, Height, 1, TextureUsage.None, SurfaceFormat.Color);
bgrData = new byte[Stride * Height];
}
catch
{
Dispose();
throw;
}
}
void LoadShaders(Canvas canvas)
{
shadersLoaded = true;
cubeShader = new Shader();
cubeShader.CompileFromFiles(canvas, "Shaders/CubeP.hlsl", "Shaders/CubeV.hlsl");
baseTexture = Texture2D.FromFile(canvas.GetDevice(), "Data/Textures/smiley.png");
}
public override void OnRender(Canvas canvas)
{
base.OnRender(canvas);
if (!shadersLoaded)
{
LoadShaders(canvas);
}
cubeShader.SetupShader(canvas);
GraphicsDevice device = canvas.GetDevice();
device.SetVertexShaderConstant(CanvasShaderConstants.VC_WORLD, transform.GetTransform());
device.SetPixelShaderConstant(0, color);
CanvasPrimitives.Cube.Render(canvas);
}
protected override void Initialize()
{
canvas = new Canvas(this.GraphicsDevice);
cameraRig = new StereoRig();
cameraRig.LoadCalibrationFile("Config/intrinsics.yml", "Config/extrinsics.yml");
apps = new List<IProgram>();
apps.Add(new GlowFingerApp());
apps.Add(new DragNDropApp());
for (int i = 0; i < apps.Count; i++)
{
apps[i].OnCreate();
}
base.Initialize();
}
public void CompileFromFiles(Canvas canvas, string psName, string vsName)
{
this.psFileName = psName;
this.vsFileName = vsName;
ShaderProfile psProf = ShaderProfile.PS_1_1;
switch (PSTarget)
{
case 2:
psProf = ShaderProfile.PS_2_0;
break;
case 3:
psProf = ShaderProfile.PS_3_0;
break;
}
ShaderProfile vsProf = ShaderProfile.VS_1_1;
switch (VSTarget)
{
case 2:
vsProf = ShaderProfile.VS_2_0;
break;
case 3:
vsProf = ShaderProfile.VS_3_0;
break;
}
CompiledShader psShader = ShaderCompiler.CompileFromFile(psFileName, null, null, CompilerOptions.PackMatrixRowMajor, "main", psProf, TargetPlatform.Windows);
Log.GetInstance().WriteLine(psShader.ErrorsAndWarnings);
CompiledShader vsShader = ShaderCompiler.CompileFromFile(vsFileName, null, null, CompilerOptions.PackMatrixRowMajor, "main", vsProf, TargetPlatform.Windows);
Log.GetInstance().WriteLine(vsShader.ErrorsAndWarnings);
errorMessage = null;
if (vsShader.ErrorsAndWarnings.Length > 1)
errorMessage = "Vertex Shader: " + vsShader.ErrorsAndWarnings;
if (psShader.ErrorsAndWarnings.Length > 1)
{
if (errorMessage == null)
errorMessage = "Pixel Shader: " + psShader.ErrorsAndWarnings;
else
errorMessage = errorMessage + "\n Pixel Shader: " + psShader.ErrorsAndWarnings;
}
if (psShader.Success && vsShader.Success)
{
ps = new PixelShader(canvas.GetDevice(), psShader.GetShaderCode());
vs = new VertexShader(canvas.GetDevice(), vsShader.GetShaderCode());
compiled = true;
}
}
public override void OnRender(Canvas canvas)
{
base.OnRender(canvas);
if (!shadersLoaded)
{
LoadShaders(canvas);
}
if (collisionPoints == null)
return;
glowShader.SetupShader(canvas);
GraphicsDevice device = canvas.GetDevice();
device.SetPixelShaderConstant(0, color);
for (int i = 0; i < collisionPoints.Length; i++)
{
Matrix transform = Matrix.CreateScale(collisionPoints[i].Radius);
transform.Translation = collisionPoints[i].Center;
device.SetVertexShaderConstant(CanvasShaderConstants.VC_WORLD, transform);
CanvasPrimitives.Sphere.Render(canvas);
}
}
void RenderEdgesStyle(Canvas canvas)
{
GraphicsDevice device = canvas.GetDevice();
edgeShader.SetupShader(canvas);
device.Textures[0] = renderTarget.GetTexture();
//device.Textures[2] = noiseTexture;
device.SetVertexShaderConstant(CanvasShaderConstants.VC_INVTEXRES, Vector2.Zero);
device.SetPixelShaderConstant(0, Vector2.One / new Vector2(renderTarget.Width, renderTarget.Height));
DepthStencilBuffer dsOld = device.DepthStencilBuffer;
device.DepthStencilBuffer = canvas.GetDepthStencil();
device.SetRenderTarget(0, renderTarget);
CanvasPrimitives.Quad.SetPositions(Vector2.One * -1, Vector2.One);
CanvasPrimitives.Quad.Render(canvas);
device.SetRenderTarget(0, null);
device.DepthStencilBuffer = dsOld;
}
public void SetupShader(Canvas canvas)
{
if (!compiled)
{
CompileFromFiles(canvas, psFileName, vsFileName);
if (errorMessage != null)
Log.GetInstance().WriteLine(errorMessage);
return;
}
canvas.GetDevice().PixelShader = ps;
canvas.GetDevice().VertexShader = vs;
}
public void CompositeFinalImage(Canvas canvas)
{
BoundingRect[] drawRects = GetCameraDrawBoxes();
GraphicsDevice device = canvas.GetDevice();
DepthStencilBuffer dsOld = device.DepthStencilBuffer;
device.DepthStencilBuffer = canvas.GetDepthStencil();
device.SetRenderTarget(0, renderTarget);
canvas.DrawImage(cameraAImage, drawRects[0].Min, drawRects[0].Max, (int)(ImageParameters.FlipY));
canvas.DrawImage(cameraBImage, drawRects[1].Min, drawRects[1].Max, (int)(ImageParameters.FlipY));
device.RenderState.AlphaBlendEnable = true;
device.RenderState.SourceBlend = Blend.SourceAlpha;
device.RenderState.DestinationBlend = Blend.InverseSourceAlpha;
canvas.DrawImage(renderLayerA.GetImage(), drawRects[0].Min, drawRects[0].Max);
canvas.DrawImage(renderLayerB.GetImage(), drawRects[1].Min, drawRects[1].Max);
device.SetRenderTarget(0, null);
device.DepthStencilBuffer = dsOld;
//RenderMatrixStyle(canvas);
//RenderEdgesStyle(canvas);
//CanvasPrimitives.Quad.SetPositions(Vector2.One * -2, Vector2.One);
device.RenderState.DepthBufferEnable = false;
device.RenderState.DepthBufferWriteEnable = false;
canvas.DrawImage(renderTarget.GetTexture());
/*
CanvasPrimitives.Quad.SetPositions(Vector2.One * -1, Vector2.One);
device.RenderState.DepthBufferEnable = true;
device.RenderState.DepthBufferWriteEnable = true;
render3D.SetupShader(canvas);
Matrix scaleMat = Matrix.CreateScale(0.05f);
device.SetVertexShaderConstant(CanvasShaderConstants.VC_MODELVIEW, virtualCameraA.ViewProjection);
for (int i = 0; i < detectionPoints.Length; i++)
{
device.SetPixelShaderConstant(0, fingerColors[i % fingerColors.Length].ToVector4());
scaleMat.Translation = detectionPoints[i].Center;
device.SetVertexShaderConstant(CanvasShaderConstants.VC_WORLD, scaleMat);
CanvasPrimitives.Quad.Render(canvas);
}
*/
}
public static void Initialize(Canvas canvas)
{
Quad = new ScreenAlignedQuad();
Cube = new RenderCube();
Sphere = new RenderSphere(canvas.GetDevice(), 17);
}
public void Render(Canvas canvas)
{
canvas.GetDevice().VertexDeclaration = CanvasVertexDeclarations.PTDec;
canvas.GetDevice().DrawUserIndexedPrimitives<VertexPositionTexture>(PrimitiveType.TriangleList, verts, 0, 8, ib, 0, 12);
}
public void SetupTextures(Canvas canvas)
{
for (int i = 0; i < textures.Length; i++)
{
if (textures[i] == null && textureFileNames[i] != string.Empty)
textures[i] = Texture2D.FromFile(canvas.GetDevice(), textureFileNames[i]);
if (textures[i] != null)
canvas.GetDevice().Textures[i] = textures[i];
}
}
public static void Initialize(Canvas canvas)
{
GraphicsDevice device = canvas.GetDevice();
PTDec = new VertexDeclaration(device, VertexPositionTexture.VertexElements);
PNDec = new VertexDeclaration(device, VertexPositionNormal.VertexElements);
}
public void RenderCameraRig(Canvas canvas)
{
if (!camerasInitialized)
{
InitializeCameras(canvas);
}
if (!shadersInitialized)
{
InitializeShaders(canvas);
}
cameraAImage = cameraA.GetCameraImage();
cameraBImage = cameraB.GetCameraImage();
DetectFingers(canvas);
//canvas.DrawImage(dsChain[dsChain.Length-1].GetTexture());
/*
redShader.SetupShader(canvas);
for (int i = 0; i < fingerPairs.Length / 2; i++)
{
device.SetPixelShaderConstant(0, fingerColors[i % fingerColors.Length].ToVector4());
CanvasPrimitives.Quad.SetPositions(fingerPairs[i*2].Min, fingerPairs[i*2].Max);
CanvasPrimitives.Quad.Render(canvas);
CanvasPrimitives.Quad.SetPositions(fingerPairs[i * 2 + 1].Min, fingerPairs[i * 2 + 1].Max);
CanvasPrimitives.Quad.Render(canvas);
}
*/
}
void LoadShaders(Canvas canvas)
{
shadersLoaded = true;
glowShader = new Shader();
glowShader.CompileFromFiles(canvas, "Shaders/GlowP.hlsl", "Shaders/CubeV.hlsl");
}
void InitializeShaders(Canvas canvas)
{
shadersInitialized = true;
undistortShader = new Shader();
undistortShader.CompileFromFiles(canvas, "Shaders/UndistortP.hlsl", "Shaders/Basic2DV.hlsl");
matrixShader = new Shader();
matrixShader.CompileFromFiles(canvas, "Shaders/MatrixP.hlsl", "Shaders/Basic2DV.hlsl");
edgeShader = new Shader();
edgeShader.CompileFromFiles(canvas, "Shaders/EdgesP.hlsl", "Shaders/Basic2DV.hlsl");
downsampleMaxLumShader = new Shader();
downsampleMaxLumShader.CompileFromFiles(canvas, "Shaders/DownsampleMaxLumP.hlsl", "Shaders/Basic2DV.hlsl");
biasLumShader = new Shader();
biasLumShader.CompileFromFiles(canvas, "Shaders/BiasLum.hlsl", "Shaders/Basic2DV.hlsl");
unicodeTable = Texture2D.FromFile(canvas.GetDevice(), "Data/Textures/japaneseTable.png");
renderTarget = new RenderTarget2D(canvas.GetDevice(), cameraA.Width*2, cameraA.Height, 1, SurfaceFormat.Color);
BuildDSChain(canvas);
redShader = new Shader();
redShader.CompileFromFiles(canvas, "Shaders/RedP.hlsl", "Shaders/Basic2DV.hlsl");
render3D = new Shader();
render3D.CompileFromFiles(canvas, "Shaders/RedP.hlsl", "Shaders/RedV.hlsl");
glowTarget = new RenderTarget2D[2];
glowTarget[0] = new RenderTarget2D(canvas.GetDevice(), renderTarget.Width / 4, renderTarget.Height / 4, 1, SurfaceFormat.Color);
glowTarget[1] = new RenderTarget2D(canvas.GetDevice(), glowTarget[0].Width, glowTarget[0].Height, 1, glowTarget[0].Format);
}
void InitializeCameras(Canvas canvas)
{
camerasInitialized = true;
cameraA = new CameraDevice(canvas, 0, 640, 480, 24);
cameraA.Play();
SetupCameraParameters(cameraA);
cameraB = new CameraDevice(canvas, 1, 640, 480, 24);
cameraB.Play();
SetupCameraParameters(cameraB);
GraphicsDevice device = canvas.GetDevice();
int width = device.PresentationParameters.BackBufferWidth/2;
int height = device.PresentationParameters.BackBufferHeight;
renderLayerA = new RenderLayer(canvas, width, height);
renderLayerB = new RenderLayer(canvas, width, height);
}
BoundingRect[] DetectFingersFromImage(Canvas canvas, Texture2D image)
{
GraphicsDevice device = canvas.GetDevice();
DepthStencilBuffer dsOld = device.DepthStencilBuffer;
device.DepthStencilBuffer = canvas.GetDepthStencil();
device.SetRenderTarget(0, dsChain[0]);
canvas.DrawImage(image, Vector2.One * -1, Vector2.One, (int)ImageParameters.FlipY);
device.SetRenderTarget(0, null);
downsampleMaxLumShader.SetupShader(canvas);
for(int i = 1; i < dsChain.Length; i++)
{
device.Textures[0] = dsChain[i-1].GetTexture();
Vector2 invRes = Vector2.One / new Vector2(dsChain[i-1].Width, dsChain[i-1].Height);
device.SetVertexShaderConstant(CanvasShaderConstants.VC_INVTEXRES, invRes);
device.SetPixelShaderConstant(0, invRes);
device.SetRenderTarget(0, dsChain[i]);
CanvasPrimitives.Quad.Render(canvas);
device.SetRenderTarget(0, null);
}
biasLumShader.SetupShader(canvas);
int idx = dsChain.Length - 1;
device.Textures[0] = dsChain[idx].GetTexture();
Vector2 iRes = Vector2.One / new Vector2(dsChain[idx].Width, dsChain[idx].Height);
device.SetVertexShaderConstant(CanvasShaderConstants.VC_INVTEXRES, iRes);
device.SetRenderTarget(0, dsChain[idx]);
CanvasPrimitives.Quad.Render(canvas);
device.SetRenderTarget(0, null);
device.DepthStencilBuffer = dsOld;
return FloodFillBounds(dsChain[idx].GetTexture());
}
void DetectFingers(Canvas canvas)
{
BoundingRect[] drawRects = GetCameraDrawBoxes();
fingersA = DetectFingersFromImage(canvas, cameraA.GetCameraImage());
fingersB = DetectFingersFromImage(canvas, cameraB.GetCameraImage());
fingerPairs = DetectFingerPairs();
ComputeDetectionPoints();
}
void BuildDSChain(Canvas canvas)
{
dsChain = new RenderTarget2D[4];
int dsChainRes = Math.Min(renderTarget.Width, renderTarget.Height);
for (int i = 0; i < dsChain.Length; i++)
{
dsChain[i] = new RenderTarget2D(canvas.GetDevice(), dsChainRes, dsChainRes, 1, SurfaceFormat.Color);
dsChainRes /= 2;
}
}
void RenderMatrixStyle(Canvas canvas)
{
GraphicsDevice device = canvas.GetDevice();
matrixShader.SetupShader(canvas);
device.Textures[0] = cameraA.GetCameraImage();
device.Textures[1] = unicodeTable;
//device.Textures[2] = noiseTexture;
device.SetVertexShaderConstant(CanvasShaderConstants.VC_INVTEXRES, Vector2.Zero);
device.SetPixelShaderConstant(0, new Vector4(1.0f / (float)unicodeTable.Width, 1.0f / (float)unicodeTable.Height,
(float)unicodeTableRows / (float)unicodeTable.Width, (float)unicodeTableColumns / (float)unicodeTable.Height));
device.SetPixelShaderConstant(1, new Vector4(0, 1.0f, 0, 1.0f));
DepthStencilBuffer dsOld = device.DepthStencilBuffer;
device.DepthStencilBuffer = canvas.GetDepthStencil();
device.SetRenderTarget(0, renderTarget);
CanvasPrimitives.Quad.Render(canvas);
device.SetRenderTarget(0, null);
device.DepthStencilBuffer = dsOld;
cameraAImage = renderTarget.GetTexture();
}
public virtual void OnRender(Canvas canvas)
{
}
public void SetupMaterial(Canvas canvas)
{
shader.SetupShader(canvas);
SetupTextures(canvas);
}
public void Render(Canvas canvas)
{
GraphicsDevice device = canvas.GetDevice();
device.VertexDeclaration = Geometry.renderElement.VertexDec;
device.Indices = Geometry.renderElement.IndexBuffer;
device.Vertices[0].SetSource(Geometry.renderElement.VertexBuffer, 0, Geometry.renderElement.VertexStride);
device.DrawIndexedPrimitives(PrimitiveType.TriangleList, 0, Geometry.renderElement.StartVertex, Geometry.renderElement.VertexCount, 0, Geometry.renderElement.PrimitiveCount);
}
void UndistortCameraImages(Canvas canvas)
{
GraphicsDevice device = canvas.GetDevice();
undistortShader.SetupShader(canvas);
device.Textures[0] = cameraA.GetCameraImage();
device.SetPixelShaderConstant(0, new Vector4((float)DistortParamsA[0],
(float)DistortParamsA[1], (float)DistortParamsA[2], (float)DistortParamsA[3]));
device.SetPixelShaderConstant(1, new Vector4((float)CameraParamsA[0],
(float)CameraParamsA[1], (float)CameraParamsA[2], (float)CameraParamsA[3])
/ new Vector4(cameraA.Width, cameraA.Height, cameraA.Width, cameraA.Height));
DepthStencilBuffer dsOld = device.DepthStencilBuffer;
device.DepthStencilBuffer = canvas.GetDepthStencil();
device.SetRenderTarget(0, renderTarget);
CanvasPrimitives.Quad.Render(canvas);
device.SetRenderTarget(0, null);
device.DepthStencilBuffer = dsOld;
cameraAImage = renderTarget.GetTexture();
}
public RenderLayer(Canvas canvas, int width, int height)
{
RenderTarget = new RenderTarget2D(canvas.GetDevice(), width, height, 1, SurfaceFormat.Color);
GlowTarget = new RenderTarget2D(canvas.GetDevice(), width, height, 1, SurfaceFormat.Color);
}