/// <summary>
/// Find and expose all formats exposed by the frame source currently toggled
/// </summary>
private void InitializeResolutionSettings()
{
cmbResolution.Items.Clear();
_filteredSourceFormat = new List <MediaFrameFormat>();
//retrieve all supported formats
List <MediaFrameFormat> supportedFormats = _selectedFrameSource.SupportedFormats.ToList();
uint maxResolution = 0;
int maxResolutionPropertyIndex = 0;
int supportedFormatIndex = 0;
// filter out and skip mjpeg and rgb24 video formats
foreach (MediaFrameFormat format in supportedFormats)
{
VideoMediaFrameFormat videoFormat = format.VideoFormat;
string subtype = format.Subtype.ToLowerInvariant();
if (subtype == MediaEncodingSubtypes.Mjpg.ToLowerInvariant() ||
(format.FrameRate.Denominator <= 0) //workaround for some bad camera f/w
)
{
continue;
}
uint resolution = videoFormat.Width * videoFormat.Height;
if (resolution > maxResolution)
{
maxResolution = resolution;
maxResolutionPropertyIndex = supportedFormatIndex;
}
Debug.WriteLine("Resolution of : {0} x {1}", videoFormat.Width, videoFormat.Height);
Debug.WriteLine("Type: {0} and subtype: {1}", format.MajorType, format.Subtype);
string aspectRatio = GetAspectRatio(videoFormat.Width, videoFormat.Height);
cmbResolution.Items.Add(string.Format("{0}, {1}, {2}MP, {3}x{4}@{5}fps",
format.Subtype,
aspectRatio,
((float)((videoFormat.Width * videoFormat.Height) / 1000000.0)).ToString("0.00"),
videoFormat.Width, videoFormat.Height,
format.FrameRate.Numerator / format.FrameRate.Denominator));
_filteredSourceFormat.Add(format);
supportedFormatIndex++;
}
// Select maximum resolution available
cmbResolution.SelectedIndex = maxResolutionPropertyIndex;
}
private void ProcessFaces(List <BitmapBounds> faces, MediaFrameReference frame, SpatialCoordinateSystem worldCoordSystem)
{
VideoMediaFrameFormat videoFormat = frame.VideoMediaFrame.VideoFormat;
SpatialCoordinateSystem cameraCoordinateSystem = frame.CoordinateSystem;
CameraIntrinsics cameraIntrinsics = frame.VideoMediaFrame.CameraIntrinsics;
System.Numerics.Matrix4x4?cameraToWorld = cameraCoordinateSystem.TryGetTransformTo(worldCoordSystem);
// If we can't locate the world, this transform will be null.
if (!cameraToWorld.HasValue)
{
return;
}
float textureWidthInv = 1.0f / videoFormat.Width;
float textureHeightInv = 1.0f / videoFormat.Height;
// The face analysis returns very "tight fitting" rectangles.
// We add some padding to make the visuals more appealing.
int paddingForFaceRect = 24;
float averageFaceWidthInMeters = 0.15f;
float pixelsPerMeterAlongX = cameraIntrinsics.FocalLength.X;
float averagePixelsForFaceAt1Meter = pixelsPerMeterAlongX * averageFaceWidthInMeters;
// Place the cube 25cm above the center of the face.
System.Numerics.Vector3 cubeOffsetInWorldSpace = new System.Numerics.Vector3(0.0f, 0.25f, 0.0f);
BitmapBounds bestRect = new BitmapBounds();
System.Numerics.Vector3 bestRectPositionInCameraSpace = System.Numerics.Vector3.Zero;
float bestDotProduct = -1.0f;
foreach (BitmapBounds faceRect in faces)
{
Point faceRectCenterPoint = new Point(faceRect.X + faceRect.Width / 2u, faceRect.Y + faceRect.Height / 2u);
// Calculate the vector towards the face at 1 meter.
System.Numerics.Vector2 centerOfFace = cameraIntrinsics.UnprojectAtUnitDepth(faceRectCenterPoint);
// Add the Z component and normalize.
System.Numerics.Vector3 vectorTowardsFace = System.Numerics.Vector3.Normalize(new System.Numerics.Vector3(centerOfFace.X, centerOfFace.Y, -1.0f));
// Estimate depth using the ratio of the current faceRect width with the average faceRect width at 1 meter.
float estimatedFaceDepth = averagePixelsForFaceAt1Meter / faceRect.Width;
// Get the dot product between the vector towards the face and the gaze vector.
// The closer the dot product is to 1.0, the closer the face is to the middle of the video image.
float dotFaceWithGaze = System.Numerics.Vector3.Dot(vectorTowardsFace, -System.Numerics.Vector3.UnitZ);
// Scale the vector towards the face by the depth, and add an offset for the cube.
System.Numerics.Vector3 targetPositionInCameraSpace = vectorTowardsFace * estimatedFaceDepth;
// Pick the faceRect that best matches the users gaze.
if (dotFaceWithGaze > bestDotProduct)
{
bestDotProduct = dotFaceWithGaze;
bestRect = faceRect;
bestRectPositionInCameraSpace = targetPositionInCameraSpace;
}
}
// Transform the cube from Camera space to World space.
System.Numerics.Vector3 bestRectPositionInWorldspace = System.Numerics.Vector3.Transform(bestRectPositionInCameraSpace, cameraToWorld.Value);
cubeRenderer.SetTargetPosition(bestRectPositionInWorldspace + cubeOffsetInWorldSpace);
// Texture Coordinates are [0,1], but our FaceRect is [0,Width] and [0,Height], so we need to normalize these coordinates
// We also add padding for the faceRects to make it more visually appealing.
float normalizedWidth = (bestRect.Width + paddingForFaceRect * 2u) * textureWidthInv;
float normalizedHeight = (bestRect.Height + paddingForFaceRect * 2u) * textureHeightInv;
float normalizedX = (bestRect.X - paddingForFaceRect) * textureWidthInv;
float normalizedY = (bestRect.Y - paddingForFaceRect) * textureHeightInv;
}
