private async Task InitializeFaceDetection()
{
if (FaceDetector.IsSupported)
{
if (_faceDetector == null)
{
_faceDetector = await FaceDetector.CreateAsync();
_faceDectorSupportedPixelFormat = FaceDetector.GetSupportedBitmapPixelFormats().FirstOrDefault();
}
}
else
{
Debug.WriteLine("Face detection is not supported");
}
if (FaceTracker.IsSupported)
{
if (_faceTracker == null)
{
_faceTracker = await FaceTracker.CreateAsync();
_faceTrackerSupportedPixelFormat = FaceTracker.GetSupportedBitmapPixelFormats().FirstOrDefault();
}
}
else
{
Debug.WriteLine("Face tracking is not suppoted");
}
}
private async Task InitializeFaceDetection()
{
if (FaceDetector.IsSupported)
{
if (faceDetector == null)
{
faceDetector = await FaceDetector.CreateAsync();
faceDetectorSupportedPixelFormat = FaceDetector.GetSupportedBitmapPixelFormats().FirstOrDefault();
}
}
else
{
Debug.WriteLine("Warning. FaceDetector is not supported on this device");
}
if (FaceTracker.IsSupported)
{
if (faceTracker == null)
{
faceTracker = await FaceTracker.CreateAsync();
faceTrackerSupportedPixelFormat = FaceTracker.GetSupportedBitmapPixelFormats().FirstOrDefault();
}
}
else
{
Debug.WriteLine("Warning. FaceTracking is not supported on this device");
}
}
/// <summary>
/// This is just one big lump of code right now which should be factored out into some kind of
/// 'frame reader' class which can then be subclassed for depth frame and video frame but
/// it was handy to have it like this while I experimented with it - the intention was
/// to tidy it up if I could get it doing more or less what I wanted :-)
/// </summary>
async Task ProcessingLoopAsync()
{
var depthMediaCapture = await this.GetMediaCaptureForDescriptionAsync(
MediaFrameSourceKind.Depth, 448, 450, 15);
var depthFrameReader = await depthMediaCapture.Item1.CreateFrameReaderAsync(depthMediaCapture.Item2);
depthFrameReader.AcquisitionMode = MediaFrameReaderAcquisitionMode.Realtime;
MediaFrameReference lastDepthFrame = null;
long depthFrameCount = 0;
float centrePointDepthInMetres = 0.0f;
// Expecting this to run at 1fps although the API (seems to) reports that it runs at 15fps
TypedEventHandler <MediaFrameReader, MediaFrameArrivedEventArgs> depthFrameHandler =
(sender, args) =>
{
using (var depthFrame = sender.TryAcquireLatestFrame())
{
if ((depthFrame != null) && (depthFrame != lastDepthFrame))
{
lastDepthFrame = depthFrame;
Interlocked.Increment(ref depthFrameCount);
// Always try to grab the depth value although, clearly, this is subject
// to a bunch of race conditions etc. as other thread access it.
centrePointDepthInMetres =
GetDepthValueAtCoordinate(depthFrame,
(int)(depthFrame.Format.VideoFormat.Width * MAGIC_DEPTH_FRAME_WIDTH_RATIO_CENTRE),
(int)(depthFrame.Format.VideoFormat.Height * MAGIC_DEPTH_FRAME_HEIGHT_RATIO_CENTRE)) ?? 0.0f;
}
}
};
long rgbProcessedCount = 0;
long facesPresentCount = 0;
long rgbDroppedCount = 0;
MediaFrameReference lastRgbFrame = null;
var faceBitmapFormats = FaceTracker.GetSupportedBitmapPixelFormats().Select(
format => format.ToString().ToLower()).ToArray();
var faceTracker = await FaceTracker.CreateAsync();
var rgbMediaCapture = await this.GetMediaCaptureForDescriptionAsync(
MediaFrameSourceKind.Color, 1280, 720, 30, faceBitmapFormats);
var rgbFrameReader = await rgbMediaCapture.Item1.CreateFrameReaderAsync(rgbMediaCapture.Item2);
rgbFrameReader.AcquisitionMode = MediaFrameReaderAcquisitionMode.Realtime;
int busyProcessingRgbFrame = 0;
var unityWorldCoordinateSystem =
Marshal.GetObjectForIUnknown(WorldManager.GetNativeISpatialCoordinateSystemPtr()) as SpatialCoordinateSystem;
// Expecting this to run at 30fps.
TypedEventHandler <MediaFrameReader, MediaFrameArrivedEventArgs> rgbFrameHandler =
(sender, args) =>
{
// Only proceed if we're not already 'busy' - i.e. we'
if (Interlocked.CompareExchange(ref busyProcessingRgbFrame, 1, 0) == 0)
{
Task.Run(
async() =>
{
using (var rgbFrame = rgbFrameReader.TryAcquireLatestFrame())
{
if ((rgbFrame != null) && (rgbFrame != lastRgbFrame))
{
++rgbProcessedCount;
lastRgbFrame = rgbFrame;
var facePosition = uVector3.zero;
using (var videoFrame = rgbFrame.VideoMediaFrame.GetVideoFrame())
{
var faces = await faceTracker.ProcessNextFrameAsync(videoFrame);
var firstFace = faces.FirstOrDefault();
if (firstFace != null)
{
++facesPresentCount;
// Take the first face and the centre point of that face to try
// and simplify things for my limited brain.
var faceCentrePointInImageCoords =
new Point(
firstFace.FaceBox.X + (firstFace.FaceBox.Width / 2.0f),
firstFace.FaceBox.Y + (firstFace.FaceBox.Height / 2.0f));
wMatrix4x4 projectionTransform = wMatrix4x4.Identity;
wMatrix4x4 viewTransform = wMatrix4x4.Identity;
wMatrix4x4 invertedViewTransform = wMatrix4x4.Identity;
var rgbCoordinateSystem = GetRgbFrameProjectionAndCoordinateSystemDetails(
rgbFrame, out projectionTransform, out invertedViewTransform);
// Scale the RGB point (1280x720)
var faceCentrePointUnitScaleRGB = ScalePointMinusOneToOne(faceCentrePointInImageCoords, rgbFrame);
// Unproject the RGB point back at unit depth as per the locatable camera
// document.
var unprojectedFaceCentrePointRGB = UnProjectVector(
new wVector3(
(float)faceCentrePointUnitScaleRGB.X,
(float)faceCentrePointUnitScaleRGB.Y,
1.0f),
projectionTransform);
// Transfrom this back by the inverted view matrix in order to put this into
// the RGB camera coordinate system
var faceCentrePointCameraCoordsRGB =
wVector3.Transform(unprojectedFaceCentrePointRGB, invertedViewTransform);
// Get the transform from the camera coordinate system to the Unity world
// coordinate system, could probably cache this?
var cameraRGBToWorldTransform =
rgbCoordinateSystem.TryGetTransformTo(unityWorldCoordinateSystem);
if (cameraRGBToWorldTransform.HasValue)
{
// Transform to world coordinates
var faceCentrePointWorldCoords = wVector4.Transform(
new wVector4(
faceCentrePointCameraCoordsRGB.X,
faceCentrePointCameraCoordsRGB.Y,
faceCentrePointCameraCoordsRGB.Z, 1),
cameraRGBToWorldTransform.Value);
// Where's the camera in world coordinates?
var cameraOriginWorldCoords = wVector4.Transform(
new wVector4(0, 0, 0, 1),
cameraRGBToWorldTransform.Value);
// Multiply Z by -1 for Unity
var cameraPoint = new uVector3(
cameraOriginWorldCoords.X,
cameraOriginWorldCoords.Y,
-1.0f * cameraOriginWorldCoords.Z);
// Multiply Z by -1 for Unity
var facePoint = new uVector3(
faceCentrePointWorldCoords.X,
faceCentrePointWorldCoords.Y,
-1.0f * faceCentrePointWorldCoords.Z);
facePosition =
cameraPoint +
(facePoint - cameraPoint).normalized * centrePointDepthInMetres;
}
}
}
if (facePosition != uVector3.zero)
{
UnityEngine.WSA.Application.InvokeOnAppThread(
() =>
{
this.faceMarker.transform.position = facePosition;
},
false
);
}
}
}
Interlocked.Exchange(ref busyProcessingRgbFrame, 0);
}
);
}
else
{
Interlocked.Increment(ref rgbDroppedCount);
}
// NB: this is a bit naughty as I am accessing these counters across a few threads so
// accuracy might suffer here.
UnityEngine.WSA.Application.InvokeOnAppThread(
() =>
{
this.textMesh.text =
$"{depthFrameCount} depth,{rgbProcessedCount} rgb done, {rgbDroppedCount} rgb drop," +
$"{facesPresentCount} faces, ({centrePointDepthInMetres:N2})";
},
false);
};
depthFrameReader.FrameArrived += depthFrameHandler;
rgbFrameReader.FrameArrived += rgbFrameHandler;
await depthFrameReader.StartAsync();
await rgbFrameReader.StartAsync();
// Wait forever then dispose...just doing this to keep track of what needs disposing.
await Task.Delay(-1);
depthFrameReader.FrameArrived -= depthFrameHandler;
rgbFrameReader.FrameArrived -= rgbFrameHandler;
rgbFrameReader.Dispose();
depthFrameReader.Dispose();
rgbMediaCapture.Item1.Dispose();
depthMediaCapture.Item1.Dispose();
Marshal.ReleaseComObject(unityWorldCoordinateSystem);
}
