/// <summary>
/// Initializes a new MediaCapture instance and starts the Preview streaming to the CamPreview UI element.
/// </summary>
/// <returns>Async Task object returning true if initialization and streaming were successful and false if an exception occurred.</returns>
//private async Task<bool> StartWebcamStreaming()
//{
// bool successful = true;
// try
// {
// this.mediaCapture = new MediaCapture();
// // For this scenario, we only need Video (not microphone) so specify this in the initializer.
// // NOTE: the appxmanifest only declares "webcam" under capabilities and if this is changed to include
// // microphone (default constructor) you must add "microphone" to the manifest or initialization will fail.
// MediaCaptureInitializationSettings settings = new MediaCaptureInitializationSettings();
// settings.StreamingCaptureMode = StreamingCaptureMode.Video;
// await this.mediaCapture.InitializeAsync(settings);
// this.mediaCapture.Failed += this.MediaCapture_CameraStreamFailed;
// // Cache the media properties as we'll need them later.
// var deviceController = this.mediaCapture.VideoDeviceController;
// this.videoProperties = deviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties;
// // Immediately start streaming to our CaptureElement UI.
// // NOTE: CaptureElement's Source must be set before streaming is started.
// this.CamPreview.Source = this.mediaCapture;
// await this.mediaCapture.StartPreviewAsync();
// // Ensure the Semaphore is in the signalled state.
// this.frameProcessingSemaphore.Release();
// // Use a 66 milisecond interval for our timer, i.e. 15 frames per second
// TimeSpan timerInterval = TimeSpan.FromMilliseconds(66);
// this.frameProcessingTimer = Windows.System.Threading.ThreadPoolTimer.CreatePeriodicTimer(new Windows.System.Threading.TimerElapsedHandler(ProcessCurrentVideoFrame), timerInterval);
// }
// catch (System.UnauthorizedAccessException)
// {
// // If the user has disabled their webcam this exception is thrown; provide a descriptive message to inform the user of this fact.
// //this.rootPage.NotifyUser("Webcam is disabled or access to the webcam is disabled for this app.\nEnsure Privacy Settings allow webcam usage.", NotifyType.ErrorMessage);
// successful = false;
// }
// catch (Exception ex)
// {
// //this.rootPage.NotifyUser(ex.ToString(), NotifyType.ErrorMessage);
// successful = false;
// }
// return successful;
//}
/// <summary>
/// Safely stops webcam streaming (if running) and releases MediaCapture object.
/// </summary>
//private async void ShutdownWebCam()
//{
// if (this.frameProcessingTimer != null)
// {
// this.frameProcessingTimer.Cancel();
// }
// if (this.mediaCapture != null)
// {
// if (this.mediaCapture.CameraStreamState == Windows.Media.Devices.CameraStreamState.Streaming)
// {
// try
// {
// await this.mediaCapture.StopPreviewAsync();
// }
// catch (Exception)
// {
// ; // Since we're going to destroy the MediaCapture object there's nothing to do here
// }
// }
// this.mediaCapture.Dispose();
// }
// this.frameProcessingTimer = null;
// this.CamPreview.Source = null;
// this.mediaCapture = null;
// this.CameraStreamingButton.IsEnabled = true;
//}
/// <summary>
/// This method is invoked by a ThreadPoolTimer to execute the FaceTracker and Visualization logic at approximately 15 frames per second.
/// </summary>
/// <remarks>
/// Keep in mind this method is called from a Timer and not sychronized with the camera stream. Also, the processing time of FaceTracker
/// will vary depending on the size of each frame and the number of faces being tracked. That is, a large image with several tracked faces may
/// take longer to process.
/// </remarks>
/// <param name="timer">Timer object invoking this call</param>
private async void ProcessCurrentVideoFrame(ThreadPoolTimer timer)
{
//if (this.currentState != ScenarioState.Streaming)
//{
// return;
//}
// If a lock is being held it means we're still waiting for processing work on the previous frame to complete.
// In this situation, don't wait on the semaphore but exit immediately.
if (!frameProcessingSemaphore.Wait(0))
{
return;
}
try
{
IList <DetectedFace> faces = null;
// Create a VideoFrame object specifying the pixel format we want our capture image to be (NV12 bitmap in this case).
// GetPreviewFrame will convert the native webcam frame into this format.
const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Nv12;
using (VideoFrame previewFrame = new VideoFrame(InputPixelFormat, (int)this.videoProperties.Width, (int)this.videoProperties.Height))
{
await this.mediaCapture.GetPreviewFrameAsync(previewFrame);
// The returned VideoFrame should be in the supported NV12 format but we need to verify this.
if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat))
{
faces = await this.faceTracker.ProcessNextFrameAsync(previewFrame);
}
else
{
throw new System.NotSupportedException("PixelFormat '" + InputPixelFormat.ToString() + "' is not supported by FaceDetector");
}
// Create our visualization using the frame dimensions and face results but run it on the UI thread.
var previewFrameSize = new Windows.Foundation.Size(previewFrame.SoftwareBitmap.PixelWidth, previewFrame.SoftwareBitmap.PixelHeight);
var ignored = rootPage.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
this.SetupVisualization(previewFrameSize, faces);
});
}
}
catch (Exception ex)
{
//var ignored = rootPage.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
//{
// this.rootPage.NotifyUser(ex.ToString(), NotifyType.ErrorMessage);
//});
}
finally
{
frameProcessingSemaphore.Release();
}
}
/// <summary>
/// Captures a single frame from the running webcam stream and executes the FaceDetector on the image. If successful calls SetupVisualization to display the results.
/// </summary>
/// <returns>Async Task object returning true if the capture was successful and false if an exception occurred.</returns>
private async Task <bool> TakeSnapshotAndFindFaces()
{
bool successful = true;
try
{
if (this.currentState != ScenarioState.Streaming)
{
return(false);
}
WriteableBitmap displaySource = null;
IList <DetectedFace> faces = null;
// Create a VideoFrame object specifying the pixel format we want our capture image to be (NV12 bitmap in this case).
// GetPreviewFrame will convert the native webcam frame into this format.
const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Nv12;
using (VideoFrame previewFrame = new VideoFrame(InputPixelFormat, (int)this.videoProperties.Width, (int)this.videoProperties.Height))
{
await this.mediaCapture.GetPreviewFrameAsync(previewFrame);
// The returned VideoFrame should be in the supported NV12 format but we need to verify this.
if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat))
{
faces = await this.faceDetector.DetectFacesAsync(previewFrame.SoftwareBitmap);
}
else
{
this.rootPage.NotifyUser("PixelFormat '" + InputPixelFormat.ToString() + "' is not supported by FaceDetector", NotifyType.ErrorMessage);
}
// Create a WritableBitmap for our visualization display; copy the original bitmap pixels to wb's buffer.
// Note that WriteableBitmap doesn't support NV12 and we have to convert it to 32-bit BGRA.
using (SoftwareBitmap convertedSource = SoftwareBitmap.Convert(previewFrame.SoftwareBitmap, BitmapPixelFormat.Bgra8))
{
displaySource = new WriteableBitmap(convertedSource.PixelWidth, convertedSource.PixelHeight);
convertedSource.CopyToBuffer(displaySource.PixelBuffer);
}
// Create our display using the available image and face results.
this.SetupVisualization(displaySource, faces);
}
}
catch (Exception ex)
{
this.rootPage.NotifyUser(ex.ToString(), NotifyType.ErrorMessage);
successful = false;
}
return(successful);
}
private async void DetectFaces()
{
if (file != null)
{
// Open the image file and decode the bitmap into memory.
// We'll need to make 2 bitmap copies: one for the FaceDetector and another to display.
using (IRandomAccessStream fileStream = await file.OpenAsync(FileAccessMode.Read))
{
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(fileStream);
BitmapTransform transform = this.ComputeScalingTransformForSourceImage(decoder);
using (SoftwareBitmap originalBitmap = await decoder.GetSoftwareBitmapAsync(decoder.BitmapPixelFormat, BitmapAlphaMode.Ignore, transform, ExifOrientationMode.IgnoreExifOrientation, ColorManagementMode.DoNotColorManage))
{
// We need to convert the image into a format that's compatible with FaceDetector.
// Gray8 should be a good type but verify it against FaceDetector’s supported formats.
const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Gray8;
if (FaceDetector.IsBitmapPixelFormatSupported(InputPixelFormat))
{
using (detectorInput = SoftwareBitmap.Convert(originalBitmap, InputPixelFormat))
{
// Create a WritableBitmap for our visualization display; copy the original bitmap pixels to wb's buffer.
displaySource = new WriteableBitmap(originalBitmap.PixelWidth, originalBitmap.PixelHeight);
originalBitmap.CopyToBuffer(displaySource.PixelBuffer);
NotifyUser("Detecting...", NotifyType.StatusMessage);
// Initialize our FaceDetector and execute it against our input image.
// NOTE: FaceDetector initialization can take a long time, and in most cases
// you should create a member variable and reuse the object.
// However, for simplicity in this scenario we instantiate a new instance each time.
FaceDetector detector = await FaceDetector.CreateAsync();
faces = await detector.DetectFacesAsync(detectorInput);
// Create our display using the available image and face results.
DrawDetectedFaces(displaySource, faces);
}
}
else
{
NotifyUser("PixelFormat '" + InputPixelFormat.ToString() + "' is not supported by FaceDetector", NotifyType.ErrorMessage);
}
}
}
}
}
private async void ProcessCurrentVideoFrame(ThreadPoolTimer timer)
{
// If a lock is being held it means we're still waiting for processing work on the previous frame to complete.
// In this situation, don't wait on the semaphore but exit immediately.
if (!frameProcessingSemaphore.Wait(0))
{
return;
}
try
{
IList <DetectedFace> faces = null;
// Create a VideoFrame object specifying the pixel format we want our capture image to be (NV12 bitmap in this case).
// GetPreviewFrame will convert the native webcam frame into this format.
const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Nv12;
using (VideoFrame previewFrame = new VideoFrame(InputPixelFormat, (int)this.videoProperties.Width, (int)this.videoProperties.Height))
{
await this.mediaCapture.GetPreviewFrameAsync(previewFrame);
//Setting the format of the picture to send to cognitive services
var imageEncodingProp = ImageEncodingProperties.CreateJpeg();
var stream = new InMemoryRandomAccessStream();
//Capturing the picture and stoing into the Main memory
await mediaCapture.CapturePhotoToStreamAsync(imageEncodingProp, stream);
stream.Seek(0);
//Making a copy of the bite stream to send it to the cognitive services
var age_stream = stream.CloneStream();
//Getting the list of the emotions of the faces
var emotions = await GetEmotions(stream.AsStreamForRead());
//Getting the list of the gender and age of the faces
var ageandgender = await GetFaces(age_stream.AsStreamForRead());
// The returned VideoFrame should be in the supported NV12 format but we need to verify this.
if (FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat))
{
//Returning the dected faces using the Media analysis library in .Net no need for internet here
faces = await this.faceTracker.ProcessNextFrameAsync(previewFrame);
}
else
{
throw new System.NotSupportedException("PixelFormat '" + InputPixelFormat.ToString() + "' is not supported by FaceDetector");
}
// Create our visualization using the frame dimensions and face results but run it on the UI thread.
var previewFrameSize = new Windows.Foundation.Size(previewFrame.SoftwareBitmap.PixelWidth, previewFrame.SoftwareBitmap.PixelHeight);
var ignored = this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
this.SetupVisualization(previewFrameSize, faces, emotions, ageandgender);
});
}
}
catch (Exception ex)
{
}
finally
{
frameProcessingSemaphore.Release();
}
}
/// <summary>
/// Loads an image file (selected by the user) and runs the FaceDetector on the loaded bitmap. If successful calls SetupVisualization to display the results.
/// </summary>
/// <param name="sender">Button user clicked</param>
/// <param name="e">Event data</param>
private async void OpenFile_Click(object sender, RoutedEventArgs e)
{
FileOpenPicker photoPicker = new FileOpenPicker();
photoPicker.ViewMode = PickerViewMode.Thumbnail;
photoPicker.SuggestedStartLocation = PickerLocationId.PicturesLibrary;
photoPicker.FileTypeFilter.Add(".jpg");
photoPicker.FileTypeFilter.Add(".jpeg");
photoPicker.FileTypeFilter.Add(".png");
photoPicker.FileTypeFilter.Add(".bmp");
StorageFile photoFile = await photoPicker.PickSingleFileAsync();
if (photoFile == null)
{
return;
}
this.ClearVisualization();
this.rootPage.NotifyUser("Opening...", NotifyType.StatusMessage);
try
{
// Open the image file and decode the bitmap into memory.
// We'll need to make 2 bitmap copies: one for the FaceDetector and another to display.
using (IRandomAccessStream fileStream = await photoFile.OpenAsync(Windows.Storage.FileAccessMode.Read))
{
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(fileStream);
BitmapTransform transform = this.ComputeScalingTransformForSourceImage(decoder);
using (SoftwareBitmap originalBitmap = await decoder.GetSoftwareBitmapAsync(decoder.BitmapPixelFormat, BitmapAlphaMode.Ignore, transform, ExifOrientationMode.IgnoreExifOrientation, ColorManagementMode.DoNotColorManage))
{
// We need to convert the image into a format that's compatible with FaceDetector.
// Gray8 should be a good type but verify it against FaceDetector’s supported formats.
const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Gray8;
if (FaceDetector.IsBitmapPixelFormatSupported(InputPixelFormat))
{
using (SoftwareBitmap detectorInput = SoftwareBitmap.Convert(originalBitmap, InputPixelFormat))
{
// Create a WritableBitmap for our visualization display; copy the original bitmap pixels to WriteableBitmap's buffer.
WriteableBitmap displaySource = new WriteableBitmap(originalBitmap.PixelWidth, originalBitmap.PixelHeight);
originalBitmap.CopyToBuffer(displaySource.PixelBuffer);
this.rootPage.NotifyUser("Detecting...", NotifyType.StatusMessage);
// Initialize our FaceDetector and execute it against our input image.
// NOTE: FaceDetector initialization can take a long time, and in most cases
// you should create a member variable and reuse the object.
// However, for simplicity in this scenario we instantiate a new instance each time.
FaceDetector detector = await FaceDetector.CreateAsync();
IList <DetectedFace> faces = await detector.DetectFacesAsync(detectorInput);
// Create our display using the available image and face results.
this.SetupVisualization(displaySource, faces);
}
}
else
{
this.rootPage.NotifyUser("PixelFormat '" + InputPixelFormat.ToString() + "' is not supported by FaceDetector", NotifyType.ErrorMessage);
}
}
}
}
catch (Exception ex)
{
this.ClearVisualization();
this.rootPage.NotifyUser(ex.ToString(), NotifyType.ErrorMessage);
}
}
/// <summary>
/// Extracts a frame from the camera stream and detects if any faces are found. Used as a precursor to making an expensive API
/// call to get proper face details.
/// </summary>
/// <remarks>
/// Keep in mind this method is called from a Timer and not synchronized with the camera stream. Also, the processing time of FaceTracker
/// will vary depending on the size of each frame and the number of faces being tracked. That is, a large image with several tracked faces may
/// take longer to process.
/// </remarks>
private async Task <ApiRequestParameters> ProcessCurrentVideoFrameAsync()
{
// If a lock is being held it means we're still waiting for processing work on the previous frame to complete.
// In this situation, don't wait on the semaphore but exit immediately.
if (!frameProcessingSemaphore.Wait(0))
{
return(null);
}
try
{
// Create a VideoFrame object specifying the pixel format we want our capture image to be (NV12 bitmap in this case).
// GetPreviewFrame will convert the native webcam frame into this format.
const BitmapPixelFormat InputPixelFormat = BitmapPixelFormat.Nv12;
using (var previewFrame = new VideoFrame(InputPixelFormat, (int)videoProperties.Width, (int)videoProperties.Height))
{
await mediaCapture.GetPreviewFrameAsync(previewFrame);
// The returned VideoFrame should be in the supported NV12 format but we need to verify this.
if (!FaceDetector.IsBitmapPixelFormatSupported(previewFrame.SoftwareBitmap.BitmapPixelFormat))
{
throw new NotSupportedException("PixelFormat '" + InputPixelFormat.ToString() + "' is not supported by FaceDetector");
}
var faces = await faceTracker.ProcessNextFrameAsync(previewFrame);
if (faces.Any())
{
// Found faces so create a bounding rectangle and store the parameters to make the API call and process the response.
using (var ms = new MemoryStream())
{
// It'll be faster to send a smaller rectangle of the faces found instead of the whole image. This is what we do here.
var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, ms.AsRandomAccessStream());
// To use the encoder to resize we need to change the bitmap format. Might be a better way to do this, I can't see it.
var converted = SoftwareBitmap.Convert(previewFrame.SoftwareBitmap, BitmapPixelFormat.Rgba16);
encoder.SetSoftwareBitmap(converted);
//var bounds = boundingBoxCreator.BoundingBoxForFaces(faces, converted.PixelWidth, converted.PixelHeight);
//encoder.BitmapTransform.Bounds = bounds;
await encoder.FlushAsync();
LogStatusMessage($"Found face(s) on camera: {faces.Count}", StatusSeverity.Info, false);
return(new ApiRequestParameters
{
Image = ms.ToArray(),
Faces = faces
});
}
}
return(null);
}
}
catch (Exception ex)
{
LogStatusMessage("Unable to process current frame: " + ex.ToString(), StatusSeverity.Error, false);
return(null);
}
finally
{
frameProcessingSemaphore.Release();
}
}
