protected override void OnElementChanged(ElementChangedEventArgs <View> e)
{
base.OnElementChanged(e);
if (e.OldElement != null || Element == null)
{
return;
}
if (e.NewElement != null)
{
if (Control == null)
{
liveCameraStream = new UICameraPreview();
SetNativeControl(liveCameraStream);
var cameraView = ((CameraView)e.NewElement);
liveCameraStream.OnDetected += (list) =>
{
cameraView?.TriggerOnDetected(list);
};
cameraView.ViewSizeChanged += (obj, arg) =>
{
liveCameraStream.SizeChange();
};
}
}
}
private async void UIButtonStart_OnClick(object sender, RoutedEventArgs e)
{
try
{
UICameraPreview.Stop();
if (UICameraPreview.CameraHelper != null)
{
await UICameraPreview.CameraHelper.CleanUpAsync();
}
m_isCameraFrameDimensionInitialized = false;
// Initialize skill with the selected supported device
m_skeletalDetectorSkill = await m_skeletalDetectorDescriptor.CreateSkillAsync(m_availableExecutionDevices[0]) as SkeletalDetectorSkill;
// Initialize the CameraPreview control, register frame arrived event callback
UIImageViewer.Visibility = Visibility.Collapsed;
UICameraPreview.Visibility = Visibility.Visible;
await UICameraPreview.StartAsync();
UICameraPreview.CameraHelper.FrameArrived += CameraHelper_FrameArrived;
m_currentFrameSourceToggled = FrameSourceToggledType.Camera;
}
catch (Exception ex)
{
await(new MessageDialog(ex.Message)).ShowAsync();
m_currentFrameSourceToggled = FrameSourceToggledType.None;
}
finally
{
m_lock.Release();
}
}
/// <summary>
/// Triggered when UICameraToggle is clicked, initializes frame grabbing from the camera stream
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UICameraToggle_Click(object sender, RoutedEventArgs e)
{
UICameraToggle.IsEnabled = false;
await m_lock.WaitAsync();
try
{
UICameraPreview.Stop();
if (UICameraPreview.CameraHelper != null)
{
await UICameraPreview.CameraHelper.CleanUpAsync();
}
m_isCameraFrameDimensionInitialized = false;
// Instantiate skill only if object is null or selected execution device has changed
if ((m_paramsSkillObj.m_selectedDeviceId != UISkillExecutionDevices.SelectedIndex) || (m_paramsSkillObj.m_skill == null))
{
if (m_paramsSkillObj.m_skill != null)
{
// Release previous instance
m_paramsSkillObj.m_skill = null;
}
// Update selected device
m_paramsSkillObj.m_selectedDeviceId = UISkillExecutionDevices.SelectedIndex;
// Initialize skill with the selected supported device
m_paramsSkillObj.m_skill = await m_paramsSkillObj.m_skillDescriptor.CreateSkillAsync(m_paramsSkillObj.m_availableExecutionDevices[UISkillExecutionDevices.SelectedIndex]) as IntruderDetectionSkill;
// Instantiate a binding object that will hold the skill's input and output resource
m_paramsSkillObj.m_binding = await m_paramsSkillObj.m_skill.CreateSkillBindingAsync() as IntruderDetectionBinding;
}
// Initialize the CameraPreview control, register frame arrived event callback
UICameraPreview.Visibility = Visibility.Visible;
await UICameraPreview.StartAsync();
UICameraPreview.CameraHelper.FrameArrived += CameraHelper_FrameArrived;
// Set a specific resolution if available
var lrFrameFormat = UICameraPreview.CameraHelper.FrameFormatsAvailable.Find((format) => ((format.VideoFormat.Width == 1280) && (format.VideoFormat.Height == 720)));
if (lrFrameFormat != null)
{
await UICameraPreview.CameraHelper.PreviewFrameSource.SetFormatAsync(lrFrameFormat);
}
m_currentFrameSourceToggled = FrameSourceToggledType.Camera;
}
catch (Exception ex)
{
await(new MessageDialog(ex.Message)).ShowAsync();
m_currentFrameSourceToggled = FrameSourceToggledType.None;
}
finally
{
m_lock.Release();
}
}
/// <summary>
/// Triggered when UIButtonFilePick is clicked, grabs a frame from an image file.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UIButtonFilePick_Click(object sender, RoutedEventArgs e)
{
// Stop Camera preview
UICameraPreview.Stop();
if (UICameraPreview.CameraHelper != null)
{
await UICameraPreview.CameraHelper.CleanUpAsync();
}
UICameraPreview.Visibility = Visibility.Collapsed;
UIImageViewer.Visibility = Visibility.Visible;
// Disable subsequent trigger of this event callback
UICameraToggle.IsEnabled = false;
UIButtonFilePick.IsEnabled = false;
await m_lock.WaitAsync();
try
{
// Initialize skill with the selected supported device
m_skeletalDetectorSkill = await m_skeletalDetectorDescriptor.CreateSkillAsync(m_availableExecutionDevices[UISkillExecutionDevices.SelectedIndex]) as SkeletalDetectorSkill;
// Instantiate a binding object that will hold the skill's input and output resource
m_skeletalDetectorBinding = await m_skeletalDetectorSkill.CreateSkillBindingAsync() as SkeletalDetectorBinding;
var frame = await LoadVideoFrameFromFilePickedAsync();
if (frame != null)
{
SoftwareBitmapSource source = new SoftwareBitmapSource();
await source.SetBitmapAsync(frame.SoftwareBitmap);
UIImageViewer.Source = source;
UIImageViewer_SizeChanged(null, null);
await RunSkillAsync(frame, false);
}
m_skeletalDetectorSkill = null;
m_skeletalDetectorBinding = null;
m_currentFrameSourceToggled = FrameSourceToggledType.ImageFile;
}
catch (Exception ex)
{
await(new MessageDialog(ex.Message)).ShowAsync();
m_currentFrameSourceToggled = FrameSourceToggledType.None;
}
m_lock.Release();
// Enable subsequent trigger of this event callback
UIButtonFilePick.IsEnabled = true;
UICameraToggle.IsEnabled = true;
}
/// <summary>
/// Triggers when the camera button is clicked, results in a still of the preview frame at the moment it was taken.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UIButtonCamera_Click(object sender, RoutedEventArgs e)
{
// Stop Camera preview
UICameraPreview.Stop();
if (UICameraPreview.CameraHelper != null)
{
await UICameraPreview.CameraHelper.CleanUpAsync();
}
UICameraPreview.Visibility = Visibility.Collapsed;
UIImageViewer.Visibility = Visibility.Visible;
// Disable subsequent trigger of this event callback
UICameraToggle.IsEnabled = false;
UIButtonFilePick.IsEnabled = false;
UIButtonCamera.IsEnabled = false;
await m_lock.WaitAsync();
try
{
var frame = m_cachedFrame;
if (frame != null)
{
SoftwareBitmapSource source = new SoftwareBitmapSource();
await source.SetBitmapAsync(frame.SoftwareBitmap);
UIImageViewer.Source = source;
UIImageViewer_SizeChanged(null, null);
UIImageViewer.UpdateLayout();
await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
m_bodyRenderer.Update(m_skeletalDetectorBinding.Bodies, true);
m_bodyRenderer.IsVisible = true;
UISkillOutputDetails.Text = $"Found {m_skeletalDetectorBinding.Bodies.Count} bodies";
});
}
m_currentFrameSourceToggled = FrameSourceToggledType.Capture;
}
catch (Exception ex)
{
await(new MessageDialog(ex.Message)).ShowAsync();
m_currentFrameSourceToggled = FrameSourceToggledType.None;
}
m_lock.Release();
// Enable subsequent trigger of this event callback
UIButtonFilePick.IsEnabled = true;
UICameraToggle.IsEnabled = true;
}
protected override void OnElementChanged(ElementChangedEventArgs <View> e)
{
base.OnElementChanged(e);
if (e.OldElement != null || Element == null)
{
return;
}
if (e.NewElement != null)
{
if (Control == null)
{
var cameraView = ((CameraView)e.NewElement);
liveCameraStream = new UICameraPreview(cameraView.DefaultTorchOn, cameraView.VibrationOnDetected, cameraView.AutoStartScanning);
SetNativeControl(liveCameraStream);
liveCameraStream.OnDetected += (list) =>
{
cameraView?.TriggerOnDetected(list);
};
}
}
}
private async void UICameraToggle_Click(object sender, RoutedEventArgs e)
{
await m_lock.WaitAsync();
try
{
UICameraPreview.Stop();
if (UICameraPreview.CameraHelper != null)
{
await UICameraPreview.CameraHelper.CleanUpAsync();
}
m_isCameraFrameDimensionInitialized = false;
// Initialize skill with the selected supported device
m_skill = await m_skillDescriptor.CreateSkillAsync(m_availableExecutionDevices[UISkillExecutionDevices.SelectedIndex]) as FaceSentimentAnalyzerSkill;
// Instantiate a binding object that will hold the skill's input and output resource
m_binding = await m_skill.CreateSkillBindingAsync() as FaceSentimentAnalyzerBinding;
// Initialize the CameraPreview control, register frame arrived event callback
UIImageViewer.Visibility = Visibility.Collapsed;
UICameraPreview.Visibility = Visibility.Visible;
await UICameraPreview.StartAsync();
UICameraPreview.CameraHelper.FrameArrived += CameraHelper_FrameArrived;
m_currentFrameSourceToggled = FrameSourceToggledType.Camera;
}
catch (Exception ex)
{
await(new MessageDialog(ex.Message)).ShowAsync();
m_currentFrameSourceToggled = FrameSourceToggledType.None;
}
finally
{
m_lock.Release();
}
}
/// <summary>
/// Triggered when UIButtonFilePick is clicked
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UIButtonFilePick_Click(object sender, RoutedEventArgs e)
{
// Disable subsequent trigger of this event callback
UIButtonFilePick.IsEnabled = false;
UICameraToggle.IsEnabled = false;
m_newBackgroundImageAvailable = true;
m_ImageClickedOnce = true;
// Stop Camera preview
UICameraPreview.Stop();
if (UICameraPreview.CameraHelper != null)
{
await UICameraPreview.CameraHelper.CleanUpAsync();
}
UICameraPreview.Visibility = Visibility.Collapsed;
try
{
m_currentFrameSourceToggled = FrameSourceToggledType.ImageFile;
var frame = await LoadVideoFrameFromFilePickedAsync();
// Instantiate skill only if object is null or selected execution device has changed
if ((m_paramsSkillObj.m_selectedDeviceId != UISkillExecutionDevices.SelectedIndex) || (m_paramsSkillObj.m_skill == null))
{
//Release previous instance
if (m_paramsSkillObj.m_skill != null)
{
m_paramsSkillObj.m_skill = null;
}
// Update selected device
m_paramsSkillObj.m_selectedDeviceId = UISkillExecutionDevices.SelectedIndex;
// Initialize skill with the selected supported device
m_paramsSkillObj.m_skill = await m_paramsSkillObj.m_skillDescriptor.CreateSkillAsync(m_paramsSkillObj.m_availableExecutionDevices[m_paramsSkillObj.m_selectedDeviceId]) as BackgroundReplacementSkill;
// Instantiate a binding object that will hold the skill's input and output resource
m_paramsSkillObj.m_binding = await m_paramsSkillObj.m_skill.CreateSkillBindingAsync() as BackgroundReplacementBinding;
}
if (frame != null)
{
// Update input image and run the skill against it
await m_paramsSkillObj.m_binding.SetInputImageAsync(frame);
if (m_newBackgroundImageAvailable)
{
while (m_backgroundImage == null)
{
//wait here until background image is available
}
await m_paramsSkillObj.m_binding.SetBackgroundImageAsync(m_backgroundImage);
m_newBackgroundImageAvailable = false;
}
// Evaluate skill
await m_paramsSkillObj.m_skill.EvaluateAsync(m_paramsSkillObj.m_binding);
VideoFrame replacedOutput = null;
await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, async() =>
{
// Retrieve result and display
replacedOutput = await m_paramsSkillObj.m_binding.GetOutputImageAsync();
if (replacedOutput.SoftwareBitmap == null)
{
SoftwareBitmap softwareBitmapOut = await SoftwareBitmap.CreateCopyFromSurfaceAsync(replacedOutput.Direct3DSurface);
softwareBitmapOut = SoftwareBitmap.Convert(softwareBitmapOut, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
await m_bitmapSource.SetBitmapAsync(softwareBitmapOut);
UIOutputViewer.Source = m_bitmapSource;
}
else
{
await m_bitmapSource.SetBitmapAsync(replacedOutput.SoftwareBitmap);
UIOutputViewer.Source = m_bitmapSource;
}
});
}
}
catch (Exception ex)
{
await(new MessageDialog(ex.Message)).ShowAsync();
}
// Enable subsequent trigger of this event callback
UIButtonFilePick.IsEnabled = true;
UICameraToggle.IsEnabled = true;
}
/// <summary>
/// Triggered when UICameraToggle is clicked, initializes frame grabbing from the camera stream
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UICameraToggle_Click(object sender, RoutedEventArgs e)
{
UICameraToggle.IsEnabled = false;
await m_lock.WaitAsync();
try
{
UICameraPreview.Stop();
if (UICameraPreview.CameraHelper != null)
{
await UICameraPreview.CameraHelper.CleanUpAsync();
}
m_isCameraFrameDimensionInitialized = false;
// Instantiate skill only if object is null or selected execution device has changed
if ((m_paramsSkillObj.m_selectedDeviceId != UISkillExecutionDevices.SelectedIndex) || (m_paramsSkillObj.m_skill == null))
{
if (m_paramsSkillObj.m_skill != null)
{
// Release previous instance
m_paramsSkillObj.m_skill = null;
}
// Update selected device
m_paramsSkillObj.m_selectedDeviceId = UISkillExecutionDevices.SelectedIndex;
// Initialize skill with the selected supported device
m_paramsSkillObj.m_skill = await m_paramsSkillObj.m_skillDescriptor.CreateSkillAsync(m_paramsSkillObj.m_availableExecutionDevices[UISkillExecutionDevices.SelectedIndex]) as SuperResolutionWinMLSkill;
// Instantiate a binding object that will hold the skill's input and output resource
m_paramsSkillObj.m_binding = await m_paramsSkillObj.m_skill.CreateSkillBindingAsync() as SuperResolutionWinMLBinding;
}
// Initialize the CameraPreview control, register frame arrived event callback
UICameraPreview.Visibility = Visibility.Visible;
await UICameraPreview.StartAsync();
UICameraPreview.CameraHelper.FrameArrived += CameraHelper_FrameArrived;
// Set a specific resolution if available. This version of super resolution skill works best on 640x360 input.
var lrFrameFormat = UICameraPreview.CameraHelper.FrameFormatsAvailable.Find((format) => ((format.VideoFormat.Width == 640) && (format.VideoFormat.Height == 360)));
if (lrFrameFormat != null)
{
await UICameraPreview.CameraHelper.PreviewFrameSource.SetFormatAsync(lrFrameFormat);
}
// Set super resolution output on top as default
m_currentFrameSourceToggled = FrameSourceToggledType.Camera;
m_currentOutputType = ScaledOutputType.SuperResolution;
Canvas.SetZIndex(UIOutputViewer, 0);
Canvas.SetZIndex(UICameraPreview, -99);
UIButtonSwitchOutput.Content = "Switch to Linear Scaling";
UIButtonSwitchOutput.IsEnabled = true;
}
catch (Exception ex)
{
await(new MessageDialog(ex.Message)).ShowAsync();
m_currentFrameSourceToggled = FrameSourceToggledType.None;
}
finally
{
m_lock.Release();
}
}
