/// <summary>
/// Static factory method
/// </summary>
/// <param name="videoDeviceInformation"></param>
/// <param name="imageDescriptor"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public static async Task <FrameReaderFrameSource> CreateFromVideoDeviceInformationAsync(
DeviceInformation videoDeviceInformation,
ISkillFeatureImageDescriptor imageDescriptor,
EventHandler <string> failureHandler)
{
// Create new MediaCapture connected to our device
var result = new FrameReaderFrameSource()
{
m_mediaCapture = new MediaCapture(),
m_desiredImageDescriptor = imageDescriptor,
m_failureHandler = failureHandler,
m_mediaCaptureInitializationSettings = new MediaCaptureInitializationSettings
{
SharingMode = MediaCaptureSharingMode.ExclusiveControl,
MemoryPreference = MediaCaptureMemoryPreference.Auto,
StreamingCaptureMode = StreamingCaptureMode.Video,
VideoDeviceId = videoDeviceInformation.Id,
}
};
result.m_mediaCapture.Failed += result.MediaCapture_Failed;
await result.IntializeFrameSourceAsync();
return(result);
}
/// <summary>
/// Static factory
/// </summary>
/// <returns></returns>
public static async Task <MediaPlayerFrameSource> CreateFromStorageFileAsyncTask(
StorageFile storageFile,
ISkillFeatureImageDescriptor imageDescriptor,
EventHandler <string> failureHandler)
{
var result = new MediaPlayerFrameSource()
{
m_desiredImageDescriptor = imageDescriptor,
m_mediaPlayer = new MediaPlayer()
{
Source = MediaSource.CreateFromStorageFile(storageFile),
IsVideoFrameServerEnabled = true,
RealTimePlayback = true,
IsMuted = true,
IsLoopingEnabled = true
}
};
result.m_mediaPlayer.CommandManager.IsEnabled = false;
result.m_mediaPlayer.MediaOpened += result.MediaPlayer_MediaOpened;
result.m_mediaPlayer.MediaEnded += result.MediaPlayer_MediaEnded;
result.m_mediaPlayer.MediaFailed += result.MediaPlayer_MediaFailed;
result.m_failureHandler = failureHandler;
await Task.Run(() => result.m_frameSourceReadyEvent.WaitOne());
return(result);
}
/// <summary>
/// Static factory
/// </summary>
/// <param name="storageFile"></param>
/// <param name="imageDescriptor"></param>
/// <returns></returns>
public static async Task <ImageFileFrameSource> CreateFromStorageFileAsyncTask(
StorageFile storageFile,
ISkillFeatureImageDescriptor imageDescriptor)
{
var result = new ImageFileFrameSource()
{
m_desiredImageDescriptor = imageDescriptor
};
await result.GetFrameFromFileAsync(storageFile);
return(result);
}
/// <summary>
/// Get a list of information strings extracted from the ISkillFeatureDescriptor
/// </summary>
/// <param name="desc"></param>
/// <returns></returns>
public static List <KeyValuePair <string, string> > GetSkillFeatureDescriptorStrings(ISkillFeatureDescriptor desc)
{
if (desc == null)
{
return(new List <KeyValuePair <string, string> >());
}
List <KeyValuePair <string, string> > result = new List <KeyValuePair <string, string> >()
{
new KeyValuePair <string, string>("Name", $"{desc.Name}"),
new KeyValuePair <string, string>("Description", $"{desc.Description}"),
new KeyValuePair <string, string>("IsRequired", $"{desc.IsRequired}"),
new KeyValuePair <string, string>("Type", $"{desc.FeatureKind}"),
};
if (desc is ISkillFeatureImageDescriptor)
{
ISkillFeatureImageDescriptor imageDesc = desc as ISkillFeatureImageDescriptor;
result.Add(new KeyValuePair <string, string>("Width", $"{(imageDesc.Width == -1 ? "Free Dimension" : imageDesc.Width.ToString())}"));
result.Add(new KeyValuePair <string, string>("Height", $"{(imageDesc.Height == -1 ? "Free Dimension" : imageDesc.Height.ToString())}"));
result.Add(new KeyValuePair <string, string>("SupportedBitmapPixelFormat", $"{imageDesc.SupportedBitmapPixelFormat}"));
result.Add(new KeyValuePair <string, string>("SupportedBitmapAlphaMode", $"{imageDesc.SupportedBitmapAlphaMode}"));
}
else if (desc is ISkillFeatureTensorDescriptor)
{
ISkillFeatureTensorDescriptor tensorDesc = desc as ISkillFeatureTensorDescriptor;
result.Add(new KeyValuePair <string, string>("ElementKind", $"{tensorDesc.ElementKind}"));
string shape = "[";
for (int i = 0; i < tensorDesc.Shape.Count; i++)
{
shape += $"{ (tensorDesc.Shape[i] == -1 ? "Free Dimension" : tensorDesc.Shape[i].ToString())}";
if (i < tensorDesc.Shape.Count - 1)
{
shape += ", ";
}
}
shape += "]";
result.Add(new KeyValuePair <string, string>("Shape", shape));
}
else if (desc is ISkillFeatureMapDescriptor)
{
ISkillFeatureMapDescriptor mapDesc = desc as ISkillFeatureMapDescriptor;
result.Add(new KeyValuePair <string, string>("KeyElementKind", $"{mapDesc.KeyElementKind}"));
result.Add(new KeyValuePair <string, string>("ValueElementKind", $"{mapDesc.ValueElementKind}"));
string validKeys = "";
foreach (var validKey in mapDesc.ValidKeys)
{
validKeys += $"{validKey}\n";
}
result.Add(new KeyValuePair <string, string>("ValidKeys", validKeys));
}
return(result);
}
/// <summary>
/// Initialize the ObjectDetector skill
/// </summary>
/// <param name="device"></param>
/// <returns></returns>
private async Task InitializeObjectDetectorAsync(ISkillExecutionDevice device = null)
{
if (device != null)
{
m_skill = await m_descriptor.CreateSkillAsync(device) as ObjectDetectorSkill;
}
else
{
m_skill = await m_descriptor.CreateSkillAsync() as ObjectDetectorSkill;
}
m_binding = await m_skill.CreateSkillBindingAsync() as ObjectDetectorBinding;
m_inputImageFeatureDescriptor = m_binding["InputImage"].Descriptor as SkillFeatureImageDescriptor;
}
/// <summary>
/// Construct a string from the ISkillFeatureDescriptor specified that can be used to display its content
/// </summary>
/// <param name="desc"></param>
/// <returns></returns>
public static string GetSkillFeatureDescriptorString(ISkillFeatureDescriptor desc)
{
if (desc == null)
{
return("");
}
string result = $"Name: {desc.Name}" +
$"\nDescription: {desc.Description}" +
$"\nIsRequired: {desc.IsRequired}" +
$"\nType: {desc.FeatureKind}";
if (desc is ISkillFeatureImageDescriptor)
{
ISkillFeatureImageDescriptor imageDesc = desc as ISkillFeatureImageDescriptor;
result += $"\nWidth: {imageDesc.Width}" +
$"\nHeight: {imageDesc.Height}" +
$"\nSupportedBitmapPixelFormat: {imageDesc.SupportedBitmapPixelFormat}" +
$"\nSupportedBitmapAlphaMode: {imageDesc.SupportedBitmapAlphaMode}";
}
else if (desc is ISkillFeatureTensorDescriptor)
{
ISkillFeatureTensorDescriptor tensorDesc = desc as ISkillFeatureTensorDescriptor;
result += $"\nElementKind: {tensorDesc.ElementKind}" +
"\nShape: [";
for (int i = 0; i < tensorDesc.Shape.Count; i++)
{
result += $"{tensorDesc.Shape[i]}";
if (i < tensorDesc.Shape.Count - 1)
{
result += ", ";
}
}
result += "]";
}
else if (desc is ISkillFeatureMapDescriptor)
{
ISkillFeatureMapDescriptor mapDesc = desc as ISkillFeatureMapDescriptor;
result += $"\nKeyElementKind: {mapDesc.KeyElementKind}" +
$"\nValueElementKind: {mapDesc.ValueElementKind}" +
$"\nValidKeys:";
foreach (var validKey in mapDesc.ValidKeys)
{
result += $"\n\t{validKey}";
}
}
return(result);
}
/// <summary>
/// Configure an IFrameSource from a StorageFile or MediaCapture instance to produce optionally a specified format of frame
/// </summary>
/// <param name="source"></param>
/// <param name="inputImageDescriptor"></param>
/// <returns></returns>
private async Task ConfigureFrameSourceAsync(object source, ISkillFeatureImageDescriptor inputImageDescriptor = null)
{
await m_lock.WaitAsync();
{
// Reset bitmap rendering component
UIProcessedPreview.Source = null;
m_renderTargetFrame = null;
m_processedBitmapSource = new SoftwareBitmapSource();
UIProcessedPreview.Source = m_processedBitmapSource;
// Clean up previous frame source
if (m_frameSource != null)
{
m_frameSource.FrameArrived -= FrameSource_FrameAvailable;
var disposableFrameSource = m_frameSource as IDisposable;
if (disposableFrameSource != null)
{
// Lock disposal based on frame source consumers
disposableFrameSource.Dispose();
}
}
// Create new frame source and register a callback if the source fails along the way
m_frameSource = await FrameSourceFactory.CreateFrameSourceAsync(
source,
(sender, message) =>
{
NotifyUser(message);
},
inputImageDescriptor);
// TODO: Workaround for a bug in ObjectDetectorBinding when binding consecutively VideoFrames with Direct3DSurface and SoftwareBitmap
await m_skillWrappers[0].InitializeSkillAsync(m_skillWrappers[0].Skill.Device);
// Set additional input features as exposed in the UI
await m_skillWrappers[0].Binding["InputObjectKindFilterList"].SetFeatureValueAsync(m_objectKindFilterList);
await m_skillWrappers[0].Binding["InputConfidenceThreshold"].SetFeatureValueAsync((float)UIConfidenceThresholdControl.Value);
}
m_lock.Release();
// If we obtained a valid frame source, start it
if (m_frameSource != null)
{
m_frameSource.FrameArrived += FrameSource_FrameAvailable;
await m_frameSource.StartAsync();
}
}
/// <summary>
/// Configure an IFrameSource from a StorageFile or MediaCapture instance to produce optionally a specified format of frame
/// </summary>
/// <param name="source"></param>
/// <param name="inputImageDescriptor"></param>
/// <returns></returns>
private async Task ConfigureFrameSourceAsync(object source, ISkillFeatureImageDescriptor inputImageDescriptor = null)
{
await m_lock.WaitAsync();
{
// Reset bitmap rendering component
UIProcessedPreview.Source = null;
m_renderTargetFrame = null;
m_processedBitmapSource = new SoftwareBitmapSource();
UIProcessedPreview.Source = m_processedBitmapSource;
// Clean up previous frame source
if (m_frameSource != null)
{
m_frameSource.FrameArrived -= FrameSource_FrameAvailable;
var disposableFrameSource = m_frameSource as IDisposable;
if (disposableFrameSource != null)
{
// Lock disposal based on frame source consumers
disposableFrameSource.Dispose();
}
}
// Create new frame source and register a callback if the source fails along the way
m_frameSource = await FrameSourceFactory.CreateFrameSourceAsync(
source,
(sender, message) =>
{
NotifyUser(message);
},
inputImageDescriptor);
// Clear existing trackers
m_frameCounter = 0;
m_trackerBindings.Clear();
m_trackerHistories.Clear();
}
m_lock.Release();
// If we obtained a valid frame source, start it
if (m_frameSource != null)
{
m_frameSource.FrameArrived += FrameSource_FrameAvailable;
await m_frameSource.StartAsync();
}
}
/// <summary>
/// Configure an IFrameSource from a StorageFile or MediaCapture instance to produce optionally a specified format of frame
/// </summary>
/// <param name="source"></param>
/// <param name="inputImageDescriptor"></param>
/// <returns></returns>
private async Task ConfigureFrameSourceAsync(object source, ISkillFeatureImageDescriptor inputImageDescriptor = null)
{
await m_lock.WaitAsync();
{
// Reset bitmap rendering component
UIProcessedPreview.Source = null;
m_renderTargetFrame = null;
m_processedBitmapSource = new SoftwareBitmapSource();
UIProcessedPreview.Source = m_processedBitmapSource;
// Clean up previous frame source
if (m_frameSource != null)
{
m_frameSource.FrameArrived -= FrameSource_FrameAvailable;
var disposableFrameSource = m_frameSource as IDisposable;
if (disposableFrameSource != null)
{
// Lock disposal based on frame source consumers
disposableFrameSource.Dispose();
}
}
// Create new frame source and register a callback if the source fails along the way
m_frameSource = await FrameSourceFactory.CreateFrameSourceAsync(
source,
(sender, message) =>
{
NotifyUser(message);
},
inputImageDescriptor);
// TODO: Workaround for a bug in ObjectDetectorBinding when binding consecutively VideoFrames with Direct3DSurface and SoftwareBitmap
m_binding = await m_skill.CreateSkillBindingAsync() as ObjectDetectorBinding;
}
m_lock.Release();
// If we obtained a valid frame source, start it
if (m_frameSource != null)
{
m_frameSource.FrameArrived += FrameSource_FrameAvailable;
await m_frameSource.StartAsync();
}
}
/// <summary>
/// Initialize the ObjectDetector skill
/// </summary>
/// <param name="device"></param>
/// <returns></returns>
private async Task InitializeObjectDetectorAsync(ISkillExecutionDevice device = null)
{
await m_skillWrappers[0].InitializeSkillAsync(device);
m_inputImageFeatureDescriptor = m_skillWrappers[0].Binding["InputImage"].Descriptor as SkillFeatureImageDescriptor;
}
/// <summary>
/// Create an IFrameSource from a source object. Currently supports Windows.Storage.StorageFile
/// and Windows.Media.Capture.MediaCapture source objects.
/// </summary>
/// <param name="source"></param>
/// <param name="failureHandler"></param>
/// <param name="imageDescriptor"></param>
/// <returns></returns>
public static async Task <IFrameSource> CreateFrameSourceAsync(object source, EventHandler <string> failureHandler, ISkillFeatureImageDescriptor imageDescriptor = null)
{
try
{
if (source is Windows.Storage.StorageFile)
{
var sourceFile = source as Windows.Storage.StorageFile;
if (sourceFile.ContentType.StartsWith("image"))
{
return(await ImageFileFrameSource.CreateFromStorageFileAsyncTask(sourceFile, imageDescriptor));
}
else if (sourceFile.ContentType.StartsWith("video"))
{
return(await MediaPlayerFrameSource.CreateFromStorageFileAsyncTask(sourceFile, imageDescriptor, failureHandler));
}
else
{
throw new ArgumentException("Invalid file type received: " + sourceFile.ContentType);
}
}
else if (source is Windows.Devices.Enumeration.DeviceInformation)
{
return(await FrameReaderFrameSource.CreateFromVideoDeviceInformationAsync(source as Windows.Devices.Enumeration.DeviceInformation, imageDescriptor, failureHandler));
}
else
{
throw new ArgumentException();
}
}
catch (Exception ex)
{
failureHandler(null, ex.Message);
}
return(null);
}
