// Used an explicit tuple here as I'm in C# 6.0
async Task <Tuple <MediaCapture, MediaFrameSource> > GetMediaCaptureForDescriptionAsync(
MediaFrameSourceKind sourceKind,
int width,
int height,
int frameRate,
string[] bitmapFormats = null)
{
MediaCapture mediaCapture = null;
MediaFrameSource frameSource = null;
var allSources = await MediaFrameSourceGroup.FindAllAsync();
// Ignore frame rate here on the description as both depth streams seem to tell me they are
// 30fps whereas I don't think they are (from the docs) so I leave that to query later on.
// NB: LastOrDefault here is a NASTY, NASTY hack - just my way of getting hold of the
// *LAST* depth stream rather than the *FIRST* because I'm assuming that the *LAST*
// one is the longer distance stream rather than the short distance stream.
// I should fix this and find a better way of choosing the right depth stream rather
// than relying on some ordering that's not likely to always work!
var sourceInfo =
allSources.SelectMany(group => group.SourceInfos)
.LastOrDefault(
si =>
(si.MediaStreamType == MediaStreamType.VideoRecord) &&
(si.SourceKind == sourceKind) &&
(si.VideoProfileMediaDescription.Any(
desc =>
desc.Width == width &&
desc.Height == height &&
desc.FrameRate == frameRate)));
if (sourceInfo != null)
{
var sourceGroup = sourceInfo.SourceGroup;
mediaCapture = new MediaCapture();
await mediaCapture.InitializeAsync(
new MediaCaptureInitializationSettings()
{
// I want software bitmaps
MemoryPreference = MediaCaptureMemoryPreference.Cpu,
SourceGroup = sourceGroup,
StreamingCaptureMode = StreamingCaptureMode.Video
}
);
frameSource = mediaCapture.FrameSources[sourceInfo.Id];
var selectedFormat = frameSource.SupportedFormats.First(
format =>
format.VideoFormat.Width == width && format.VideoFormat.Height == height &&
format.FrameRate.Numerator / format.FrameRate.Denominator == frameRate &&
((bitmapFormats == null) || (bitmapFormats.Contains(format.Subtype.ToLower()))));
await frameSource.SetFormatAsync(selectedFormat);
}
return(Tuple.Create(mediaCapture, frameSource));
}
private async Task InitMediaSourceAsync()
{
var allGroups = await MediaFrameSourceGroup.FindAllAsync();
if (allGroups.Count == 0)
{
Debug.LogError("cannot found MediaFrameSourceGroup. アプリケーションマニュフェストを確認してください。");
return;
}
MediaFrameSourceGroup sourceGroup =
allGroups.FirstOrDefault(g => g.SourceInfos.Any(s => s.SourceKind == MediaFrameSourceKind.Depth));
if (sourceGroup == null)
{
Debug.LogError("深度カメラが見つからないようです。");
return;
}
try
{
await InitializeMediaCaptureAsync(sourceGroup);
}
catch (Exception exception)
{
Debug.LogError("InitializeMediaCaptureAsyncに失敗しました" + exception.Message);
await CleanupMediaCaptureAsync();
return;
}
MediaFrameSource source = _mediaCapture.FrameSources.Values
.FirstOrDefault(s => s.Info.SourceKind == MediaFrameSourceKind.Depth);
if (source == null)
{
Debug.LogError("sourceが見つかりません。");
}
MediaFrameFormat format = source.SupportedFormats.FirstOrDefault(f =>
String.Equals(f.Subtype, MediaEncodingSubtypes.D16, StringComparison.OrdinalIgnoreCase));
if (format == null)
{
return;
}
await source.SetFormatAsync(format);
_depthReader = await _mediaCapture.CreateFrameReaderAsync(source, format.Subtype);
MediaFrameReaderStartStatus status = await _depthReader.StartAsync();
if (status != MediaFrameReaderStartStatus.Success)
{
Debug.LogError("_depthReader.StartAsyncに失敗しました");
}
}
/// <summary>
/// Initializes MediaCapture's frame source with a compatible format, if possible.
/// Throws Exception if no compatible stream(s) available
/// </summary>
/// <returns></returns>
private async Task InitializeMediaFrameSourceAsync()
{
if (m_mediaCapture == null)
{
return;
}
// Get preview or record stream as source
Func <KeyValuePair <string, MediaFrameSource>, MediaStreamType, bool> filterFrameSources = (source, type) =>
{
return(source.Value.Info.MediaStreamType == type && source.Value.Info.SourceKind == MediaFrameSourceKind.Color);
};
m_frameSource = m_mediaCapture.FrameSources.FirstOrDefault(source => filterFrameSources(source, MediaStreamType.VideoPreview)).Value
?? m_mediaCapture.FrameSources.FirstOrDefault(source => filterFrameSources(source, MediaStreamType.VideoRecord)).Value;
// if no preview stream is available, bail
if (m_frameSource == null)
{
throw new Exception("No preview or record stream available");
}
// If we can, let's attempt to change the format set on the source to our preferences
if (m_mediaCaptureInitializationSettings.SharingMode == MediaCaptureSharingMode.ExclusiveControl)
{
// Filter MediaType given resolution and framerate preference, and filter out non-compatible subtypes
// Prefer a BGRA8 format and defer to other supported subtypes if none is supported on the source
var selectedFormat = m_frameSource.SupportedFormats.Where(format =>
format.FrameRate.Numerator / format.FrameRate.Denominator > 15 &&
string.Compare(format.Subtype, MediaEncodingSubtypes.Bgra8, true) == 0
)?.OrderBy(format => Math.Abs((int)(format.VideoFormat.Width * format.VideoFormat.Height) - (1920 * 1080))).FirstOrDefault();
if (selectedFormat == null)
{
selectedFormat = m_frameSource.SupportedFormats.Where(format =>
format.FrameRate.Numerator / format.FrameRate.Denominator > 15 &&
(string.Compare(format.Subtype, MediaEncodingSubtypes.Nv12, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Yuy2, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Rgb32, true) == 0)
)?.OrderBy(format => Math.Abs((int)(format.VideoFormat.Width * format.VideoFormat.Height) - (1920 * 1080))).FirstOrDefault();
}
if (selectedFormat == null)
{
throw new Exception("No compatible formats available");
}
await m_frameSource.SetFormatAsync(selectedFormat);
}
FrameWidth = m_frameSource.CurrentFormat.VideoFormat.Width;
FrameHeight = m_frameSource.CurrentFormat.VideoFormat.Height;
}
public async void InitializeManager()
{
_helper = new OpenCVHelper();
// Find the sources
var allGroups = await MediaFrameSourceGroup.FindAllAsync();
var sourceGroups = allGroups.Select(g => new
{
Group = g,
SourceInfo = g.SourceInfos.FirstOrDefault(i => i.SourceKind == MediaFrameSourceKind.Color)
}).Where(g => g.SourceInfo != null).ToList();
if (sourceGroups.Count == 0)
{
// No camera sources found
return;
}
var selectedSource = sourceGroups.FirstOrDefault();
// Initialize MediaCapture
try
{
await InitializeMediaCaptureAsync(selectedSource.Group);
}
catch (Exception exception)
{
Debug.WriteLine("MediaCapture initialization error: " + exception.Message);
await CleanupMediaCaptureAsync();
return;
}
// Create the frame reader
MediaFrameSource frameSource = _mediaCapture.FrameSources[selectedSource.SourceInfo.Id];
var format = frameSource.SupportedFormats.OrderByDescending(x => x.VideoFormat.Width * x.VideoFormat.Height).FirstOrDefault();
await frameSource.SetFormatAsync(format);
BitmapSize size = new BitmapSize() // Choose a lower resolution to make the image processing more performant
{
Height = format.VideoFormat.Height,
Width = format.VideoFormat.Width
};
_reader = await _mediaCapture.CreateFrameReaderAsync(frameSource, MediaEncodingSubtypes.Bgra8, size);
_reader.FrameArrived += HandleFrameArrive;
await _reader.StartAsync();
}
private async Task SetCameraFormat()
{
colorFrameSource = mediaCapture.FrameSources[colorSourceInfo.Id];
var preferredFormat = colorFrameSource.SupportedFormats.Where(format =>
{
return(format.VideoFormat.Width >= 1080 && format.Subtype == MediaEncodingSubtypes.Argb32);
}).FirstOrDefault();
if (preferredFormat == null)
{
return;
}
await colorFrameSource.SetFormatAsync(preferredFormat);
}
/// <summary>
/// Sets the frame format of the current frame source.
/// </summary>
private async Task ChangeMediaFormatAsync(FrameFormatModel format)
{
if (_source == null)
{
_logger.Log("Unable to set format when source is not set.");
return;
}
if (format != null && !format.HasSameFormat(_source.CurrentFormat))
{
await _source.SetFormatAsync(format.Format);
_logger.Log($"Format set to {format.DisplayName}");
}
}
private Task SetFrameType(MediaFrameSource frameSource, int width, int height, int framerate)
{
var preferredFormat = frameSource.SupportedFormats.Where(format =>
{
return(format.VideoFormat.Width == width &&
format.VideoFormat.Height == height &&
(int)Math.Round(((double)format.FrameRate.Numerator / format.FrameRate.Denominator)) == framerate);
});
if (preferredFormat.Count() == 0)
{
throw new ArgumentException(String.Format("No frame type exists for {0}x{1}@{2}", width, height, framerate));
}
return(frameSource.SetFormatAsync(preferredFormat.First()).AsTask());
}
private async Task ChangeMediaFormatAsync2(FrameFormatModel format)
{
if (_source2 == null)
{
_logger2.Log("Unable to set format when source is not set.");
return;
}
// Set the source format if the selected one is different from the current one.
if (format != null && !format.HasSameFormat(_source2.CurrentFormat))
{
await _source2.SetFormatAsync(format.Format);
_logger2.Log($"Format set to {format.DisplayName}");
}
}
/// <summary>
/// Initializes MediaCapture in compatible format, if possible.
/// Throws Exception if no compatible stream(s) available
/// </summary>
/// <returns></returns>
private async Task InitializeMediaFrameSourceAsync()
{
if (m_mediaCapture == null)
{
return;
}
// Get preview or record stream as source
Func <KeyValuePair <string, MediaFrameSource>, MediaStreamType, bool> filterFrameSources = (source, type) =>
{
return(source.Value.Info.MediaStreamType == type && source.Value.Info.SourceKind == MediaFrameSourceKind.Color);
};
m_frameSource = m_mediaCapture.FrameSources.FirstOrDefault(source => filterFrameSources(source, MediaStreamType.VideoPreview)).Value
?? m_mediaCapture.FrameSources.FirstOrDefault(source => filterFrameSources(source, MediaStreamType.VideoRecord)).Value;
// if no preview stream are available, bail
if (m_frameSource == null)
{
throw new Exception("No preview or record streams available");
}
// Filter MediaType given resolution and framerate preference, and filter out non-compatible subtypes
var formats = m_frameSource.SupportedFormats.Where(format =>
format.FrameRate.Numerator / format.FrameRate.Denominator > 15 &&
(string.Compare(format.Subtype, MediaEncodingSubtypes.Nv12, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Bgra8, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Yuy2, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Rgb32, true) == 0)
)?.OrderBy(format => Math.Abs((int)(format.VideoFormat.Width * format.VideoFormat.Height) - (1920 * 1080)));
var selectedFormat = formats.FirstOrDefault();
if (selectedFormat != null)
{
await m_frameSource.SetFormatAsync(selectedFormat);
FrameWidth = m_frameSource.CurrentFormat.VideoFormat.Width;
FrameHeight = m_frameSource.CurrentFormat.VideoFormat.Height;
}
else
{
throw new Exception("No compatible formats available");
}
}
private async Task <bool> InitialiserSourceImages(uint largeur, uint hauteur, string sousTypeVideo)
{
if (_mediaCapture.FrameSources.Count > 0)
{
MediaFrameSource sourceImages = _mediaCapture.FrameSources.First().Value;
ReglerControleurVideo(sourceImages);
MediaFrameFormat formatVideo = sourceImages.SupportedFormats.FirstOrDefault(f => f.VideoFormat.Width == largeur && f.VideoFormat.Height == hauteur && f.Subtype == sousTypeVideo);
if (formatVideo != null)
{
await sourceImages.SetFormatAsync(formatVideo);
_lecteurMedia = await _mediaCapture.CreateFrameReaderAsync(sourceImages);
_lecteurMedia.FrameArrived += _lecteurMedia_FrameArrived;
await _lecteurMedia.StartAsync();
return(true);
}
else
{
Debug.WriteLine("Le format demandé " + largeur.ToString() + "x" + hauteur.ToString() + " en " + sousTypeVideo + " n'est pas supporté par cette caméra !");
Debug.WriteLine("Voici les formats supportés :");
foreach (MediaFrameFormat formatImages in sourceImages.SupportedFormats)
{
Debug.WriteLine(formatImages.MajorType + " " + formatImages.VideoFormat.Width.ToString() + "x" + formatImages.VideoFormat.Height.ToString() + " " + formatImages.Subtype);
}
return(false);
}
}
else
{
Debug.WriteLine("Aucun périphérique de capture détecté !");
return(false);
}
}
public async void StartCapture()
{
System.Diagnostics.Debug.WriteLine("Starting capture.");
var videoDevice = await GetBestVideoDevice();
if (videoDevice == null)
{
System.Diagnostics.Debug.WriteLine("Failed to find video device.");
return;
}
MediaCaptureVideoProfile profile;
MediaCaptureVideoProfileMediaDescription description;
if (!GetBestProfileAndDescription(videoDevice, out profile, out description))
{
System.Diagnostics.Debug.WriteLine("Failed to find profile and description.");
return;
}
var settings = new MediaCaptureInitializationSettings
{
MemoryPreference = MediaCaptureMemoryPreference.Cpu,
VideoDeviceId = videoDevice.Id,
VideoProfile = profile,
RecordMediaDescription = description,
};
var mediaCapture = new MediaCapture();
await mediaCapture.InitializeAsync(settings);
MediaFrameSource source = null;
MediaFrameFormat format = null;
if (!GetBestSourceAndFormat(mediaCapture, out source, out format))
{
System.Diagnostics.Debug.WriteLine("Failed to find source and format.");
return;
}
System.Diagnostics.Debug.WriteLine(string.Format("Selected Video Format: Width: {0}, Height: {1}, Major Type: {2}, Subtype: {3}, Frame Rate: {4}/{5}",
format.VideoFormat.Width,
format.VideoFormat.Height,
format.MajorType,
format.Subtype,
format.FrameRate == null ? "null" : format.FrameRate.Numerator.ToString(),
format.FrameRate == null ? "null" : format.FrameRate.Denominator.ToString()));
await source.SetFormatAsync(format);
mediaFrameReader = await mediaCapture.CreateFrameReaderAsync(source, MediaEncodingSubtypes.Bgra8);
if (await mediaFrameReader.StartAsync() != MediaFrameReaderStartStatus.Success)
{
System.Diagnostics.Debug.WriteLine("Failed to start media frame reader.");
return;
}
mediaFrameReader.FrameArrived += MediaFrameReader_FrameArrived;
System.Diagnostics.Debug.WriteLine("Capture started.");
}
//#############################################################################################
/// <summary>
/// Set the frame format for capture. Needed for allowing camera launch.
/// </summary>
private async Task SetFrameFormat()
{
_frameSource = _mediaCapture.FrameSources[_selectedSourceInfo.Id]; // get the source found in FindSource method
var format = _frameSource.SupportedFormats.FirstOrDefault();
await _frameSource.SetFormatAsync(format);
}
async Task Start()
{
// Socket listener
audioSocketListener = new StreamSocketListener();
audioSocketListener.ConnectionReceived += OnConnectionAudio;
await audioSocketListener.BindServiceNameAsync(audioServiceName);
videoSocketListener = new StreamSocketListener();
videoSocketListener.ConnectionReceived += OnConnectionVideo;
await videoSocketListener.BindServiceNameAsync(videoServiceName);
// Find a media source group which gives us webcam and microphone input streams
var sourceGroups = await MediaFrameSourceGroup.FindAllAsync();
MediaFrameSourceGroup selectedSourceGroup = null;
MediaCaptureVideoProfile selectedVideoProfile = null;
MediaCaptureVideoProfileMediaDescription selectedDescription = null;
foreach (MediaFrameSourceGroup sourceGroup in sourceGroups)
{
var videoProfiles = MediaCapture.FindKnownVideoProfiles(sourceGroup.Id, KnownVideoProfile.VideoConferencing);
foreach (MediaCaptureVideoProfile videoProfile in videoProfiles)
{
foreach (var desc in videoProfile.SupportedRecordMediaDescription)
{
if (desc.Width == videoWidth && desc.Height == videoHeight && desc.FrameRate == frameRate)
{
selectedSourceGroup = sourceGroup;
selectedVideoProfile = videoProfile;
selectedDescription = desc;
}
}
}
}
if (selectedSourceGroup == null)
{
Debug.Log("No source group was found.");
return;
}
mediaCapture = new MediaCapture();
var settings = new MediaCaptureInitializationSettings()
{
AudioProcessing = AudioProcessing.Raw,
MemoryPreference = MediaCaptureMemoryPreference.Cpu,
RecordMediaDescription = selectedDescription,
SharingMode = MediaCaptureSharingMode.ExclusiveControl,
SourceGroup = selectedSourceGroup,
StreamingCaptureMode = StreamingCaptureMode.AudioAndVideo,
VideoProfile = selectedVideoProfile,
};
try
{
await mediaCapture.InitializeAsync(settings);
}
catch (Exception ex)
{
Debug.Log("MediaCapture initialization failed: " + ex.Message);
return;
}
var audioFrameSources = mediaCapture.FrameSources.Where(src => src.Value.Info.MediaStreamType == MediaStreamType.Audio);
if (audioFrameSources.Count() == 0)
{
Debug.Log("No audio source was found.");
return;
}
MediaFrameSource audioFrameSource = audioFrameSources.FirstOrDefault().Value;
var videoFrameSources = mediaCapture.FrameSources.Where(src => src.Value.Info.SourceKind == MediaFrameSourceKind.Color);
if (videoFrameSources.Count() == 0)
{
Debug.Log("No video source was found.");
return;
}
// MediaFrameSource videoFrameSource = videoFrameSources.FirstOrDefault().Value;
MediaFrameSource videoFrameSource = null;
MediaFrameFormat selectedFormat = null;
foreach (var kv in videoFrameSources)
{
MediaFrameSource source = kv.Value;
foreach (MediaFrameFormat format in source.SupportedFormats)
{
if (format.VideoFormat.Width == videoWidth && format.VideoFormat.Height == videoHeight &&
format.FrameRate.Numerator == frameRate && format.FrameRate.Denominator == 1)
{
videoFrameSource = source;
selectedFormat = format;
break;
}
}
if (videoFrameSource != null)
{
break;
}
}
if (selectedFormat != null)
{
await videoFrameSource.SetFormatAsync(selectedFormat);
}
else
{
Debug.Log("Cannot find a proper MediaFrameFormat.");
return;
}
// Start streaming
audioFrameReader = await mediaCapture.CreateFrameReaderAsync(audioFrameSource);
audioFrameReader.FrameArrived += AudioFrameArrived;
videoFrameReader = await mediaCapture.CreateFrameReaderAsync(videoFrameSource);
videoFrameReader.FrameArrived += VideoFrameArrived;
var audioStartStatus = audioFrameReader.StartAsync();
var videoStartStatus = videoFrameReader.StartAsync();
if (await audioStartStatus != MediaFrameReaderStartStatus.Success)
{
Debug.Log("The audioFrameReader couldn't start.");
}
if (await videoStartStatus != MediaFrameReaderStartStatus.Success)
{
Debug.Log("The videoFrameReader couldn't start.");
}
}
private async Task <bool> InitializeMediaCaptureAsync()
{
if (captureStatus != CaptureStatus.Clean)
{
Debug.Log(": InitializeMediaCaptureAsync() fails because of incorrect status");
return(false);
}
if (mediaCapture != null)
{
return(false);
}
var allGroups = await MediaFrameSourceGroup.FindAllAsync();
foreach (var group in allGroups)
{
Debug.Log(group.DisplayName + ", " + group.Id);
}
if (allGroups.Count <= 0)
{
Debug.Log(": InitializeMediaCaptureAsync() fails because there is no MediaFrameSourceGroup");
return(false);
}
// Initialize mediacapture with the source group.
mediaCapture = new MediaCapture();
var settings = new MediaCaptureInitializationSettings {
SourceGroup = allGroups[group_id],
// This media capture can share streaming with other apps.
SharingMode = MediaCaptureSharingMode.SharedReadOnly,
// Only stream video and don't initialize audio capture devices.
StreamingCaptureMode = StreamingCaptureMode.Video,
// Set to CPU to ensure frames always contain CPU SoftwareBitmap images
// instead of preferring GPU D3DSurface images.
MemoryPreference = MediaCaptureMemoryPreference.Cpu
};
await mediaCapture.InitializeAsync(settings);
Debug.Log(": MediaCapture is successfully initialized in shared mode.");
MediaFrameSource targetFrameSource = mediaCapture.FrameSources.Values.ElementAt(id);
MediaFrameFormat targetResFormat = targetFrameSource.SupportedFormats[0];
try {
await targetFrameSource.SetFormatAsync(targetResFormat);
frameReader = await mediaCapture.CreateFrameReaderAsync(targetFrameSource, targetResFormat.Subtype);
frameReader.FrameArrived += OnFrameArrived;
videoWidth = Convert.ToInt32(targetResFormat.VideoFormat.Width);
videoHeight = Convert.ToInt32(targetResFormat.VideoFormat.Height);
Debug.Log(": FrameReader is successfully initialized, " + videoWidth + "x" + videoHeight +
", Framerate: " + targetResFormat.FrameRate.Numerator + "/" + targetResFormat.FrameRate.Denominator +
", Major type: " + targetResFormat.MajorType + ", Subtype: " + targetResFormat.Subtype);
}
catch (Exception e) {
Debug.Log(": FrameReader is not initialized");
Debug.Log(": Exception: " + e);
return(false);
}
captureStatus = CaptureStatus.Initialized;
return(true);
}
/// <summary>
/// Start the video stream. This just prepares the stream for capture, and doesn't start collecting frames
/// </summary>
/// <param name="streamDesc">The description of the stream to start.</param>
public async void Start(StreamDescription streamDesc)
{
#if CAN_USE_UWP_TYPES
lock (stateLock)
{
if (State != CameraState.Initialized)
{
throw new InvalidOperationException("Start cannot be called until the camera is in the Initialized state");
}
State = CameraState.Starting;
}
Resolution = streamDesc.Resolution;
CameraType = streamDesc.CameraType;
StreamDescriptionInternal desc = streamDesc as StreamDescriptionInternal;
MediaCaptureInitializationSettings initSettings = new MediaCaptureInitializationSettings()
{
SourceGroup = desc.FrameSourceGroup,
SharingMode = MediaCaptureSharingMode.ExclusiveControl,
MemoryPreference = MediaCaptureMemoryPreference.Cpu,
StreamingCaptureMode = StreamingCaptureMode.Video
};
// initialize the media device
mediaCapture = new MediaCapture();
try
{
await mediaCapture.InitializeAsync(initSettings);
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine($"MediaCapture initialization failed: {ex.Message}");
mediaCapture.Dispose();
mediaCapture = null;
}
if (mediaCapture != null)
{
// get access to the video device controller for property settings
videoDeviceController = mediaCapture.VideoDeviceController;
// choose media source
MediaFrameSource frameSource = mediaCapture.FrameSources[desc.FrameSourceInfo.Id];
MediaFrameFormat preferredFormat = null;
foreach (MediaFrameFormat format in frameSource.SupportedFormats)
{
if (format.VideoFormat.Width == desc.Resolution.Width && format.VideoFormat.Height == desc.Resolution.Height && Math.Abs((double)format.FrameRate.Numerator / (double)format.FrameRate.Denominator - desc.Resolution.Framerate) < epsilon)
{
preferredFormat = format;
break;
}
}
if (preferredFormat != null && preferredFormat != frameSource.CurrentFormat)
{
await frameSource.SetFormatAsync(preferredFormat);
}
else
{
System.Diagnostics.Debug.WriteLine($"failed to set desired frame format");
}
// set up frame readercapture frame data
frameReader = await mediaCapture.CreateFrameReaderAsync(frameSource);
frameReader.FrameArrived += OnMediaFrameArrived;
await frameReader.StartAsync();
lock (stateLock)
{
State = CameraState.Ready;
OnCameraStarted?.Invoke(this, true);
}
}
else
{
lock (stateLock)
{
// drop back to initialized when the camera doesn't initialize
State = CameraState.Initialized;
OnCameraStarted?.Invoke(this, false);
}
}
#else
await Task.CompletedTask;
#endif
}
private async void init()
{
var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync();
LogError("checkpoint 1.1");
var targetGroups = frameSourceGroups.Select(g => new
{
Group = g,
SourceInfos = new MediaFrameSourceInfo[]
{
g.SourceInfos.FirstOrDefault(info => info.SourceKind == MediaFrameSourceKind.Color),
g.SourceInfos.FirstOrDefault(info => info.SourceKind == MediaFrameSourceKind.Depth),
}
}).Where(g => g.SourceInfos.Any(info => info != null)).ToList();
LogError("checkpoint 1.2");
if (targetGroups.Count == 0)
{
LogError("No source groups found.");
return;
}
MediaFrameSourceGroup mediaSourceGroup = targetGroups[0].Group;
LogError("checkpoint 1.3");
mediaCapture = new MediaCapture();
LogError("checkpoint 1.4");
var settings = new MediaCaptureInitializationSettings()
{
SourceGroup = mediaSourceGroup,
SharingMode = MediaCaptureSharingMode.ExclusiveControl,
MemoryPreference = MediaCaptureMemoryPreference.Cpu,
StreamingCaptureMode = StreamingCaptureMode.Video
};
LogError("checkpoint 1.5");
await mediaCapture.InitializeAsync(settings);
LogError("checkpoint 1.6");
MediaFrameSource colorSource =
mediaCapture.FrameSources.Values.FirstOrDefault(
s => s.Info.SourceKind == MediaFrameSourceKind.Color);
MediaFrameSource depthSource =
mediaCapture.FrameSources.Values.FirstOrDefault(
s => s.Info.SourceKind == MediaFrameSourceKind.Depth);
LogError("checkpoint 1.7");
if (colorSource == null || depthSource == null)
{
LogError("Cannot find color or depth stream.");
return;
}
MediaFrameFormat colorFormat = colorSource.SupportedFormats.Where(format =>
{
return(format.VideoFormat.Width >= 640 &&
format.Subtype == MediaEncodingSubtypes.Rgb24);
}).FirstOrDefault();
MediaFrameFormat depthFormat = depthSource.SupportedFormats.Where(format =>
{
return(format.VideoFormat.Width >= 640 &&
format.Subtype == MediaEncodingSubtypes.D16);
}).FirstOrDefault();
await colorSource.SetFormatAsync(colorFormat);
await depthSource.SetFormatAsync(depthFormat);
_colorSourceId = colorSource.Info.Id;
_depthSourceId = depthSource.Info.Id;
_frameReader = await mediaCapture.CreateMultiSourceFrameReaderAsync(
new[] { colorSource, depthSource });
_frameReader.FrameArrived += FrameReader_FrameArrived;
MultiSourceMediaFrameReaderStartStatus startStatus = await _frameReader.StartAsync();
if (startStatus != MultiSourceMediaFrameReaderStartStatus.Success)
{
throw new InvalidOperationException("Unable to start reader: " + startStatus);
}
this.CorrelationFailed += MainPage_CorrelationFailed;
Task.Run(() => NotifyAboutCorrelationFailure(_tokenSource.Token));
}
/// <summary>
///
/// </summary>
/// <returns></returns>
private async Task <bool> InitializeMediaCaptureAsync()
{
if (captureStatus != CaptureStatus.Clean)
{
Debug.Log(TAG + " " + id + ": InitializeMediaCaptureAsync() fails because of incorrect status");
return(false);
}
if (mediaCapture != null)
{
return(false);
}
var allGroups = await MediaFrameSourceGroup.FindAllAsync();
foreach (var group in allGroups)
{
Debug.Log(group.DisplayName + ", " + group.Id);
}
if (allGroups.Count <= 0)
{
Debug.Log(TAG + " " + id + ": InitializeMediaCaptureAsync() fails because there is no MediaFrameSourceGroup");
return(false);
}
// Initialize mediacapture with the source group.
mediaCapture = new MediaCapture();
var settings = new MediaCaptureInitializationSettings {
SourceGroup = allGroups[0],
// This media capture can share streaming with other apps.
SharingMode = MediaCaptureSharingMode.SharedReadOnly,
// Only stream video and don't initialize audio capture devices.
StreamingCaptureMode = StreamingCaptureMode.Video,
// Set to CPU to ensure frames always contain CPU SoftwareBitmap images
// instead of preferring GPU D3DSurface images.
MemoryPreference = MediaCaptureMemoryPreference.Cpu
};
await mediaCapture.InitializeAsync(settings);
Debug.Log(TAG + " " + id + ": MediaCapture is successfully initialized in shared mode.");
// logging all frame source information
string logString = "";
foreach (var frameSource in mediaCapture.FrameSources)
{
var info = frameSource.Value.Info;
logString += info.Id + ", " + info.MediaStreamType + ", " + info.SourceKind + "\n";
logString += "Total number of SupportedFormats is " + frameSource.Value.SupportedFormats.Count + "\n";
foreach (var format in frameSource.Value.SupportedFormats)
{
logString += format.VideoFormat.Width + " x " + format.VideoFormat.Height + ", Major type: " + format.MajorType + ", Subtype: " + format.Subtype +
", Framerate: " + format.FrameRate.Numerator + "/" + format.FrameRate.Denominator + "\n";
}
}
Debug.Log(logString);
MediaFrameSource targetFrameSource = mediaCapture.FrameSources.Values.ElementAt(id);
MediaFrameFormat targetResFormat = targetFrameSource.SupportedFormats[0];
try {
// choose the smallest resolution
//var targetResFormat = mediaFrameSourceVideoPreview.SupportedFormats.OrderBy(x => x.VideoFormat.Width * x.VideoFormat.Height).FirstOrDefault();
// choose the specific resolution
//var targetResFormat = mediaFrameSourceVideoPreview.SupportedFormats.OrderBy(x => (x.VideoFormat.Width == 1344 && x.VideoFormat.Height == 756)).FirstOrDefault();
await targetFrameSource.SetFormatAsync(targetResFormat);
frameReader = await mediaCapture.CreateFrameReaderAsync(targetFrameSource, targetResFormat.Subtype);
frameReader.FrameArrived += OnFrameArrived;
videoWidth = Convert.ToInt32(targetResFormat.VideoFormat.Width);
videoHeight = Convert.ToInt32(targetResFormat.VideoFormat.Height);
Debug.Log(TAG + " " + id + ": FrameReader is successfully initialized, " + videoWidth + "x" + videoHeight +
", Framerate: " + targetResFormat.FrameRate.Numerator + "/" + targetResFormat.FrameRate.Denominator +
", Major type: " + targetResFormat.MajorType + ", Subtype: " + targetResFormat.Subtype);
}
catch (Exception e) {
Debug.Log(TAG + " " + id + ": FrameReader is not initialized");
Debug.Log(TAG + " " + id + ": Exception: " + e);
return(false);
}
captureStatus = CaptureStatus.Initialized;
return(true);
}
/// <summary>
/// Initializes MediaCapture's frame source with a compatible format, if possible.
/// Throws Exception if no compatible stream(s) available
/// </summary>
/// <returns></returns>
private async Task InitializeMediaFrameSourceAsync()
{
if (m_mediaCapture == null)
{
return;
}
// Get preview or record stream as source
Func <KeyValuePair <string, MediaFrameSource>, MediaStreamType, bool> filterFrameSources = (source, type) =>
{
return(source.Value.Info.MediaStreamType == type && source.Value.Info.SourceKind == MediaFrameSourceKind.Color);
};
m_frameSource = m_mediaCapture.FrameSources.FirstOrDefault(source => filterFrameSources(source, MediaStreamType.VideoPreview)).Value
?? m_mediaCapture.FrameSources.FirstOrDefault(source => filterFrameSources(source, MediaStreamType.VideoRecord)).Value;
// if no preview stream is available, bail
if (m_frameSource == null)
{
throw new Exception("No preview or record stream available");
}
// Get preferred camera frame format described in a ISkillFeatureImageDescriptor if specified
int preferredFrameWidth = 1920;
if (m_desiredImageDescriptor != null && m_desiredImageDescriptor.Width != -1)
{
preferredFrameWidth = Math.Abs(m_desiredImageDescriptor.Width);
}
int preferredFrameHeight = 1080;
if (m_desiredImageDescriptor != null && m_desiredImageDescriptor.Height != -1)
{
preferredFrameHeight = Math.Abs(m_desiredImageDescriptor.Height);
}
string preferredMediaEncodingSubtype = MediaEncodingSubtypes.Bgra8;
if (m_desiredImageDescriptor != null)
{
preferredMediaEncodingSubtype = BitmapPixelFormatToMediaEncodingSubtype(m_desiredImageDescriptor.SupportedBitmapPixelFormat);
}
// If we can, let's attempt to change the format set on the source to our preferences
if (m_mediaCaptureInitializationSettings.SharingMode == MediaCaptureSharingMode.ExclusiveControl)
{
// Filter camera MediaType given frame format preference, and filter out non-compatible subtypes
var selectedFormat = m_frameSource.SupportedFormats.Where(format =>
format.FrameRate.Numerator / format.FrameRate.Denominator > 15 &&
string.Compare(format.Subtype, preferredMediaEncodingSubtype, true) == 0
)?.OrderBy(format => Math.Abs((int)(format.VideoFormat.Width * format.VideoFormat.Height) - (preferredFrameWidth * preferredFrameHeight))).FirstOrDefault();
// Defer to other supported subtypes if the one prescribed is not supported on the source
if (selectedFormat == null)
{
selectedFormat = m_frameSource.SupportedFormats.Where(format =>
format.FrameRate.Numerator / format.FrameRate.Denominator > 15 &&
(string.Compare(format.Subtype, MediaEncodingSubtypes.Bgra8, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Nv12, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Yuy2, true) == 0 ||
string.Compare(format.Subtype, MediaEncodingSubtypes.Rgb32, true) == 0)
)?.OrderBy(format => Math.Abs((int)(format.VideoFormat.Width * format.VideoFormat.Height) - (preferredFrameWidth * preferredFrameHeight))).FirstOrDefault();
}
if (selectedFormat == null)
{
throw new Exception("No compatible formats available");
}
await m_frameSource.SetFormatAsync(selectedFormat);
}
FrameWidth = m_frameSource.CurrentFormat.VideoFormat.Width;
FrameHeight = m_frameSource.CurrentFormat.VideoFormat.Height;
}