/// <summary> /// Bind and evaluate the frame with the ObjectDetector skill /// </summary> /// <param name="frame"></param> /// <returns></returns> private async Task RunSkillsAsync(VideoFrame frame) { m_bindTime = 0F; m_evalTime = 0F; m_frameCounter++; // Update all trackers m_evalStopwatch.Restart(); var bindTasks = new List <Task>(); foreach (var binding in m_trackerBindings) { bindTasks.Add(binding.SetInputImageAsync(frame).AsTask()); } await Task.WhenAll(bindTasks); m_evalStopwatch.Stop(); m_bindTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f; m_evalStopwatch.Restart(); var evalTasks = new List <Task>(); foreach (var binding in m_trackerBindings) { evalTasks.Add(m_trackerSkill.EvaluateAsync(binding).AsTask()); } await Task.WhenAll(evalTasks); m_evalStopwatch.Stop(); m_evalTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f; // Add results to histories for (int i = 0; i < m_trackerBindings.Count; i++) { m_trackerHistories[i].Enqueue( new TrackerResult() { label = m_trackerHistories[i].Peek().label, boundingRect = m_trackerBindings[i].BoundingRect, succeeded = m_trackerBindings[i].Succeeded } ); if (m_trackerHistories[i].Count > m_maxTrackerHistoryLength) { m_trackerHistories[i].Dequeue(); } } // Run detector if no successful trackers or we've reached our desired detector period if (m_trackerBindings.Where(binding => binding.Succeeded).Count() == 0 || (m_frameCounter % m_detectorEvalInterval) == 0) { // Bind m_evalStopwatch.Restart(); await m_detectorBinding.SetInputImageAsync(frame); m_evalStopwatch.Stop(); m_bindTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f; // Evaluate m_evalStopwatch.Restart(); await m_detectorSkill.EvaluateAsync(m_detectorBinding); m_evalStopwatch.Stop(); m_evalTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f; // Clear and re-initialize trackers m_trackerBindings.Clear(); m_trackerHistories.Clear(); IEnumerable <ObjectDetectorResult> filteredDetections = m_detectorBinding.DetectedObjects.Where(det => ((m_objectKinds?.Count ?? 0) == 0) || m_objectKinds.Contains(det.Kind)); var initializeTasks = new List <Task>(); m_evalStopwatch.Restart(); // Since we're initializing trackers in parallel, it's easiest to count it all as evaluation // (including tracker binding) Dictionary <string, int> objectKindCounter = new Dictionary <string, int>(); foreach (ObjectDetectorResult detection in filteredDetections) { // Cap at max trackers if (m_trackerBindings.Count >= m_maxNumberTrackers) { break; } // Do some sanity checks to make sure the detection is valid if (detection.Rect.Width <= 0 || detection.Rect.Height <= 0) { break; } // Create and initialize new tracker ObjectTrackerBinding binding = await m_trackerSkill.CreateSkillBindingAsync() as ObjectTrackerBinding; Rect clampedRect = new Rect( Math.Max(detection.Rect.X, 0.0), Math.Max(detection.Rect.Y, 0.0), detection.Rect.Width, detection.Rect.Height); initializeTasks.Add(m_trackerSkill.InitializeTrackerAsync(binding, frame, clampedRect).AsTask()); m_trackerBindings.Add(binding); // Add corresponding tracker history string objectKindStr = detection.Kind.ToString(); if (!objectKindCounter.ContainsKey(objectKindStr)) { objectKindCounter.Add(objectKindStr, 0); } string label = $"{objectKindStr} {++objectKindCounter[objectKindStr]}"; m_trackerHistories.Add(new Queue <TrackerResult>()); m_trackerHistories.Last().Enqueue( new TrackerResult() { label = label, boundingRect = clampedRect, succeeded = true } ); } await Task.WhenAll(initializeTasks); m_evalStopwatch.Stop(); m_evalTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f; m_frameCounter = 0; // Reset frame counter } }
/// <summary> /// Update existing trackers with the latest frame and initialize any new trackers based on user input /// This should always be called from inside the lock (m_lock) /// </summary> /// <param name="frame"></param> /// <returns></returns> private async Task RunSkillAsync(VideoFrame frame) { // Update existing trackers if (m_bindings.Count > 0) { long evalTicks = 0L; for (int i = 0; i < m_bindings.Count; i++) { await m_bindings[i].SetInputImageAsync(frame); m_skillEvalStopWatch.Restart(); await m_skill.EvaluateAsync(m_bindings[i]); m_skillEvalStopWatch.Stop(); evalTicks += m_skillEvalStopWatch.ElapsedTicks; // Add result to history m_trackerHistory[i].Enqueue( new TrackerResult() { boundingRect = m_bindings[i].BoundingRect, succeeded = m_bindings[i].Succeeded } ); if (m_trackerHistory[i].Count > m_maxTrackerHistoryLength) { m_trackerHistory[i].Dequeue(); } } // Render results await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () => { m_renderStopWatch.Restart(); m_objectTrackRenderer.ClearCanvas(); m_objectTrackRenderer.RenderTrackerResults(m_trackerHistory, true); m_renderStopWatch.Stop(); // Print performance measurements PerformanceBlock.Text = String.Format( "Skill Eval: {0:0.000}ms, Render: {1:0.000}ms", evalTicks / (Stopwatch.Frequency / 1000F), m_renderStopWatch.ElapsedTicks / (Stopwatch.Frequency / 1000F)); }); } // Initialize any new trackers if (m_drawnRects.Count > 0) { // Initialize new trackers if desired m_bboxLock.Wait(); for (int i = 0; i < m_drawnRects.Count; i++) { ObjectTrackerBinding binding = await m_skill.CreateSkillBindingAsync() as ObjectTrackerBinding; await m_skill.InitializeTrackerAsync(binding, frame, m_drawnRects[i]); m_bindings.Add(binding); // Add corresponding tracker history m_trackerHistory.Add(new Queue <TrackerResult>()); m_trackerHistory.Last().Enqueue( new TrackerResult() { boundingRect = binding.BoundingRect, succeeded = true } ); } m_drawnRects.Clear(); m_bboxLock.Release(); } }