/// <summary>
/// Bind and evaluate the frame with the ObjectDetector skill
/// </summary>
/// <param name="frame"></param>
/// <returns></returns>
private async Task RunSkillsAsync(VideoFrame frame)
{
m_bindTime = 0F;
m_evalTime = 0F;
m_frameCounter++;
// Update all trackers
m_evalStopwatch.Restart();
var bindTasks = new List <Task>();
foreach (var binding in m_trackerBindings)
{
bindTasks.Add(binding.SetInputImageAsync(frame).AsTask());
}
await Task.WhenAll(bindTasks);
m_evalStopwatch.Stop();
m_bindTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f;
m_evalStopwatch.Restart();
var evalTasks = new List <Task>();
foreach (var binding in m_trackerBindings)
{
evalTasks.Add(m_trackerSkill.EvaluateAsync(binding).AsTask());
}
await Task.WhenAll(evalTasks);
m_evalStopwatch.Stop();
m_evalTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f;
// Add results to histories
for (int i = 0; i < m_trackerBindings.Count; i++)
{
m_trackerHistories[i].Enqueue(
new TrackerResult()
{
label = m_trackerHistories[i].Peek().label,
boundingRect = m_trackerBindings[i].BoundingRect,
succeeded = m_trackerBindings[i].Succeeded
}
);
if (m_trackerHistories[i].Count > m_maxTrackerHistoryLength)
{
m_trackerHistories[i].Dequeue();
}
}
// Run detector if no successful trackers or we've reached our desired detector period
if (m_trackerBindings.Where(binding => binding.Succeeded).Count() == 0 || (m_frameCounter % m_detectorEvalInterval) == 0)
{
// Bind
m_evalStopwatch.Restart();
await m_detectorBinding.SetInputImageAsync(frame);
m_evalStopwatch.Stop();
m_bindTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f;
// Evaluate
m_evalStopwatch.Restart();
await m_detectorSkill.EvaluateAsync(m_detectorBinding);
m_evalStopwatch.Stop();
m_evalTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f;
// Clear and re-initialize trackers
m_trackerBindings.Clear();
m_trackerHistories.Clear();
IEnumerable <ObjectDetectorResult> filteredDetections = m_detectorBinding.DetectedObjects.Where(det => ((m_objectKinds?.Count ?? 0) == 0) || m_objectKinds.Contains(det.Kind));
var initializeTasks = new List <Task>();
m_evalStopwatch.Restart(); // Since we're initializing trackers in parallel, it's easiest to count it all as evaluation
// (including tracker binding)
Dictionary <string, int> objectKindCounter = new Dictionary <string, int>();
foreach (ObjectDetectorResult detection in filteredDetections)
{
// Cap at max trackers
if (m_trackerBindings.Count >= m_maxNumberTrackers)
{
break;
}
// Do some sanity checks to make sure the detection is valid
if (detection.Rect.Width <= 0 || detection.Rect.Height <= 0)
{
break;
}
// Create and initialize new tracker
ObjectTrackerBinding binding = await m_trackerSkill.CreateSkillBindingAsync() as ObjectTrackerBinding;
Rect clampedRect = new Rect(
Math.Max(detection.Rect.X, 0.0),
Math.Max(detection.Rect.Y, 0.0),
detection.Rect.Width,
detection.Rect.Height);
initializeTasks.Add(m_trackerSkill.InitializeTrackerAsync(binding, frame, clampedRect).AsTask());
m_trackerBindings.Add(binding);
// Add corresponding tracker history
string objectKindStr = detection.Kind.ToString();
if (!objectKindCounter.ContainsKey(objectKindStr))
{
objectKindCounter.Add(objectKindStr, 0);
}
string label = $"{objectKindStr} {++objectKindCounter[objectKindStr]}";
m_trackerHistories.Add(new Queue <TrackerResult>());
m_trackerHistories.Last().Enqueue(
new TrackerResult()
{
label = label,
boundingRect = clampedRect,
succeeded = true
}
);
}
await Task.WhenAll(initializeTasks);
m_evalStopwatch.Stop();
m_evalTime += (float)m_evalStopwatch.ElapsedTicks / Stopwatch.Frequency * 1000f;
m_frameCounter = 0; // Reset frame counter
}
}
/// <summary>
/// Update existing trackers with the latest frame and initialize any new trackers based on user input
/// This should always be called from inside the lock (m_lock)
/// </summary>
/// <param name="frame"></param>
/// <returns></returns>
private async Task RunSkillAsync(VideoFrame frame)
{
// Update existing trackers
if (m_bindings.Count > 0)
{
long evalTicks = 0L;
for (int i = 0; i < m_bindings.Count; i++)
{
await m_bindings[i].SetInputImageAsync(frame);
m_skillEvalStopWatch.Restart();
await m_skill.EvaluateAsync(m_bindings[i]);
m_skillEvalStopWatch.Stop();
evalTicks += m_skillEvalStopWatch.ElapsedTicks;
// Add result to history
m_trackerHistory[i].Enqueue(
new TrackerResult()
{
boundingRect = m_bindings[i].BoundingRect,
succeeded = m_bindings[i].Succeeded
}
);
if (m_trackerHistory[i].Count > m_maxTrackerHistoryLength)
{
m_trackerHistory[i].Dequeue();
}
}
// Render results
await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
m_renderStopWatch.Restart();
m_objectTrackRenderer.ClearCanvas();
m_objectTrackRenderer.RenderTrackerResults(m_trackerHistory, true);
m_renderStopWatch.Stop();
// Print performance measurements
PerformanceBlock.Text = String.Format(
"Skill Eval: {0:0.000}ms, Render: {1:0.000}ms",
evalTicks / (Stopwatch.Frequency / 1000F),
m_renderStopWatch.ElapsedTicks / (Stopwatch.Frequency / 1000F));
});
}
// Initialize any new trackers
if (m_drawnRects.Count > 0)
{
// Initialize new trackers if desired
m_bboxLock.Wait();
for (int i = 0; i < m_drawnRects.Count; i++)
{
ObjectTrackerBinding binding = await m_skill.CreateSkillBindingAsync() as ObjectTrackerBinding;
await m_skill.InitializeTrackerAsync(binding, frame, m_drawnRects[i]);
m_bindings.Add(binding);
// Add corresponding tracker history
m_trackerHistory.Add(new Queue <TrackerResult>());
m_trackerHistory.Last().Enqueue(
new TrackerResult()
{
boundingRect = binding.BoundingRect,
succeeded = true
}
);
}
m_drawnRects.Clear();
m_bboxLock.Release();
}
}