/// <summary>
/// Gets a video frame and processes it.
/// </summary>
/// <param name="Sender"></param>
/// <param name="e"></param>
void GetAndProcessFrame(object Sender, EventArgs e)
{
// Get the next image from the camera
var mat = capture.QueryFrame();
if (null == mat)
{
return;
}
// It is not clear if the whole image frame should be applied to
// the image processing stage, or if the image should be broken up
// into fixed sized regions. Or if the regions should overlap.
// Or if it is intended to be applied to just the center of the
// image.
//
// Experiments will need to be done.
// First, get the subframe that matches what I think the mobilenet aspect ratio is (1:1)
var nWidth = mat.Width - mat.Height;
var tmpi = new Mat(mat, new Rectangle((int)(nWidth / 2), 0, mat.Height, mat.Height));
var subImage = new Mat();
CvInvoke.Resize(
src: tmpi,
dst: subImage,
dsize: tmpi.Size);
// This holds the image in the various sizes and formats.
// It seems likely that stages will reuse the sizing/format so this
// reduces the work to set it up each time.
using var ic = new ImageContainer(subImage);
// This will hold the localization grid
Classification[,] localizationGrid = null;
// This will hold the set of labels that we wish to display
var newLabels = new Dictionary <string, int>();
// Process each of the select stages and make a note of teh labels
foreach (var stage in imageProcessingSteps)
{
// Process the image to get the classifications or localizations
var ret = stage.ProcessImage(ic);
// What kind of processing stage is?
if (stage.IsCategorization())
{
// This is a label, add the items to the list of labels to
// update on the display
if (ret.Length > 1)
{
foreach (var cat in ret)
{
if (cat.Probability >= 0.05)
{
newLabels[cat.Label] = -1;
}
else if (ret[0, 0].Probability >= 0.5)
{
newLabels[ret[0, 0].Label] = -1;
}
}
}
}
else
{
localizationGrid = ret;
}
}
// Update the labels displayed
// First, retain the index of the previous labels
var Used = new bool[textLabels.Length];
foreach (var l in usedLabels)
{
if (!newLabels.TryGetValue(l.Key, out var oldIdx))
{
continue;
}
newLabels[l.Key] = l.Value;
Used[l.Value] = true;
}
// Next, assign indices to the remaining labels.
var tmp = new string[newLabels.Count];
newLabels.Keys.CopyTo(tmp, 0);
foreach (var l in tmp)
{
var lidx = newLabels[l];
// Is it already assigned?
if (lidx >= 0)
{
continue;
}
// find a free label
for (var idx2 = 0; idx2 < textLabels.Length; idx2++)
{
if (!Used[idx2])
{
// Assign it this one
Used[idx2] = true;
newLabels[l] = idx2;
break;
}
}
}
// Now go thru and draw all of the labels on the screen
foreach (var kv in newLabels)
{
if (kv.Value < 0)
{
continue;
}
textLabels[kv.Value].Text = kv.Key;
}
// Blank out the text for the unused labels
for (int lidx = 0; lidx < Used.Length; lidx++)
{
if (!Used[lidx])
{
textLabels[lidx].Text = "";
}
}
usedLabels = newLabels;
// Draw the localization rectangles
if (null != localizationGrid)
{
for (int x = 0; x < 6; x++)
{
for (int y = 0; y < 6; y++)
{
if (localizationGrid[y, x].Probability >= 0.25)
{
CvInvoke.Rectangle(subImage,
new Rectangle((x * subImage.Width) / 6, (y * subImage.Height) / 6, subImage.Width / 6, subImage.Height / 6),
new Bgr(0, 255 * localizationGrid[y, x].Probability, 0).MCvScalar, 10);
}
}
}
}
// Draw the images obtained from the camera
imageBox1.Image = mat;
imageBox2.Image = ic.Image(new Size(128, 128), Emgu.CV.CvEnum.DepthType.Cv8U, false);
}