/// <summary> /// Gets a video frame and processes it. /// </summary> /// <param name="Sender"></param> /// <param name="e"></param> void GetAndProcessFrame(object Sender, EventArgs e) { // Get the next image from the camera var mat = capture.QueryFrame(); if (null == mat) { return; } // It is not clear if the whole image frame should be applied to // the image processing stage, or if the image should be broken up // into fixed sized regions. Or if the regions should overlap. // Or if it is intended to be applied to just the center of the // image. // // Experiments will need to be done. // First, get the subframe that matches what I think the mobilenet aspect ratio is (1:1) var nWidth = mat.Width - mat.Height; var tmpi = new Mat(mat, new Rectangle((int)(nWidth / 2), 0, mat.Height, mat.Height)); var subImage = new Mat(); CvInvoke.Resize( src: tmpi, dst: subImage, dsize: tmpi.Size); // This holds the image in the various sizes and formats. // It seems likely that stages will reuse the sizing/format so this // reduces the work to set it up each time. using var ic = new ImageContainer(subImage); // This will hold the localization grid Classification[,] localizationGrid = null; // This will hold the set of labels that we wish to display var newLabels = new Dictionary <string, int>(); // Process each of the select stages and make a note of teh labels foreach (var stage in imageProcessingSteps) { // Process the image to get the classifications or localizations var ret = stage.ProcessImage(ic); // What kind of processing stage is? if (stage.IsCategorization()) { // This is a label, add the items to the list of labels to // update on the display if (ret.Length > 1) { foreach (var cat in ret) { if (cat.Probability >= 0.05) { newLabels[cat.Label] = -1; } else if (ret[0, 0].Probability >= 0.5) { newLabels[ret[0, 0].Label] = -1; } } } } else { localizationGrid = ret; } } // Update the labels displayed // First, retain the index of the previous labels var Used = new bool[textLabels.Length]; foreach (var l in usedLabels) { if (!newLabels.TryGetValue(l.Key, out var oldIdx)) { continue; } newLabels[l.Key] = l.Value; Used[l.Value] = true; } // Next, assign indices to the remaining labels. var tmp = new string[newLabels.Count]; newLabels.Keys.CopyTo(tmp, 0); foreach (var l in tmp) { var lidx = newLabels[l]; // Is it already assigned? if (lidx >= 0) { continue; } // find a free label for (var idx2 = 0; idx2 < textLabels.Length; idx2++) { if (!Used[idx2]) { // Assign it this one Used[idx2] = true; newLabels[l] = idx2; break; } } } // Now go thru and draw all of the labels on the screen foreach (var kv in newLabels) { if (kv.Value < 0) { continue; } textLabels[kv.Value].Text = kv.Key; } // Blank out the text for the unused labels for (int lidx = 0; lidx < Used.Length; lidx++) { if (!Used[lidx]) { textLabels[lidx].Text = ""; } } usedLabels = newLabels; // Draw the localization rectangles if (null != localizationGrid) { for (int x = 0; x < 6; x++) { for (int y = 0; y < 6; y++) { if (localizationGrid[y, x].Probability >= 0.25) { CvInvoke.Rectangle(subImage, new Rectangle((x * subImage.Width) / 6, (y * subImage.Height) / 6, subImage.Width / 6, subImage.Height / 6), new Bgr(0, 255 * localizationGrid[y, x].Probability, 0).MCvScalar, 10); } } } } // Draw the images obtained from the camera imageBox1.Image = mat; imageBox2.Image = ic.Image(new Size(128, 128), Emgu.CV.CvEnum.DepthType.Cv8U, false); }