private void DrawAugmentedReality(FoundTemplateDesc found, Graphics gr)
{
string key = Path.GetDirectoryName(this.templateFile) + @"\" + found.template.name;
if (!this.AugmentedRealityImages.ContainsKey(key))
{
if (File.Exists(key))
{
this.AugmentedRealityImages[key] = Image.FromFile(key);
}
else
{
return;
}
}
Image image = this.AugmentedRealityImages[key];
Point point = found.sample.contour.SourceBoundingRect.Center();
GraphicsState gstate = gr.Save();
gr.TranslateTransform((float)point.X, (float)point.Y);
gr.RotateTransform((float)((180.0 * found.angle) / 3.1415926535897931));
gr.ScaleTransform((float)found.scale, (float)found.scale);
gr.DrawImage(image, new Point(-image.Width / 2, -image.Height / 2));
gr.Restore(gstate);
}
private void DrawAugmentedReality(FoundTemplateDesc found, Graphics gr)
{
string fileName = Path.GetDirectoryName(templateFile) + "\\" + found.template.name;
if (!AugmentedRealityImages.ContainsKey(fileName))
{
if (!File.Exists(fileName))
{
return;
}
AugmentedRealityImages[fileName] = System.Drawing.Image.FromFile(fileName);
}
System.Drawing.Image img = AugmentedRealityImages[fileName];
System.Drawing.Point p = found.sample.contour.SourceBoundingRect.Center();
var state = gr.Save();
gr.TranslateTransform(p.X, p.Y);
gr.RotateTransform((float)(180f * found.angle / Math.PI));
gr.ScaleTransform((float)(found.scale), (float)(found.scale));
gr.DrawImage(img, new System.Drawing.Point(-img.Width / 2, -img.Height / 2));
gr.Restore(state);
}
/// <summary>
/// Get the nearest templateClass for the refContour
/// </summary>
/// <param name="refContour">the contour wich class's is to be found</param>
/// <param name="r">the area of the contour</param>
/// <param name="classes">the list of the classes within to search</param>
/// <returns>the nearest class or "not found"</returns>
public static FoundTemplateDesc GetNearestClass(Contour <Point> refContour, Rectangle r, List <TemplateClass> classes, HandType handType)
{
contourClasses = classes;
List <FoundTemplateDesc> foundedTemplates = new List <FoundTemplateDesc>();
Template refTemp = new Template(refContour, r.Height * r.Width);
foreach (TemplateClass tc in contourClasses)
{
if (tc.htype == handType)
{
FoundTemplateDesc templateDesc = TemplateFinder.CompareTemplates(tc, refTemp);
if (templateDesc != null)
{
foundedTemplates.Add(templateDesc);
}
}
}
foundedTemplates = foundedTemplates.OrderBy(t => t.rate).ToList();
return((foundedTemplates.Count == 0) ? null : foundedTemplates.First());
}
public RockDesc(FoundTemplateDesc template)
{
this.rate = template.rate;
this.angle = template.angle;
this.scale = template.scale;
}
/// <summary>
/// find hands based on the informations provided by the skeleton
/// </summary>
/// <param name="skeleton">the skeleton provided by Kinect</param>
/// <param name="depthFrame">the depthFrame</param>
/// <returns>a handgesture as a string</returns>
public string FindHandGesture(HandType handtype)
{
//reset the values for hand detection
this.currentContour = null;
this.depthImage = new Image <Gray, byte>(this.depthFrameWidth, this.depthFrameHeight);
this.handJoint = KinectHelper.Instance.GetHandJoint(handtype, skeleton);
float zhand = handJoint.Position.Z;
JointType jointtype = (handtype == HandType.LEFT) ? JointType.HandLeft : JointType.HandRight;
if (min < zhand && zhand < max && CheckOrientation(handtype))
{
//set the ROI around the hand
Joint hand = skeleton.Joints[jointtype].ScaleTo((int)(this.depthFrameWidth * 1), (int)(this.depthFrameHeight * 1));
PointF center = new PointF(hand.Position.X, hand.Position.Y);
this.depthImage = new Image <Gray, byte>(this.depthFrameWidth, this.depthFrameHeight);
depthImage.Bytes = GetEverythingBetween(min, max, depthFrame);
recSize = new Size(75 - (int)(50 * (zhand - MinDepthDistance)), 90 - (int)(35.7 * (zhand - MinDepthDistance)));
if (CheckRectangleIsInside(depthImageBoth.Width, depthImageBoth.Height, recSize, center))
{
depthImage.ROI = new Rectangle((int)center.X - recSize.Width / 2, (int)center.Y - recSize.Height / 2, recSize.Width, recSize.Height);
if (!CheckForMovement(handtype))
{
//get the contour
ExtractContourAndHull(depthImage.Copy(), handtype);
//find the name of the gesture if there is one
if (this.currentContour != null)
{
FoundTemplateDesc result = TemplateFinder.GetNearestClass(this.currentContour, depthImage.ROI, this.templates, handtype);
if (result != null)
{
return(result.name.ToString());
}
}
}
if ((isLeftHandTracked == false && handtype == HandType.LEFT) || (isRightHandTracked == false && handtype == HandType.RIGHT))
{
if (handtype == HandType.RIGHT)
{
isRightHandTracked = true;
}
else
{
isLeftHandTracked = true;
}
}
}
}
else
{
if (handtype == HandType.LEFT)
{
isLeftHandTracked = false;
}
else
{
isRightHandTracked = false;
}
}
return(null);
}
/// <summary>
/// Processes image in gray colors
/// </summary>
public void ProcessImage(Image <Gray, byte> grayFrame)
{
// smoothing -> downscaling and upscaling via binary interpolation
Image <Gray, byte> smoothedGrayFrame = grayFrame.PyrDown();
smoothedGrayFrame = smoothedGrayFrame.PyrUp();
Image <Gray, byte> cannyFrame = null;
if (noiseFilter)
{
cannyFrame = smoothedGrayFrame.Canny(new Gray(cannyThreshold), new Gray(cannyThreshold));
}
if (blur)
{
grayFrame = smoothedGrayFrame;
}
// transform into binary image (2 values)
CvInvoke.cvAdaptiveThreshold(grayFrame, grayFrame, 255, Emgu.CV.CvEnum.ADAPTIVE_THRESHOLD_TYPE.CV_ADAPTIVE_THRESH_MEAN_C, Emgu.CV.CvEnum.THRESH.CV_THRESH_BINARY, adaptiveThresholdBlockSize + adaptiveThresholdBlockSize % 2 + 1, adaptiveThresholdParameter);
// invert
grayFrame._Not();
// logic OR will smooth additional noise
if (cannyFrame != null)
{
grayFrame._Or(cannyFrame);
}
this.binarizedFrame = grayFrame;
if (cannyFrame != null)
{
cannyFrame = cannyFrame.Dilate(3);
}
// find all contours
var sourceContours = grayFrame.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_NONE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_LIST);
// filter all contours
contours = FilterContours(sourceContours, cannyFrame, grayFrame.Width, grayFrame.Height);
lock (foundTemplates)
foundTemplates.Clear();
samples.Clear();
// process all contours in parallel
lock (templates)
Parallel.ForEach <Contour <Point> >(contours, (contour) => {
// create a template sample from each contour and try to find its relevant template
var arr = contour.ToArray();
Template sample = new Template(arr, contour.Area, samples.templateSize);
lock (samples)
samples.Add(sample);
if (!onlyFindContours)
{
// find a template according to the sample
FoundTemplateDesc desc = finder.FindTemplate(templates, sample);
if (desc != null)
{
lock (foundTemplates)
foundTemplates.Add(desc);
}
}
}
);
FilterByIntersection(ref foundTemplates);
}
//main picture boc pain event
private void PictureBoxImageTextDeepLearning_Paint(object sender, PaintEventArgs e)
{
bool Re = false;
//when foregin is ready
if (d != null)
{
//initiate local vars
Font font;
Brush brush;
Brush brush2;
Pen pen;
bool flag2;
//when is ready top detected unconjuncted shapes set draw parameters
if (!ReferenceEquals(d.frame, null))
{
font = new Font(d.Font.FontFamily, 24f);
e.Graphics.DrawString(d.lbFPS.Text, new Font(d.Font.FontFamily, 16f), Brushes.Yellow, new PointF(1f, 1f));
brush = new SolidBrush(Color.FromArgb(0xff, 0, 0, 0));
brush2 = new SolidBrush(Color.FromArgb(0xff, 0xff, 0xff, 0xff));
pen = new Pen(Color.FromArgb(150, 0, 0xff, 0));
flag2 = false;
if (!flag2)
{
using (List <Contour <Point> > .Enumerator enumerator = d.processor.contours.GetEnumerator())
{
while (true)
{
flag2 = enumerator.MoveNext();
if (!flag2)
{
break;
}
Contour <Point> current = enumerator.Current;
if (current.Total > 1)
{
e.Graphics.DrawLines(Pens.Red, current.ToArray());
}
}
}
}
}
else
{
return;
}
lock (d.processor.foundTemplates)
{
using (List <FoundTemplateDesc> .Enumerator enumerator2 = d.processor.foundTemplates.GetEnumerator())
{
while (true)
{
flag2 = enumerator2.MoveNext();
if (!flag2)
{
break;
}
FoundTemplateDesc current = enumerator2.Current;
if (current.template.name.EndsWith(".png") || current.template.name.EndsWith(".jpg"))
{
d.DrawAugmentedReality(current, e.Graphics);
continue;
}
Rectangle sourceBoundingRect = current.sample.contour.SourceBoundingRect;
Point point = new Point((sourceBoundingRect.Left + sourceBoundingRect.Right) / 2, sourceBoundingRect.Top);
string name = current.template.name;
if (d.showAngle)
{
name = name + $"angle={((180.0 * current.angle) / 3.1415926535897931):000}°scale={current.scale:0.0}";
}
if (!Recognized)
{
textBoxImageTextDeepLearning.Text += name;
textBoxImageTextDeepLearning.Refresh();
textBoxImageTextDeepLearning.Update();
Re = true;
}
e.Graphics.DrawRectangle(pen, sourceBoundingRect);
e.Graphics.DrawString(name, font, brush, new PointF((float)((point.X + 1) - (font.Height / 3)), (float)((point.Y + 1) - font.Height)));
e.Graphics.DrawString(name, font, brush2, new PointF((float)(point.X - (font.Height / 3)), (float)(point.Y - font.Height)));
}
}
}
}
if (Re)
{
Recognized = true;
}
PictureBoxImageTextDeepLearning.Update();
PictureBoxImageTextDeepLearning.Refresh();
}
private void ibMain_Paint(object sender, PaintEventArgs e)
{
Font font;
Brush brush;
Brush brush2;
Pen pen;
bool flag2;
if (!ReferenceEquals(this.frame, null))
{
font = new Font(this.Font.FontFamily, 24f);
e.Graphics.DrawString(this.lbFPS.Text, new Font(this.Font.FontFamily, 16f), Brushes.Yellow, new PointF(1f, 1f));
brush = new SolidBrush(Color.FromArgb(0xff, 0, 0, 0));
brush2 = new SolidBrush(Color.FromArgb(0xff, 0xff, 0xff, 0xff));
pen = new Pen(Color.FromArgb(150, 0, 0xff, 0));
flag2 = !this.cbShowContours.Checked;
if (!flag2)
{
using (List <Contour <Point> > .Enumerator enumerator = this.processor.contours.GetEnumerator())
{
while (true)
{
flag2 = enumerator.MoveNext();
if (!flag2)
{
break;
}
Contour <Point> current = enumerator.Current;
if (current.Total > 1)
{
e.Graphics.DrawLines(Pens.Red, current.ToArray());
}
}
}
}
}
else
{
return;
}
lock (this.processor.foundTemplates)
{
using (List <FoundTemplateDesc> .Enumerator enumerator2 = this.processor.foundTemplates.GetEnumerator())
{
while (true)
{
flag2 = enumerator2.MoveNext();
if (!flag2)
{
break;
}
FoundTemplateDesc current = enumerator2.Current;
if (current.template.name.EndsWith(".png") || current.template.name.EndsWith(".jpg"))
{
this.DrawAugmentedReality(current, e.Graphics);
continue;
}
Rectangle sourceBoundingRect = current.sample.contour.SourceBoundingRect;
Point point = new Point((sourceBoundingRect.Left + sourceBoundingRect.Right) / 2, sourceBoundingRect.Top);
string name = current.template.name;
if (this.showAngle)
{
name = name + $"
angle={((180.0 * current.angle) / 3.1415926535897931):000}°
scale={current.scale:0.0}";
}
e.Graphics.DrawRectangle(pen, sourceBoundingRect);
e.Graphics.DrawString(name, font, brush, new PointF((float)((point.X + 1) - (font.Height / 3)), (float)((point.Y + 1) - font.Height)));
e.Graphics.DrawString(name, font, brush2, new PointF((float)(point.X - (font.Height / 3)), (float)(point.Y - font.Height)));
}
}
}
}