/// <summary> /// Creates a DepthCorrectionMap from an uncorrected raw image and the desired TableDistance /// </summary> /// <param name="uncorrectedImage"></param> /// <param name="averageTableDistance"></param> /// <returns></returns> public DepthCorrectionMap CreateDepthCorrectionMap(DepthImage uncorrectedImage, int averageTableDistance) { DepthCorrectionMap correctionMap = new DepthCorrectionMap(uncorrectedImage.Width, uncorrectedImage.Height); int lastValue = 0; for (int x = 0; x < uncorrectedImage.Width; x++) { for (int y = 0; y < uncorrectedImage.Height; y++) { if (uncorrectedImage.Data[x, y] > 0) { //Correctly recognized point: calculate correction value correctionMap.CorrectionData[x, y] = averageTableDistance - uncorrectedImage.Data[x, y]; lastValue = averageTableDistance - uncorrectedImage.Data[x, y]; } else { //Not recognized point (height = 0) use recent calibration value as approximation correctionMap.CorrectionData[x, y] = lastValue; } } } return correctionMap; }
/// <summary> /// Applies a correctionMap to a raw DepthImage /// </summary> /// <param name="rawImage"></param> /// <param name="correctionMap"></param> /// <returns></returns> public DepthImage ApplyDepthCorrection(DepthImage rawImage, DepthCorrectionMap correctionMap) { if (rawImage.Width != correctionMap.Width ||rawImage.Height != correctionMap.Height) { throw new Exception("Image size does not match"); } //calculate the values for the section that is not involved in border cutting int BorderCutXmin = 0 + correctionMap.CutOffLeft; int BorderCutXmax = rawImage.Width - 1 - correctionMap.CutOffRight; int BorderCutYmin = 0 + correctionMap.CutOffTop; int BorderCutYmax = rawImage.Height - 1 - correctionMap.CutOffBOttom; //Apply the Depth Correction for (int x=0; x < rawImage.Width; x++) { for (int y = 0; y < rawImage.Height; y++) { if (rawImage.Data[x,y] > 0) //Only if there wasn't a reading error rawImage.Data[x, y] = rawImage.Data[x, y] + correctionMap.CorrectionData[x, y]; //Coordinates outside the CutOffBorder? if (!((BorderCutXmin <= x) && (x <= BorderCutXmax) && (BorderCutYmin <= y) && (y <= BorderCutYmax))) rawImage.Data[x, y] = 0; } } return rawImage; }
public RecognitionDataPacket() { rawDepthImage = null; correctedDepthImage = null; TableObjects = new List<TableObject>(); HandObj = null; objectmap = null; neighbourmap = null; bmpCorrectedDepth = null; bmpRawDepth = null; bmpVideoFrame = null; RecognitionDuration = 0; }
/// <summary> /// Creates a boolmap out of an depth image. Every pixel that has an height > 0 is set to true /// </summary> /// <param name="source"></param> /// <returns></returns> private bool[,] CreateBoolMap(DepthImage source) { bool[,] boolmap = new bool[source.Width,source.Height]; for (int y = 0; y < source.Height; y++) { for (int x = 0; x < source.Width; x++) { if (source.Data[x,y] > 0) { boolmap[x, y] = true; } else { boolmap[x, y] = false; } } } return boolmap; }
public List<TableObject> SeperateObjects(ref DepthImage image, out bool[,] boolmap_object, out int[,,] neighbourmap) { //Create 2 seperate depth-images, remove the objects (for hand recognition) and the hand (for object recognition) DepthImage HandImage = image.Clone(); DepthImage ObjectImage = image.Clone(); SplitUpDepthImages(ref HandImage, ref ObjectImage); //For object recognition, create a BooleanXY-Map (true for all objects with a height > 0) bool[,] boolmap_hand = CreateBoolMap(HandImage); boolmap_object = CreateBoolMap(ObjectImage); if (SettingsManager.RecognitionSet.SaveDebugMaps) { Bitmap dm_obj = MapVisualizer.VisualizeDepthImage(ObjectImage, false, false); dm_obj.Save("ObjectDepthImage.bmp"); Bitmap dm_h = MapVisualizer.VisualizeDepthImage(HandImage, false, false); dm_h.Save("HandDepthImage.bmp"); Bitmap bobj = MapVisualizer.VisualizeBoolMap(boolmap_hand, image.Width, image.Height); bobj.Save("boolmap_hand.bmp"); Bitmap bmpo = MapVisualizer.VisualizeBoolMap(boolmap_object, image.Width, image.Height); bmpo.Save("boolmap_object.bmp"); Bitmap ci = MapVisualizer.VisualizeDepthImage(image, false, false); ci.Save("normalized_depthimage.bmp"); } //Run the Hand Recognition HandRecognizer hrec = new HandRecognizer(); HandObject hand = hrec.RecognizeHands(ref boolmap_hand, image.Width, image.Height); //RunTheObjectRecognition ObjectRecognizer orec = new ObjectRecognizer(); List<TableObject> tableObjects = orec.RecognizeObjects(boolmap_object, image, out neighbourmap); //If there is a hand, add it to the TableObjects if (hand != null) tableObjects.Add(hand); //Return the ObjectList return tableObjects; }
private void b_aktualisieren_Click(object sender, RoutedEventArgs e) { if (_tmgr.LastKinectDepthFrame == null) { MessageBox.Show("Kinect muss zuerst initialisiert werden!", "Fehler!"); return; } int width, height; SettingsManager.KinectSet.GetDepthResolution(out width, out height); _dimage = new DepthImage(_tmgr.LastKinectDepthFrame, width, height); Bitmap bmp = MapVisualizer.VisualizeDepthImage(_dimage,false,false); image.Source = bmp.ToWpfBitmap(); s_cutoff_down.Value = SettingsManager.PreprocessingSet.DefaultCorrectionMap.CutOffBOttom; s_cutoff_left.Value = SettingsManager.PreprocessingSet.DefaultCorrectionMap.CutOffLeft; s_cutoff_right.Value = SettingsManager.PreprocessingSet.DefaultCorrectionMap.CutOffRight; s_cutoff_top.Value = SettingsManager.PreprocessingSet.DefaultCorrectionMap.CutOffTop; l_höhe.Text = SettingsManager.RecognitionSet.TableDistance.ToString(); }
/// <summary> /// Recognizes the Objects via a depth image and a prepared boolmap /// </summary> /// <param name="prepared_boolmap">a prepared boolmap, with only objects marked as true</param> /// <param name="image">the depth image, used to calculate the height of the recognized objects</param> /// <returns></returns> public List<TableObject> RecognizeObjects(bool[,] prepared_boolmap, DepthImage image, out int[,,] neighbourmap) { //Check in a raster whether there are any true pixels List<TPoint> TrueRasterPoints = GetTrueRasterPoints(prepared_boolmap, image); //Create a neighbourmap - an int[x,y,n] array, where the int stores the count of the whitepixel-neighbours of the object (n=0) and the size of the circle/rect used //to count these neighbours (n=1). The circle/rect is scaled up until a defined percentage of the pixels is false, meaning no part of the object neighbourmap = CreateNeighbourMap(prepared_boolmap, image, TrueRasterPoints); //DEBUG if (SettingsManager.RecognitionSet.SaveDebugMaps) { Bitmap bmp = MapVisualizer.VisualizeNeighbourMap(neighbourmap, image.Width, image.Height); bmp.Save("neigbourmap.bmp"); } //Now select the ObjectCenters (maximum neigbour-values on the neigbourmap) List<ObjectPoint> pointlist = SelectObjectCenters(neighbourmap, image.Width, image.Height); //Create TableObjects from the ObjectCenters List return GenerateTableObjects(pointlist, image); }
public DepthImage NormalizeHeights(DepthImage source) { for (int y = 0; y < source.Height; y++) { for (int x = 0; x < source.Width; x++) { int height = source.Data[x, y]; //Normalize Height (Calculate the object height in respect to the table surface) height = SettingsManager.RecognitionSet.TableDistance - height; //is the height within the Range of the table surface? then set to 0 //also set to 0 if height is negative -> "under" the table if (height < 0 || SettingsManager.RecognitionSet.TableDistanceRange > height) height = 0; //is the height too high, set it to 0 if (height > SettingsManager.RecognitionSet.HandMaximalHeight) height = 0; source.Data[x, y] = height; } } return source; }
private void DoRecognitionWork(object data) { object[] dataArray = (object[]) data; PlanarImage pimg = (PlanarImage) dataArray[0]; int[] deptharray = (int[]) dataArray[1]; Bitmap colorFrame = (Bitmap) dataArray[2]; RecognitionDataPacket rpacket = new DataStructures.RecognitionDataPacket(); DateTime dtBegin = DateTime.Now; //Create DepthImage DepthImage dimg = new DepthImage(deptharray,pimg.Width,pimg.Height); rpacket.rawDepthImage = dimg.Clone(); //Correct the image DepthMapPreprocessor dmp = new DepthMapPreprocessor(); dimg = dmp.ApplyDepthCorrection(dimg, SettingsManager.PreprocessingSet.DefaultCorrectionMap); dimg = dmp.NormalizeHeights(dimg); ObjectSeperator objectSeperator = new ObjectSeperator(); //Seperate objects bool[,] boolmap_object; int[,,] neighbourmap; List<TableObject> objects = objectSeperator.SeperateObjects(ref dimg,out boolmap_object,out neighbourmap); //if supplied, extract the relevant bitmap parts from the ColorFrame if (colorFrame != null) { ObjectVideoBitmapAssigner ovba = new ObjectVideoBitmapAssigner(); ovba.AssignVideoBitmap(objects, colorFrame); } //Extract hand object from table objects if (objects.Where( o => o.GetType() == typeof(HandObject)).Count() > 0) { rpacket.HandObj = (HandObject)objects.Where(o => o.GetType() == typeof (HandObject)).ToArray()[0]; } //Fill DataPacket with Data rpacket.correctedDepthImage = dimg; rpacket.TableObjects = objects; rpacket.objectmap = boolmap_object; rpacket.neighbourmap = neighbourmap; rpacket.bmpVideoFrame = colorFrame; TimeSpan ts = DateTime.Now - dtBegin; rpacket.RecognitionDuration = (int)Math.Round(ts.TotalMilliseconds); if (SettingsManager.RecognitionSet.SaveDebugMaps) { Bitmap bmp = MapVisualizer.VisualizeDepthImage(rpacket.rawDepthImage); bmp.Save("rawDepthImage.bmp"); } //Event OnRecognitionFinished(rpacket); }
private void SplitUpDepthImages(ref DepthImage HandImage, ref DepthImage objectImage) { for (int x=0;x<HandImage.Width;x++) { for(int y=0;y<HandImage.Height;y++) { int heigt = HandImage.Data[x, y]; if (HandImage.Data[x, y] > SettingsManager.RecognitionSet.HandMaximalHeight || HandImage.Data[x, y] <= SettingsManager.RecognitionSet.ObjectMaximalHeight) HandImage.Data[x, y] = 0; if (objectImage.Data[x, y] > SettingsManager.RecognitionSet.ObjectMaximalHeight) objectImage.Data[x, y] = 0; } } }
public static Bitmap VisualizeDepthImage(DepthImage img, bool invert=false, bool printDepthValues = false) { //Calculate Min Value int[] RawData = img.RawData; int min = 0; for (int i = 1; i < RawData.Count(); i++) { if (min == 0) min = RawData[i]; if ((RawData[i] < min) && (RawData[i] != 0)) min = RawData[i]; } //Calculate max int max = RawData[0]; for (int i = 1; i < RawData.Count(); i++) { if (RawData[i] > max) max = RawData[i]; } if (max == 0) max = 1; //The range between min and max has to fit into 255 bit (grayscale image) double scale_multiplicator = (float)255.0 / Convert.ToDouble(max - min); //Convert each pixel Bitmap grayscale = new Bitmap(img.Width, img.Height); int indexc = 0; for (var y = 0; y < img.Height; y++) { for (var x = 0; x < img.Width; x++) { int value = img.Data[x, y]; int rgb_value = (int)Math.Round((value - min) * scale_multiplicator); if (rgb_value < 0) rgb_value = 0; if (rgb_value > 255) rgb_value = 255; if (invert) rgb_value = 255 - rgb_value; Color col = new Color(); col = Color.FromArgb(rgb_value, rgb_value, rgb_value); grayscale.SetPixel(x, y, col); indexc++; } } //Add information if desired if (printDepthValues) { /* Graphics g = Graphics.FromImage(grayscale); g.DrawString("min: " + min.ToString() + "mm max: " + max.ToString() + "mm", new Font("Tahoma", 10), Brushes.Red, new PointF(0, 0)); return new Bitmap(img.Width, img.Height, g); * */ //TODO: Bild wird schwarz, bug beheben } return grayscale; }
private List<TableObject> GenerateTableObjects(List<ObjectPoint> pointlist, DepthImage image) { List<TableObject> tableobjects = new List<TableObject>(); foreach (ObjectPoint op in pointlist) { TableObject tobj = new TableObject(); tobj.Center = new TPoint(op.X, op.Y, TPoint.PointCreationType.depth); tobj.CenterDefined = true; tobj.Radius = op.RectSize; tobj.Height = image.Data[op.X, op.Y]; tableobjects.Add(tobj); } return tableobjects; }
/// <summary> /// Check in a raster whether there are any true pixels /// </summary> /// <param name="prepared_boolmap"></param> /// <param name="image"></param> /// <returns></returns> private static List<TPoint> GetTrueRasterPoints(bool[,] prepared_boolmap, DepthImage image) { List<TPoint> TrueRasterPoints = new List<TPoint>(); for (int x = 0; x < image.Width; x = x + SettingsManager.RecognitionSet.ObjectRecognitionGridSpacing) { for (int y = 0; y < image.Height; y = y + SettingsManager.RecognitionSet.ObjectRecognitionGridSpacing) { if (prepared_boolmap[x, y]) TrueRasterPoints.Add(new TPoint(x, y, TPoint.PointCreationType.depth)); } } return TrueRasterPoints; }
/// <summary> /// Creates a neighbourmap - an int[x,y,n] array, where the int stores the count of the whitepixel-neighbours of the object (n=0) and the size of the circle/rect used /// to count these neighbours (n=1). The circle/rect is scaled up until a defined percentage of the pixels is false, meaning no part of the object. the neigbourmap is only created around true rasterpoints to be eficcent /// </summary> /// <param name="boolmap">The boolmap</param> /// <param name="image">Depth Image</param> /// <param name="rasterpoints">The calculated Rasterpoints</param> /// <returns></returns> private static int[,,] CreateNeighbourMap(bool[,] boolmap, DepthImage image, List<TPoint> rasterpoints) { int[,,] neighbourmap = new int[image.Width,image.Height,2]; //Do this for each region arount the true rasterpoints foreach (TPoint rasterpoint in rasterpoints) { //calculate the area: the Gridsize after the points Position int xmin = rasterpoint.DepthX; int ymin = rasterpoint.DepthY; int xmax = rasterpoint.DepthX + SettingsManager.RecognitionSet.ObjectRecognitionGridSpacing; if (xmax >= image.Width) xmax = image.Width - 1; int ymax = rasterpoint.DepthY + SettingsManager.RecognitionSet.ObjectRecognitionGridSpacing; if (ymax >= image.Height) ymax = image.Height - 1; TRectangle area = new TRectangle(xmin, ymin, xmax, ymax); //Now calculate the values for every point in the area CalculateNeigbourValues(boolmap, area, ref neighbourmap, image.Width, image.Height); } return neighbourmap; }