// Estimate pose for of the given glyph private void EstimateGlyphPose(ExtractedGlyphData glyph, int glyphID) { int imageCenterX = imageSize.Width >> 1; int imageCenterY = imageSize.Height >> 1; Point[] glyphPoints = new Point[4]; Matrix3x3 rotation; Vector3 translation; // get array of points with coordinates in Cartesian system with origin in image center for (int i = 0; i < 4; i++) { glyphPoints[i] = new Point( glyph.RecognizedQuadrilateral[i].X - imageCenterX, imageCenterY - glyph.RecognizedQuadrilateral[i].Y); } // estimate pose using Coplanar POSIT algorithm posit.EstimatePose(glyphPoints, out rotation, out translation); glyph.TransformationMatrix = Matrix4x4.CreateTranslation(translation) * Matrix4x4.CreateFromRotation(rotation); glyph.IsTransformationDetected = true; // check if we have previous rotation of the glyph if (!prevRotation.ContainsKey(glyphID)) { // remember it if not prevRotation.Add(glyphID, posit.BestEstimatedRotation); } else { Matrix3x3 newRotation = posit.BestEstimatedRotation; // check if best estimation is at least twice is better than the alternate // (better according to the POSIT algorithm) if (posit.AlternateEstimationError / posit.BestEstimationError < 2) { // error difference is not very big, so compare both transformations with previous // and select the one which seems to be closer to it Matrix3x3 d1 = prevRotation[glyphID] - posit.BestEstimatedRotation; Matrix3x3 d2 = prevRotation[glyphID] - posit.AlternateEstimatedRotation; float e1 = d1.GetRow(0).Square + d1.GetRow(1).Square + d1.GetRow(2).Square; float e2 = d2.GetRow(0).Square + d2.GetRow(1).Square + d2.GetRow(2).Square; if (e1 > e2) { glyph.TransformationMatrix = Matrix4x4.CreateTranslation(posit.AlternateEstimatedTranslation) * Matrix4x4.CreateFromRotation(posit.AlternateEstimatedRotation); newRotation = posit.AlternateEstimatedRotation; } } prevRotation[glyphID] = newRotation; } }
// Try recognizing the glyph in the specified image defined by the specified quadrilateral private ExtractedGlyphData RecognizeGlyph(UnmanagedImage image, List <IntPoint> quadrilateral, int counter) // counter -> HLE { // extract glyph image quadrilateralTransformation.SourceQuadrilateral = quadrilateral; UnmanagedImage glyphImage = quadrilateralTransformation.Apply(image); // otsu thresholding otsuThresholdFilter.ApplyInPlace(glyphImage); // recognize raw glyph float confidence; byte[,] glyphValues = binaryGlyphRecognizer.Recognize(glyphImage, new Rectangle(0, 0, glyphImage.Width, glyphImage.Height), out confidence); if (confidence >= minConfidenceLevel) { if ((Glyph.CheckIfGlyphHasBorder(glyphValues)) && (Glyph.CheckIfEveryRowColumnHasValue(glyphValues))) { // ---HLE //Save out image for checking System.Drawing.Bitmap managedglyphImage = glyphImage.ToManagedImage(); managedglyphImage.Save("C:\\Users\\heve\\Documents\\_Work_PointCloud\\AAG\\ImageProcessing\\glyphImage_" + counter.ToString() + ".png", System.Drawing.Imaging.ImageFormat.Png); // ---HLE ExtractedGlyphData foundGlyph = new ExtractedGlyphData(quadrilateral, glyphValues, confidence); if (glyphDatabase != null) { int rotation; foundGlyph.RecognizedGlyph = glyphDatabase.RecognizeGlyph(glyphValues, out rotation); if (rotation != -1) { foundGlyph.RecognizedQuadrilateral = foundGlyph.Quadrilateral; while (rotation > 0) { foundGlyph.RecognizedQuadrilateral.Add(foundGlyph.RecognizedQuadrilateral[0]); foundGlyph.RecognizedQuadrilateral.RemoveAt(0); rotation -= 90; } } } return(foundGlyph); } } return(null); }
// Try recognizing the glyph in the specified image defined by the specified quadrilateral private ExtractedGlyphData RecognizeGlyph(UnmanagedImage image, List <IntPoint> quadrilateral) { // extract glyph image quadrilateralTransformation.SourceQuadrilateral = quadrilateral; UnmanagedImage glyphImage = quadrilateralTransformation.Apply(image); // otsu thresholding otsuThresholdFilter.ApplyInPlace(glyphImage); // recognize raw glyph float confidence; byte[,] glyphValues = binaryGlyphRecognizer.Recognize(glyphImage, new Rectangle(0, 0, glyphImage.Width, glyphImage.Height), out confidence); if (confidence >= minConfidenceLevel) { if ((Glyph.CheckIfGlyphHasBorder(glyphValues)) && (Glyph.CheckIfEveryRowColumnHasValue(glyphValues))) { ExtractedGlyphData foundGlyph = new ExtractedGlyphData(quadrilateral, glyphValues, confidence); if (glyphDatabase != null) { int rotation; foundGlyph.RecognizedGlyph = glyphDatabase.RecognizeGlyph(glyphValues, out rotation); if (rotation != -1) { foundGlyph.RecognizedQuadrilateral = foundGlyph.Quadrilateral; while (rotation > 0) { foundGlyph.RecognizedQuadrilateral.Add(foundGlyph.RecognizedQuadrilateral[0]); foundGlyph.RecognizedQuadrilateral.RemoveAt(0); rotation -= 90; } } } return(foundGlyph); } } return(null); }
/// <summary> /// Clone the object by making its exact copy. /// </summary> /// /// <returns>Returns clone of the object.</returns> /// public object Clone( ) { ExtractedGlyphData clone = new ExtractedGlyphData( new List <IntPoint>(Quadrilateral), (byte[, ])RawData.Clone( ), Confidence); if (recognizedGlyph != null) { clone.RecognizedGlyph = (Glyph)recognizedGlyph.Clone( ); } if (recognizedQuadrilateral != null) { clone.RecognizedQuadrilateral = new List <IntPoint>(recognizedQuadrilateral); } return(clone); }
private PositionAndOrientation GetPositionAndOrientationFromGlyph(ExtractedGlyphData glyph) { double x = glyph.RecognizedQuadrilateral.Average(g => g.X); double y = glyph.RecognizedQuadrilateral.Average(g => g.Y); Matrix4x4 transformationMatrix = glyph.TransformationMatrix; float roll; float pitch; float yaw; transformationMatrix.ExtractYawPitchRoll(out yaw, out pitch, out roll); double xP = Math.Cos((double)yaw) * Math.Cos((double)pitch); double yP = Math.Sin((double)yaw) * Math.Cos((double)pitch); double zP = Math.Sin((double)pitch); //double angle = yaw * 180.0d / Math.PI + 180.0d; double angle = 180.0d - Math.Atan2(yP, zP) / Math.PI * 180.0d; return new PositionAndOrientation(x, y, angle); }
/// <summary> /// Try recognizing the glyph in the specified image defined by the specified quadrilateral. /// </summary> /// <param name="image">Specified image.</param> /// <param name="quadrilateral">Specified quadrilateral.</param> /// <returns>Glyph data.</returns> private ExtractedGlyphData RecognizeGlyph( UnmanagedImage image, List<IntPoint> quadrilateral ) { // extract glyph image quadrilateralTransformation.SourceQuadrilateral = quadrilateral; UnmanagedImage glyphImage = quadrilateralTransformation.Apply( image ); // otsu thresholding otsuThresholdFilter.ApplyInPlace( glyphImage ); // recognize raw glyph float confidence; byte[,] glyphValues = binaryGlyphRecognizer.Recognize( glyphImage, new Rectangle( 0, 0, glyphImage.Width, glyphImage.Height ), out confidence ); if ( confidence >= minConfidenceLevel ) { if ( ( Glyph.CheckIfGlyphHasBorder( glyphValues ) ) && ( Glyph.CheckIfEveryRowColumnHasValue( glyphValues ) ) ) { ExtractedGlyphData foundGlyph = new ExtractedGlyphData( quadrilateral, glyphValues, confidence, new Size(image.Width, image.Height)); if ( glyphDatabase != null ) { int rotation; foundGlyph.RecognizedGlyph = glyphDatabase.RecognizeGlyph( glyphValues, out rotation ); if ( rotation != -1 ) { foundGlyph.RecognizedQuadrilateral = foundGlyph.Quadrilateral; while ( rotation > 0 ) { foundGlyph.RecognizedQuadrilateral.Add( foundGlyph.RecognizedQuadrilateral[0] ); foundGlyph.RecognizedQuadrilateral.RemoveAt( 0 ); rotation -= 90; } } } return foundGlyph; } } return null; }
public TrackedGlyph(int id, ExtractedGlyphData glyph, Point position) { ID = id; Glyph = glyph; Position = position; }
// Get ID of the specified glyph // Note (todo?): glyph tracking needs to be improved since current implementation // is not reliable enough for the case when several glyphs of the same type are // found and those do not move smoothely. private int GetGlyphID(ExtractedGlyphData glyph) { int glyphID = -1; // get CoG of the provided glyph Point glyphCog = PointsCloud.GetCenterOfGravity(glyph.Quadrilateral); // distance to the closest simlar glyph kept in history float minDistance = float.MaxValue; // name of the passed specifed glyph string glyphName = (glyph.RecognizedGlyph != null) ? glyph.RecognizedGlyph.Name : string.Empty; // check all currently tracked glyphs foreach (TrackedGlyph trackedGlyph in trackedGlyphs.Values) { if (trackedGlyph.Age == 0) { // skip glyph whichs were already taken or just added continue; } if ( // recognized case - compare names ((trackedGlyph.Glyph.RecognizedGlyph != null) && (trackedGlyph.Glyph.RecognizedGlyph.Name == glyphName)) || // unrecognized case - compare raw data (Glyph.CheckForMatching(glyph.RawData, trackedGlyph.Glyph.RawData) != -1)) { float distance = glyphCog.DistanceTo(trackedGlyph.Position); if (distance < minDistance) { // get ID of the closest glyph with the same name minDistance = distance; glyphID = trackedGlyph.ID; } } } // if the glyph is further away than the maximum specified limit, // then it is no way can be treated as smooth motion, so reset ID // (TODO: should probably depend on glyph size ...) if ((glyphID != -1) && (minDistance > MaxAllowedDistance)) { glyphID = -1; } // if glyph was not found within tracked glyphs, then add new // glyph to tracker if (glyphID == -1) { glyphID = counter++; trackedGlyphs.Add(glyphID, new TrackedGlyph(glyphID, (ExtractedGlyphData)glyph.Clone( ), glyphCog)); } else { TrackedGlyph trackedGlyph = trackedGlyphs[glyphID]; if ((glyph.RecognizedGlyph != null) && (!IsCoordinatesDifferenceSignificant(glyph.RecognizedQuadrilateral, trackedGlyphs[glyphID].Glyph.RecognizedQuadrilateral))) { // correct coordinates of recognized glyphs to eliminate small noisy shaking glyph.RecognizedQuadrilateral = trackedGlyph.Glyph.RecognizedQuadrilateral; glyphCog = trackedGlyph.Position; } else { // update glyph with the latest CoG and recognized info trackedGlyph.Position = glyphCog; trackedGlyph.Glyph = (ExtractedGlyphData)glyph.Clone( ); } // reset age of the tracked glyph trackedGlyph.Age = 0; // add glyph's position to history trackedGlyph.AddMotionHistory(glyphCog); } return(glyphID); }
public TrackedGlyph( int id, ExtractedGlyphData glyph, Point position ) { ID = id; Glyph = glyph; Position = position; }
// Estimate pose for of the given glyph private void EstimateGlyphPose( ExtractedGlyphData glyph, int glyphID ) { int imageCenterX = imageSize.Width >> 1; int imageCenterY = imageSize.Height >> 1; Point[] glyphPoints = new Point[4]; Matrix3x3 rotation; Vector3 translation; // get array of points with coordinates in Cartesian system with origin in image center for ( int i = 0; i < 4; i++ ) { glyphPoints[i] = new Point( glyph.RecognizedQuadrilateral[i].X - imageCenterX, imageCenterY - glyph.RecognizedQuadrilateral[i].Y ); } // estimate pose using Coplanar POSIT algorithm posit.EstimatePose( glyphPoints, out rotation, out translation ); glyph.TransformationMatrix = Matrix4x4.CreateTranslation( translation ) * Matrix4x4.CreateFromRotation( rotation ); glyph.IsTransformationDetected = true; // check if we have previous rotation of the glyph if ( !prevRotation.ContainsKey( glyphID ) ) { // remember it if not prevRotation.Add( glyphID, posit.BestEstimatedRotation ); } else { Matrix3x3 newRotation = posit.BestEstimatedRotation; // check if best estimation is at least twice is better than the alternate // (better according to the POSIT algorithm) if ( posit.AlternateEstimationError / posit.BestEstimationError < 2 ) { // error difference is not very big, so compare both transformations with previous // and select the one which seems to be closer to it Matrix3x3 d1 = prevRotation[glyphID] - posit.BestEstimatedRotation; Matrix3x3 d2 = prevRotation[glyphID] - posit.AlternateEstimatedRotation; float e1 = d1.GetRow( 0 ).Square + d1.GetRow( 1 ).Square + d1.GetRow( 2 ).Square; float e2 = d2.GetRow( 0 ).Square + d2.GetRow( 1 ).Square + d2.GetRow( 2 ).Square; if ( e1 > e2 ) { glyph.TransformationMatrix = Matrix4x4.CreateTranslation( posit.AlternateEstimatedTranslation ) * Matrix4x4.CreateFromRotation( posit.AlternateEstimatedRotation ); newRotation = posit.AlternateEstimatedRotation; } } prevRotation[glyphID] = newRotation; } }
// Get ID of the specified glyph // Note (todo?): glyph tracking needs to be improved since current implementation // is not reliable enough for the case when several glyphs of the same type are // found and those do not move smoothely. private int GetGlyphID( ExtractedGlyphData glyph ) { int glyphID = -1; // get CoG of the provided glyph Point glyphCog = PointsCloud.GetCenterOfGravity( glyph.Quadrilateral ); // distance to the closest simlar glyph kept in history float minDistance = float.MaxValue; // name of the passed specifed glyph string glyphName = ( glyph.RecognizedGlyph != null ) ? glyph.RecognizedGlyph.Name : string.Empty; // check all currently tracked glyphs foreach ( TrackedGlyph trackedGlyph in trackedGlyphs.Values ) { if ( trackedGlyph.Age == 0 ) { // skip glyph whichs were already taken or just added continue; } if ( // recognized case - compare names ( ( trackedGlyph.Glyph.RecognizedGlyph != null ) && ( trackedGlyph.Glyph.RecognizedGlyph.Name == glyphName ) ) || // unrecognized case - compare raw data ( Glyph.CheckForMatching( glyph.RawData, trackedGlyph.Glyph.RawData ) != -1 ) ) { float distance = glyphCog.DistanceTo( trackedGlyph.Position ); if ( distance < minDistance ) { // get ID of the closest glyph with the same name minDistance = distance; glyphID = trackedGlyph.ID; } } } // if the glyph is further away than the maximum specified limit, // then it is no way can be treated as smooth motion, so reset ID // (TODO: should probably depend on glyph size ...) if ( ( glyphID != -1 ) && ( minDistance > MaxAllowedDistance ) ) { glyphID = -1; } // if glyph was not found within tracked glyphs, then add new // glyph to tracker if ( glyphID == -1 ) { glyphID = counter++; trackedGlyphs.Add( glyphID, new TrackedGlyph( glyphID, (ExtractedGlyphData) glyph.Clone( ), glyphCog ) ); } else { TrackedGlyph trackedGlyph = trackedGlyphs[glyphID]; if ( ( glyph.RecognizedGlyph != null ) && ( !IsCoordinatesDifferenceSignificant( glyph.RecognizedQuadrilateral, trackedGlyphs[glyphID].Glyph.RecognizedQuadrilateral ) ) ) { // correct coordinates of recognized glyphs to eliminate small noisy shaking glyph.RecognizedQuadrilateral = trackedGlyph.Glyph.RecognizedQuadrilateral; glyphCog = trackedGlyph.Position; } else { // update glyph with the latest CoG and recognized info trackedGlyph.Position = glyphCog; trackedGlyph.Glyph = (ExtractedGlyphData) glyph.Clone( ); } // reset age of the tracked glyph trackedGlyph.Age = 0; // add glyph's position to history trackedGlyph.AddMotionHistory( glyphCog ); } return glyphID; }
/// <summary> /// Search for glyphs in the specified image and recognize them. /// </summary> /// /// <param name="image">Image to search glyphs in.</param> /// /// <returns>Return a list of found glyphs.</returns> /// /// <remarks><para>The method does processing of the specified image and searches for glyphs in it of /// the specified <see cref="GlyphSize">size</see>. In the case if <see cref="GlyphDatabase">glyphs' database</see> /// is set, it tries to find a matching glyph in it for each found glyph in the image. If matching is found, /// then <see cref="ExtractedGlyphData.RecognizedGlyph">RecognizedGlyph</see> and /// <see cref="ExtractedGlyphData.RecognizedQuadrilateral">RecognizedQuadrilateral</see> /// properties of <see cref="ExtractedGlyphData"/> are set correspondingly.</para></remarks> /// /// <exception cref="UnsupportedImageFormatException">Pixel format of the specified image is not supported. /// It must be 8 bpp indexed or 24/32 bpp color image.</exception> /// public List <ExtractedGlyphData> FindGlyphs(UnmanagedImage image) { List <ExtractedGlyphData> extractedGlyphs = new List <ExtractedGlyphData>( ); if ((image.PixelFormat != PixelFormat.Format8bppIndexed) && (!Grayscale.CommonAlgorithms.BT709.FormatTranslations.ContainsKey(image.PixelFormat))) { throw new UnsupportedImageFormatException("Pixel format of the specified image is not supported."); } // 1 - grayscaling UnmanagedImage grayImage = null; if (image.PixelFormat == PixelFormat.Format8bppIndexed) { grayImage = image; } else { grayImage = UnmanagedImage.Create(image.Width, image.Height, PixelFormat.Format8bppIndexed); Grayscale.CommonAlgorithms.BT709.Apply(image, grayImage); } // 2 - Edge detection UnmanagedImage edgesImage = edgeDetector.Apply(grayImage); // 3 - Threshold edges thresholdFilter.ApplyInPlace(edgesImage); // 4 - Blob Counter blobCounter.ProcessImage(edgesImage); Blob[] blobs = blobCounter.GetObjectsInformation( ); // 5 - check each blob for (int i = 0, n = blobs.Length; i < n; i++) { List <IntPoint> edgePoints = blobCounter.GetBlobsEdgePoints(blobs[i]); List <IntPoint> corners = null; // does it look like a quadrilateral ? if (shapeChecker.IsQuadrilateral(edgePoints, out corners)) { // get edge points on the left and on the right side List <IntPoint> leftEdgePoints, rightEdgePoints; blobCounter.GetBlobsLeftAndRightEdges(blobs[i], out leftEdgePoints, out rightEdgePoints); // calculate average difference between pixel values from outside of the shape and from inside float diff = CalculateAverageEdgesBrightnessDifference( leftEdgePoints, rightEdgePoints, grayImage); // check average difference, which tells how much outside is lighter than inside on the average if (diff > 20) { // perform glyph recognition ExtractedGlyphData glyphData = RecognizeGlyph(grayImage, corners); if (glyphData != null) { extractedGlyphs.Add(glyphData); if (extractedGlyphs.Count >= maxNumberOfGlyphsToSearch) { break; } } } } } // dispose resources if (image.PixelFormat != PixelFormat.Format8bppIndexed) { grayImage.Dispose( ); } edgesImage.Dispose( ); return(extractedGlyphs); }
private String registerGlyph(ExtractedGlyphData glyphData) { String id = "id-" + glyphDatabase.Count; glyphDatabase.Add(new Glyph(id, glyphData.RawData)); return id; }
/// <summary> /// Clone the object by making its exact copy. /// </summary> /// /// <returns>Returns clone of the object.</returns> /// public object Clone() { ExtractedGlyphData clone = new ExtractedGlyphData( new List<IntPoint>(Quadrilateral), (byte[,])RawData.Clone(), Confidence, this.CoordinateSystemSize); if (recognizedGlyph != null) { clone.RecognizedGlyph = (Glyph)recognizedGlyph.Clone(); } if (recognizedQuadrilateral != null) { clone.RecognizedQuadrilateral = new List<IntPoint>(recognizedQuadrilateral); } return clone; }
/// <summary> /// Search for glyphs in the specified image and recognize them. /// </summary> /// /// <param name="image">Image to search glyphs in.</param> /// /// <returns>Return a list of found glyphs.</returns> /// /// <remarks><para>The method does processing of the specified image and searches for glyphs in it of /// the specified <see cref="GlyphSize">size</see>. In the case if <see cref="GlyphDatabase">glyphs' database</see> /// is set, it tries to find a matching glyph in it for each found glyph in the image. If matching is found, /// then <see cref="ExtractedGlyphData.RecognizedGlyph">RecognizedGlyph</see> and /// <see cref="ExtractedGlyphData.RecognizedQuadrilateral">RecognizedQuadrilateral</see> /// properties of <see cref="ExtractedGlyphData"/> are set correspondingly.</para></remarks> /// /// <exception cref="UnsupportedImageFormatException">Pixel format of the specified image is not supported. /// It must be 8 bpp indexed or 24/32 bpp color image.</exception> /// public List <ExtractedGlyphData> FindGlyphs(UnmanagedImage image) { List <ExtractedGlyphData> extractedGlyphs = new List <ExtractedGlyphData>( ); if ((image.PixelFormat != PixelFormat.Format8bppIndexed) && (!Grayscale.CommonAlgorithms.BT709.FormatTranslations.ContainsKey(image.PixelFormat))) { throw new UnsupportedImageFormatException("Pixel format of the specified image is not supported."); } // 1 - grayscaling UnmanagedImage grayImage = null; if (image.PixelFormat == PixelFormat.Format8bppIndexed) { grayImage = image; } else { grayImage = UnmanagedImage.Create(image.Width, image.Height, PixelFormat.Format8bppIndexed); Grayscale.CommonAlgorithms.BT709.Apply(image, grayImage); } // 2 - Edge detection UnmanagedImage edgesImage = edgeDetector.Apply(grayImage); // 3 - Threshold edges thresholdFilter.ApplyInPlace(edgesImage); // 4 - Blob Counter blobCounter.ProcessImage(edgesImage); Blob[] blobs = blobCounter.GetObjectsInformation( ); // --- HLE // create copy of source image, so we could draw on it Bitmap imageCopy = image.ToManagedImage(); System.Drawing.Imaging.BitmapData imageData = imageCopy.LockBits(new Rectangle(0, 0, image.Width, image.Height), System.Drawing.Imaging.ImageLockMode.ReadWrite, imageCopy.PixelFormat); // --- HLE // 5 - check each blob int counter = 1; // Counter -> HLE for (int i = 0, n = blobs.Length; i < n; i++) { List <IntPoint> edgePoints = blobCounter.GetBlobsEdgePoints(blobs[i]); List <IntPoint> corners = null; // does it look like a quadrilateral ? if (shapeChecker.IsQuadrilateral(edgePoints, out corners)) { // get edge points on the left and on the right side List <IntPoint> leftEdgePoints, rightEdgePoints; blobCounter.GetBlobsLeftAndRightEdges(blobs[i], out leftEdgePoints, out rightEdgePoints); // calculate average difference between pixel values from outside of the shape and from inside float diff = CalculateAverageEdgesBrightnessDifference( leftEdgePoints, rightEdgePoints, grayImage); // check average difference, which tells how much outside is lighter than inside on the average if (diff > 20) { // --- HLE // - draw on image Drawing.Polygon(imageData, corners, Color.Red); counter++; //counter -> HLE // --- HLE // perform glyph recognition ExtractedGlyphData glyphData = RecognizeGlyph(grayImage, corners, counter); if (glyphData != null) { extractedGlyphs.Add(glyphData); if (extractedGlyphs.Count >= maxNumberOfGlyphsToSearch) { break; } } } } } // --- HLE //Save out image for checking //imageCopy.Save("C:\\Users\\heve\\Documents\\_Work_PointCloud\\AAG\\ImageProcessing\\FoundObjects_dll.png", System.Drawing.Imaging.ImageFormat.Png); // --- HLE // dispose resources if (image.PixelFormat != PixelFormat.Format8bppIndexed) { grayImage.Dispose( ); } edgesImage.Dispose( ); return(extractedGlyphs); }