public AlignedResult CreateAlignedSecondImageKeypoints(SKBitmap firstImage, SKBitmap secondImage, bool discardTransX, AlignmentSettings settings, bool keystoneRightOnFirst) { #if __NO_EMGU__ return(null); #endif var result = new AlignedResult(); var detector = new ORBDetector(); const ImreadModes READ_MODE = ImreadModes.Color; var mat1 = new Mat(); var descriptors1 = new Mat(); var allKeyPointsVector1 = new VectorOfKeyPoint(); CvInvoke.Imdecode(GetBytes(firstImage, 1), READ_MODE, mat1); detector.DetectAndCompute(mat1, null, allKeyPointsVector1, descriptors1, false); var mat2 = new Mat(); var descriptors2 = new Mat(); var allKeyPointsVector2 = new VectorOfKeyPoint(); CvInvoke.Imdecode(GetBytes(secondImage, 1), READ_MODE, mat2); detector.DetectAndCompute(mat2, null, allKeyPointsVector2, descriptors2, false); const double THRESHOLD_PROPORTION = 1 / 4d; var thresholdDistance = Math.Sqrt(Math.Pow(firstImage.Width, 2) + Math.Pow(firstImage.Height, 2)) * THRESHOLD_PROPORTION; var distanceThresholdMask = new Mat(allKeyPointsVector2.Size, allKeyPointsVector1.Size, DepthType.Cv8U, 1); if (!settings.UseCrossCheck) { unsafe { var maskPtr = (byte *)distanceThresholdMask.DataPointer.ToPointer(); for (var i = 0; i < allKeyPointsVector2.Size; i++) { var keyPoint2 = allKeyPointsVector2[i]; for (var j = 0; j < allKeyPointsVector1.Size; j++) { var keyPoint1 = allKeyPointsVector1[j]; var physicalDistance = CalculatePhysicalDistanceBetweenPoints(keyPoint2.Point, keyPoint1.Point); if (physicalDistance < thresholdDistance) { *maskPtr = 255; } else { *maskPtr = 0; } maskPtr++; } } } } var vectorOfMatches = new VectorOfVectorOfDMatch(); var matcher = new BFMatcher(DistanceType.Hamming, settings.UseCrossCheck); matcher.Add(descriptors1); matcher.KnnMatch(descriptors2, vectorOfMatches, settings.UseCrossCheck ? 1 : 2, settings.UseCrossCheck ? new VectorOfMat() : new VectorOfMat(distanceThresholdMask)); var goodMatches = new List <MDMatch>(); for (var i = 0; i < vectorOfMatches.Size; i++) { if (vectorOfMatches[i].Size == 0) { continue; } if (vectorOfMatches[i].Size == 1 || (vectorOfMatches[i][0].Distance < 0.75 * vectorOfMatches[i][1].Distance)) //make sure matches are unique { goodMatches.Add(vectorOfMatches[i][0]); } } if (goodMatches.Count < settings.MinimumKeypoints) { return(null); } var pairedPoints = new List <PointForCleaning>(); for (var ii = 0; ii < goodMatches.Count; ii++) { var keyPoint1 = allKeyPointsVector1[goodMatches[ii].TrainIdx]; var keyPoint2 = allKeyPointsVector2[goodMatches[ii].QueryIdx]; pairedPoints.Add(new PointForCleaning { KeyPoint1 = keyPoint1, KeyPoint2 = keyPoint2, Data = new KeyPointOutlierDetectorData { Distance = (float)CalculatePhysicalDistanceBetweenPoints(keyPoint1.Point, keyPoint2.Point), Slope = (keyPoint2.Point.Y - keyPoint1.Point.Y) / (keyPoint2.Point.X - keyPoint1.Point.X) }, Match = new MDMatch { Distance = goodMatches[ii].Distance, ImgIdx = goodMatches[ii].ImgIdx, QueryIdx = ii, TrainIdx = ii } }); } if (settings.DrawKeypointMatches) { result.DirtyMatchesCount = pairedPoints.Count; result.DrawnDirtyMatches = DrawMatches(firstImage, secondImage, pairedPoints); } if (settings.DiscardOutliersByDistance || settings.DiscardOutliersBySlope) { //Debug.WriteLine("DIRTY POINTS START (ham,dist,slope,ydiff), count: " + pairedPoints.Count); //foreach (var pointForCleaning in pairedPoints) //{ // Debug.WriteLine(pointForCleaning.Match.Distance + "," + pointForCleaning.Data.Distance + "," + pointForCleaning.Data.Slope + "," + Math.Abs(pointForCleaning.KeyPoint1.Point.Y - pointForCleaning.KeyPoint2.Point.Y)); //} //Debug.WriteLine("DIRTY PAIRS:"); //PrintPairs(pairedPoints); if (settings.DiscardOutliersByDistance) { // reject distances and slopes more than some number of standard deviations from the median var medianDistance = pairedPoints.OrderBy(p => p.Data.Distance).ElementAt(pairedPoints.Count / 2).Data.Distance; var distanceStdDev = CalcStandardDeviation(pairedPoints.Select(p => p.Data.Distance).ToArray()); pairedPoints = pairedPoints.Where(p => Math.Abs(p.Data.Distance - medianDistance) < Math.Abs(distanceStdDev * (settings.KeypointOutlierThresholdTenths / 10d))).ToList(); //Debug.WriteLine("Median Distance: " + medianDistance); //Debug.WriteLine("Distance Cleaned Points count: " + pairedPoints.Count); } if (settings.DiscardOutliersBySlope) { var validSlopes = pairedPoints.Where(p => !float.IsNaN(p.Data.Slope) && float.IsFinite(p.Data.Slope)).ToArray(); var medianSlope = validSlopes.OrderBy(p => p.Data.Slope).ElementAt(validSlopes.Length / 2).Data.Slope; var slopeStdDev = CalcStandardDeviation(validSlopes.Select(p => p.Data.Slope).ToArray()); pairedPoints = validSlopes.Where(p => Math.Abs(p.Data.Slope - medianSlope) < Math.Abs(slopeStdDev * (settings.KeypointOutlierThresholdTenths / 10d))).ToList(); //Debug.WriteLine("Median Slope: " + medianSlope); //Debug.WriteLine("Slope Cleaned Points count: " + pairedPoints.Count); } //Debug.WriteLine("CLEAN POINTS START (ham,dist,slope,ydiff), count: " + pairedPoints.Count); //foreach (var pointForCleaning in pairedPoints) //{ // Debug.WriteLine(pointForCleaning.Match.Distance + "," + pointForCleaning.Data.Distance + "," + pointForCleaning.Data.Slope + "," + Math.Abs(pointForCleaning.KeyPoint1.Point.Y - pointForCleaning.KeyPoint2.Point.Y)); //} //Debug.WriteLine("CLEANED PAIRS:"); //PrintPairs(pairedPoints); for (var ii = 0; ii < pairedPoints.Count; ii++) { var oldMatch = pairedPoints[ii].Match; pairedPoints[ii].Match = new MDMatch { Distance = oldMatch.Distance, ImgIdx = oldMatch.ImgIdx, QueryIdx = ii, TrainIdx = ii }; } if (settings.DrawKeypointMatches) { result.CleanMatchesCount = pairedPoints.Count; result.DrawnCleanMatches = DrawMatches(firstImage, secondImage, pairedPoints); } } var points1 = pairedPoints.Select(p => new SKPoint(p.KeyPoint1.Point.X, p.KeyPoint1.Point.Y)).ToArray(); var points2 = pairedPoints.Select(p => new SKPoint(p.KeyPoint2.Point.X, p.KeyPoint2.Point.Y)).ToArray(); var translation1 = FindVerticalTranslation(points1, points2, secondImage); var translated1 = SKMatrix.MakeTranslation(0, translation1); points2 = translated1.MapPoints(points2); var rotation1 = FindRotation(points1, points2, secondImage); var rotated1 = SKMatrix.MakeRotation(rotation1, secondImage.Width / 2f, secondImage.Height / 2f); points2 = rotated1.MapPoints(points2); var zoom1 = FindZoom(points1, points2, secondImage); var zoomed1 = SKMatrix.MakeScale(zoom1, zoom1, secondImage.Width / 2f, secondImage.Height / 2f); points2 = zoomed1.MapPoints(points2); var translation2 = FindVerticalTranslation(points1, points2, secondImage); var translated2 = SKMatrix.MakeTranslation(0, translation2); points2 = translated2.MapPoints(points2); var rotation2 = FindRotation(points1, points2, secondImage); var rotated2 = SKMatrix.MakeRotation(rotation2, secondImage.Width / 2f, secondImage.Height / 2f); points2 = rotated2.MapPoints(points2); var zoom2 = FindZoom(points1, points2, secondImage); var zoomed2 = SKMatrix.MakeScale(zoom2, zoom2, secondImage.Width / 2f, secondImage.Height / 2f); points2 = zoomed2.MapPoints(points2); var translation3 = FindVerticalTranslation(points1, points2, secondImage); var translated3 = SKMatrix.MakeTranslation(0, translation3); points2 = translated3.MapPoints(points2); var rotation3 = FindRotation(points1, points2, secondImage); var rotated3 = SKMatrix.MakeRotation(rotation3, secondImage.Width / 2f, secondImage.Height / 2f); points2 = rotated3.MapPoints(points2); var zoom3 = FindZoom(points1, points2, secondImage); var zoomed3 = SKMatrix.MakeScale(zoom3, zoom3, secondImage.Width / 2f, secondImage.Height / 2f); points2 = zoomed3.MapPoints(points2); var keystoned1 = SKMatrix.MakeIdentity(); var keystoned2 = SKMatrix.MakeIdentity(); if (settings.DoKeystoneCorrection) { keystoned1 = FindTaper(points2, points1, secondImage, keystoneRightOnFirst); points1 = keystoned1.MapPoints(points1); keystoned2 = FindTaper(points1, points2, secondImage, !keystoneRightOnFirst); points2 = keystoned2.MapPoints(points2); } var horizontaled = SKMatrix.MakeIdentity(); if (!discardTransX) { var horizontalAdj = FindHorizontalTranslation(points1, points2, secondImage); horizontaled = SKMatrix.MakeTranslation(horizontalAdj, 0); points2 = horizontaled.MapPoints(points2); } var tempMatrix1 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix1, translated1, rotated1); var tempMatrix2 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix2, tempMatrix1, zoomed1); var tempMatrix3 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix3, tempMatrix2, translated2); var tempMatrix4 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix4, tempMatrix3, rotated2); var tempMatrix5 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix5, tempMatrix4, zoomed2); var tempMatrix6 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix6, tempMatrix5, translated3); var tempMatrix7 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix7, tempMatrix6, rotated3); var tempMatrix8 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix8, tempMatrix7, zoomed3); var tempMatrix9 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix9, tempMatrix8, keystoned2); var tempMatrix10 = new SKMatrix(); SKMatrix.Concat(ref tempMatrix10, tempMatrix9, horizontaled); var finalMatrix = tempMatrix10; result.TransformMatrix2 = finalMatrix; var alignedImage2 = new SKBitmap(secondImage.Width, secondImage.Height); using (var canvas = new SKCanvas(alignedImage2)) { canvas.SetMatrix(finalMatrix); canvas.DrawBitmap(secondImage, 0, 0); } result.AlignedBitmap2 = alignedImage2; result.TransformMatrix1 = keystoned1; var alignedImage1 = new SKBitmap(firstImage.Width, firstImage.Height); using (var canvas = new SKCanvas(alignedImage1)) { canvas.SetMatrix(keystoned1); canvas.DrawBitmap(firstImage, 0, 0); } result.AlignedBitmap1 = alignedImage1; return(result); }
public AlignedResult CreateAlignedSecondImageEcc(SKBitmap firstImage, SKBitmap secondImage, bool discardTransX, AlignmentSettings settings) { #if __NO_EMGU__ return(null); #endif var topDownsizeFactor = settings.EccDownsizePercentage / 100f; var eccs = new List <double>(); using var mat1 = new Mat(); using var mat2 = new Mat(); using var warpMatrix = Mat.Eye(2, 3, DepthType.Cv32F, 1); var termCriteria = new MCvTermCriteria(settings.EccIterations, Math.Pow(10, -settings.EccEpsilonLevel)); for (var ii = settings.EccPyramidLayers - 1; ii >= 0; ii--) { var downsize = topDownsizeFactor / Math.Pow(2, ii); CvInvoke.Imdecode(GetBytes(firstImage, downsize), ImreadModes.Grayscale, mat1); CvInvoke.Imdecode(GetBytes(secondImage, downsize), ImreadModes.Grayscale, mat2); try { var ecc = CvInvoke.FindTransformECC(mat2, mat1, warpMatrix, MotionType.Euclidean, termCriteria); eccs.Add(ecc); } catch (CvException e) { if (e.Status == (int)ErrorCodes.StsNoConv) { return(null); } throw; } if (warpMatrix.IsEmpty) { return(null); } unsafe { var ptr = (float *)warpMatrix.DataPointer.ToPointer(); //ScaleX ptr++; //SkewX ptr++; //TransX *ptr *= 2; //scale up the shifting ptr++; //SkewY ptr++; //ScaleY ptr++; //TransY *ptr *= 2; //scale up the shifting } } var lastUpscaleFactor = 1 / (2 * topDownsizeFactor); ScaleUpCvMatOfFloats(warpMatrix, lastUpscaleFactor); if (eccs.Last() * 100 < settings.EccThresholdPercentage) { return(null); } var skMatrix = ConvertCvMatOfFloatsToSkMatrix(warpMatrix, discardTransX); var result = new AlignedResult { TransformMatrix2 = skMatrix }; using var alignedMat = new Mat(); using var fullSizeColorSecondMat = new Mat(); CvInvoke.Imdecode(GetBytes(secondImage, 1), ImreadModes.Color, fullSizeColorSecondMat); CvInvoke.WarpAffine(fullSizeColorSecondMat, alignedMat, warpMatrix, fullSizeColorSecondMat.Size); #if __IOS__ result.AlignedBitmap2 = alignedMat.ToCGImage().ToSKBitmap(); #elif __ANDROID__ result.AlignedBitmap2 = alignedMat.ToBitmap().ToSKBitmap(); #endif return(result); }
public AlignedResult CreateAlignedSecondImage(SKBitmap firstImage, SKBitmap secondImage, int downsizePercentage, int iterations, int epsilonLevel, int eccCutoff, bool discardTransX) { var downsizeFactor = downsizePercentage / 100f; using (var downsizedFirstGrayMat = new Mat()) using (var downsizedSecondGrayMat = new Mat()) { CvInvoke.Imdecode(GetBytes(firstImage, downsizeFactor), ImreadModes.Grayscale, downsizedFirstGrayMat); CvInvoke.Imdecode(GetBytes(secondImage, downsizeFactor), ImreadModes.Grayscale, downsizedSecondGrayMat); using (var transformMatrix = Mat.Eye(2, 3, DepthType.Cv32F, 1)) { var criteria = new MCvTermCriteria(iterations, Math.Pow(10, -epsilonLevel)); double ecc; try { ecc = CvInvoke.FindTransformECC(downsizedSecondGrayMat, downsizedFirstGrayMat, transformMatrix, MotionType.Euclidean, criteria); } catch (CvException e) { if (e.Status == (int)ErrorCodes.StsNoConv) { return(null); } throw; } if (transformMatrix.IsEmpty || ecc * 100 < eccCutoff) { return(null); } var skMatrix = SKMatrix.MakeIdentity(); unsafe { //|ScaleX SkewX TransX| //|SkewY ScaleY TransY| //|Persp0 Persp1 Persp2| (row omitted for affine) var ptr = (float *)transformMatrix.DataPointer.ToPointer(); //ScaleX skMatrix.ScaleX = *ptr; ptr++; //SkewX skMatrix.SkewX = *ptr; ptr++; //TransX if (!discardTransX) { *ptr /= downsizeFactor; //scale up the shifting } else { *ptr = 0; //discard for side-by-side } skMatrix.TransX = *ptr; ptr++; //SkewY skMatrix.SkewY = *ptr; ptr++; //ScaleY skMatrix.ScaleY = *ptr; ptr++; //TransY *ptr /= downsizeFactor; //scale up the shifting skMatrix.TransY = *ptr; } var result = new AlignedResult { TransformMatrix = skMatrix }; using (var alignedMat = new Mat()) using (var fullSizeColorSecondMat = new Mat()) { CvInvoke.Imdecode(GetBytes(secondImage, 1), ImreadModes.Color, fullSizeColorSecondMat); CvInvoke.WarpAffine(fullSizeColorSecondMat, alignedMat, transformMatrix, fullSizeColorSecondMat.Size); #if __IOS__ result.AlignedBitmap = alignedMat.ToCGImage().ToSKBitmap(); #elif __ANDROID__ result.AlignedBitmap = alignedMat.ToBitmap().ToSKBitmap(); #endif return(result); } } } }