// algorithm plot: // 1. Try edge match // Find max confidence // Try put them together // 2. Try color match // Find max confidence // Try put them together // 3. If still fail, that is from other page // 4. If success, find the best tweak // 5. Join the images according to the final parameters // the return value is how many pages should be in the original images private int bestMatchTwo() { int pageCount = 0; double maxConfidence = 0.0; ColorfulContourMap map1 = new ColorfulContourMap(); ColorfulContourMap map2 = new ColorfulContourMap(); double overlap = 0.0; bool matched = false; // Double-thresholding algorithm (Worked for 5-piece precisely torned paper): // search for all unmatched pairs, calculate the confidence level ( fixed the problem with different results caused by different ordering ) // reject all matches that has confidence less than 80 ( eliminate some of the false positives) // calculate the intersection for each matching pair // the pair with the lowest intersection is the best // if the intersection of the lowest pair is still greater than 5000, there are no match for the given fragments List <MatchMetricData> matchMetricData = new List <MatchMetricData>(); // search for all unmatched pairs, calculate the confidence level // if the max confidence level is 0, that means the remaining pieces cannot be matched by turning angle // for this case, use the color matching algorithm // get the potential matching edges foreach (ColorfulContourMap cmap in Form1.contourMaps) { if (cmap.matched == false) { foreach (ColorfulContourMap cmap2 in Form1.contourMaps) { if (cmap2.matched == false && cmap != cmap2) { Match match = DNAUtil.partialMatch(cmap.extractDNA(), cmap2.extractDNA()); double confidence = match.confidence; matchMetricData.Add(new MatchMetricData { map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match }); if (confidence > maxConfidence) { maxConfidence = confidence; map1 = cmap; map2 = cmap2; } } } } } // if there are no pieces eligible for edge matching, the metric data should have all elements with confidence level 0 // at this time, calculate the metrics for color matching // Double-thresholding algorithm works for color matching too if (matchMetricData.Count == 0 || matchMetricData.OrderBy(o => o.confident).Last().confident == 0) { foreach (ColorfulContourMap cmap in Form1.contourMaps) { if (cmap.matched == false) { foreach (ColorfulContourMap cmap2 in Form1.contourMaps) { if (cmap2.matched == false && cmap != cmap2) { Match match = DNAUtil.partialColorMatch(cmap.extractDNA(), cmap2.extractDNA()); double confidence = match.confidence; matchMetricData.Add(new MatchMetricData { map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match }); if (confidence > maxConfidence) { maxConfidence = confidence; map1 = cmap; map2 = cmap2; } } } } } } // if there are still no matches, the process is done // It will consider the rest of the pieces from another page // 1st funnel: select the most potential matching edges matchMetricData = matchMetricData.Where(o => o.confident > Constants.MIN_CONFIDENCE).OrderBy(o => o.confident).Reverse().ToList(); Console.WriteLine(maxConfidence); Console.WriteLine(map1.imageIndex); Console.WriteLine(map2.imageIndex); // calculate the intersection for each matching pair List <MatchMetricData> data2 = new List <MatchMetricData>(); foreach (MatchMetricData m in matchMetricData) { Image <Bgr, byte> pic1 = Form1.sourceImages[m.map1.imageIndex].Clone(); Image <Bgr, byte> pic2 = Form1.sourceImages[m.map2.imageIndex].Clone(); Point centroid1 = new Point(); Point centroid2 = new Point(); double angle = 0.0; Match edgeMatch = m.match; Transformation.transformation(m.dna1, m.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle); angle = angle * 180 / Math.PI; angle = -angle; Console.WriteLine(centroid1.ToString()); Console.WriteLine(centroid2.ToString()); Console.WriteLine(angle); Image <Bgr, byte> mask1 = pic1.Clone(); Image <Bgr, byte> mask2 = pic2.Clone(); Image <Bgr, byte> joined = pic1.Clone(); Image <Bgr, byte> joined_mask = joined.Clone(); ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(0, 0), Form1.BKG_WHITE); data2.Add(new MatchMetricData { map1 = m.map1, map2 = m.map2, overlap = result.overlap, dna1 = m.dna1, dna2 = m.dna2, match = m.match, confident = m.confident }); } if (data2.Count == 0) { MessageBox.Show("No match found"); return(1); // add 1 to full image found count } // the pair with highest confidence with a valid intersection is the best // 2nd funnel: select only the matches that can actually match the picture together MatchMetricData MinOverlap = data2.OrderBy(o => o.overlap).First(); if (MinOverlap.overlap < Constants.THRESHOLD) { Console.WriteLine("Map1 " + MinOverlap.map1.imageIndex); Console.WriteLine("Map2 " + MinOverlap.map2.imageIndex); Console.WriteLine("Overlap " + MinOverlap.overlap); // correct until this point // add the resulting image into the queue Image <Bgr, byte> pic1 = Form1.sourceImages[MinOverlap.map1.imageIndex].Clone(); Image <Bgr, byte> pic2 = Form1.sourceImages[MinOverlap.map2.imageIndex].Clone(); Point centroid1 = new Point(); Point centroid2 = new Point(); double angle = 0.0; Match edgeMatch = MinOverlap.match; Transformation.transformation(MinOverlap.dna1, MinOverlap.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle); // correct until this point angle = angle * 180 / Math.PI; angle = -angle; Image <Bgr, byte> mask1 = pic1.Clone(); Image <Bgr, byte> mask2 = pic2.Clone(); Image <Bgr, byte> joined = pic1.Clone(); Image <Bgr, byte> joined_mask = joined.Clone(); // tweak ReturnColorImg bestResult = new ReturnColorImg(); double minOverlap = 999999; Console.WriteLine(centroid1.ToString()); Console.WriteLine(centroid2.ToString()); // correct until this point // The tweaking is messing up with the result, I will just ignore this for a moment // the tweaking now works for 1-button matching // but the algorithm runs extremely slow ( still way faster than gluing the pieces :) for (int i = -2; i < 3; i++) { for (int j = -2; j < 3; j++) { ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(i, j), Form1.BKG_WHITE); if (result.overlap < minOverlap && result.success) // if the overlap is 0, that means the transformation is failed { bestResult = result; minOverlap = result.overlap; } //DisplayImage dip = new DisplayImage(result.img, new Point(0,0), new Point(i, j), (int)result.overlap); //dip.Show(); } } Form1.sourceImages.Add(bestResult.img); MinOverlap.map1.matched = true; // correct MinOverlap.map2.matched = true; // correct List <ColorfulContourMap> cmap = ColorfulContourMap.getAllContourMap(bestResult.img, Form1.sourceImages.Count - 1); Form1.contourMaps.AddRange(cmap); this.overlap += bestResult.overlap; // correct refresh(); MessageBox.Show("Best Match Found."); return(0); } else { Console.WriteLine("Failed"); return(1); } }
private void button1_Click(object sender, EventArgs e) { if (radioButton1.Checked) { pic1 = Form1.sourceImages[map1.imageIndex].Clone(); pic2 = Form1.sourceImages[map2.imageIndex].Clone(); map1.DrawTo(pic1); map2.DrawTo(pic2); edgeMatch = DNAUtil.partialMatch(DNA1, DNA2); List <Point> pointToDraw1 = new List <Point>(); List <Point> pointToDraw2 = new List <Point>(); for (int i = edgeMatch.t11; i < edgeMatch.t12; i++) { pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y)); } for (int i = edgeMatch.t21; i < edgeMatch.t22; i++) { pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); } pic1.DrawPolyline(pointToDraw1.ToArray(), false, new Bgr(0, 255, 0), 2); pic2.DrawPolyline(pointToDraw2.ToArray(), false, new Bgr(0, 255, 0), 2); pictureBox1.Image = pic1.Resize(pictureBox1.Width, pictureBox1.Height, INTER.CV_INTER_LINEAR).ToBitmap(); pictureBox2.Image = pic2.Resize(pictureBox2.Width, pictureBox2.Height, INTER.CV_INTER_LINEAR).ToBitmap(); } else { pic1 = Form1.sourceImages[map1.imageIndex].Clone(); pic2 = Form1.sourceImages[map2.imageIndex].Clone(); map1.DrawTo(pic1); map2.DrawTo(pic2); // add color matching algorithm here edgeMatch = DNAUtil.partialColorMatch(DNA1, DNA2); List <Point> pointToDraw1 = new List <Point>(); List <Point> pointToDraw2 = new List <Point>(); if (edgeMatch.t11 > edgeMatch.t12) { /*for(int i=edgeMatch.t11; i < DNA1.Count; i++) * { * pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y)); * } * for(int i=0; i<=edgeMatch.t12; i++) * { * pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y)); * }*/ for (int i = edgeMatch.t12; i < edgeMatch.t11; i++) { pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y)); } } else { for (int i = edgeMatch.t11; i < edgeMatch.t12; i++) { pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y)); } } // the piece 2 should draw reversely /*if (edgeMatch.t21 > edgeMatch.t22) * { * for (int i = edgeMatch.t22; i > -1; i--) * { * pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); * } * for (int i = DNA2.Count-1; i >= edgeMatch.t21; i--) * { * pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); * } * * } * else * { * * for (int i = edgeMatch.t22; i >= edgeMatch.t21; i--) * { * pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); * * } * }*/ if (edgeMatch.t21 > edgeMatch.t22) { /*for (int i = edgeMatch.t21; i< DNA2.Count; i++) * { * pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); * } * for (int i = 0; i <= edgeMatch.t22; i++) * { * pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); * }*/ for (int i = edgeMatch.t22; i < edgeMatch.t21; i++) { pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); } } else { for (int i = edgeMatch.t21; i < edgeMatch.t22; i++) { pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y)); } } pic1.DrawPolyline(pointToDraw1.ToArray(), false, new Bgr(0, 255, 0), 2); pic2.DrawPolyline(pointToDraw2.ToArray(), false, new Bgr(0, 255, 0), 2); pictureBox1.Image = pic1.Resize(pictureBox1.Width, pictureBox1.Height, INTER.CV_INTER_LINEAR).ToBitmap(); pictureBox2.Image = pic2.Resize(pictureBox2.Width, pictureBox2.Height, INTER.CV_INTER_LINEAR).ToBitmap(); } }