// algorithm plot:
// 1. Try edge match
// Find max confidence
// Try put them together
// 2. Try color match
// Find max confidence
// Try put them together
// 3. If still fail, that is from other page
// 4. If success, find the best tweak
// 5. Join the images according to the final parameters
// the return value is how many pages should be in the original images
private int bestMatchTwo()
{
int pageCount = 0;
double maxConfidence = 0.0;
ColorfulContourMap map1 = new ColorfulContourMap();
ColorfulContourMap map2 = new ColorfulContourMap();
double overlap = 0.0;
bool matched = false;
// Double-thresholding algorithm (Worked for 5-piece precisely torned paper):
// search for all unmatched pairs, calculate the confidence level ( fixed the problem with different results caused by different ordering )
// reject all matches that has confidence less than 80 ( eliminate some of the false positives)
// calculate the intersection for each matching pair
// the pair with the lowest intersection is the best
// if the intersection of the lowest pair is still greater than 5000, there are no match for the given fragments
List <MatchMetricData> matchMetricData = new List <MatchMetricData>();
// search for all unmatched pairs, calculate the confidence level
// if the max confidence level is 0, that means the remaining pieces cannot be matched by turning angle
// for this case, use the color matching algorithm
// get the potential matching edges
foreach (ColorfulContourMap cmap in Form1.contourMaps)
{
if (cmap.matched == false)
{
foreach (ColorfulContourMap cmap2 in Form1.contourMaps)
{
if (cmap2.matched == false && cmap != cmap2)
{
Match match = DNAUtil.partialMatch(cmap.extractDNA(), cmap2.extractDNA());
double confidence = match.confidence;
matchMetricData.Add(new MatchMetricData {
map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match
});
if (confidence > maxConfidence)
{
maxConfidence = confidence;
map1 = cmap;
map2 = cmap2;
}
}
}
}
}
// if there are no pieces eligible for edge matching, the metric data should have all elements with confidence level 0
// at this time, calculate the metrics for color matching
// Double-thresholding algorithm works for color matching too
if (matchMetricData.Count == 0 || matchMetricData.OrderBy(o => o.confident).Last().confident == 0)
{
foreach (ColorfulContourMap cmap in Form1.contourMaps)
{
if (cmap.matched == false)
{
foreach (ColorfulContourMap cmap2 in Form1.contourMaps)
{
if (cmap2.matched == false && cmap != cmap2)
{
Match match = DNAUtil.partialColorMatch(cmap.extractDNA(), cmap2.extractDNA());
double confidence = match.confidence;
matchMetricData.Add(new MatchMetricData {
map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match
});
if (confidence > maxConfidence)
{
maxConfidence = confidence;
map1 = cmap;
map2 = cmap2;
}
}
}
}
}
}
// if there are still no matches, the process is done
// It will consider the rest of the pieces from another page
// 1st funnel: select the most potential matching edges
matchMetricData = matchMetricData.Where(o => o.confident > Constants.MIN_CONFIDENCE).OrderBy(o => o.confident).Reverse().ToList();
Console.WriteLine(maxConfidence);
Console.WriteLine(map1.imageIndex);
Console.WriteLine(map2.imageIndex);
// calculate the intersection for each matching pair
List <MatchMetricData> data2 = new List <MatchMetricData>();
foreach (MatchMetricData m in matchMetricData)
{
Image <Bgr, byte> pic1 = Form1.sourceImages[m.map1.imageIndex].Clone();
Image <Bgr, byte> pic2 = Form1.sourceImages[m.map2.imageIndex].Clone();
Point centroid1 = new Point();
Point centroid2 = new Point();
double angle = 0.0;
Match edgeMatch = m.match;
Transformation.transformation(m.dna1, m.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
angle = angle * 180 / Math.PI;
angle = -angle;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString());
Console.WriteLine(angle);
Image <Bgr, byte> mask1 = pic1.Clone();
Image <Bgr, byte> mask2 = pic2.Clone();
Image <Bgr, byte> joined = pic1.Clone();
Image <Bgr, byte> joined_mask = joined.Clone();
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(0, 0), Form1.BKG_WHITE);
data2.Add(new MatchMetricData {
map1 = m.map1, map2 = m.map2, overlap = result.overlap, dna1 = m.dna1, dna2 = m.dna2, match = m.match, confident = m.confident
});
}
if (data2.Count == 0)
{
MessageBox.Show("No match found");
return(1); // add 1 to full image found count
}
// the pair with highest confidence with a valid intersection is the best
// 2nd funnel: select only the matches that can actually match the picture together
MatchMetricData MinOverlap = data2.OrderBy(o => o.overlap).First();
if (MinOverlap.overlap < Constants.THRESHOLD)
{
Console.WriteLine("Map1 " + MinOverlap.map1.imageIndex);
Console.WriteLine("Map2 " + MinOverlap.map2.imageIndex);
Console.WriteLine("Overlap " + MinOverlap.overlap); // correct until this point
// add the resulting image into the queue
Image <Bgr, byte> pic1 = Form1.sourceImages[MinOverlap.map1.imageIndex].Clone();
Image <Bgr, byte> pic2 = Form1.sourceImages[MinOverlap.map2.imageIndex].Clone();
Point centroid1 = new Point();
Point centroid2 = new Point();
double angle = 0.0;
Match edgeMatch = MinOverlap.match;
Transformation.transformation(MinOverlap.dna1, MinOverlap.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
// correct until this point
angle = angle * 180 / Math.PI;
angle = -angle;
Image <Bgr, byte> mask1 = pic1.Clone();
Image <Bgr, byte> mask2 = pic2.Clone();
Image <Bgr, byte> joined = pic1.Clone();
Image <Bgr, byte> joined_mask = joined.Clone();
// tweak
ReturnColorImg bestResult = new ReturnColorImg();
double minOverlap = 999999;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString()); // correct until this point
// The tweaking is messing up with the result, I will just ignore this for a moment
// the tweaking now works for 1-button matching
// but the algorithm runs extremely slow ( still way faster than gluing the pieces :)
for (int i = -2; i < 3; i++)
{
for (int j = -2; j < 3; j++)
{
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(i, j), Form1.BKG_WHITE);
if (result.overlap < minOverlap && result.success) // if the overlap is 0, that means the transformation is failed
{
bestResult = result;
minOverlap = result.overlap;
}
//DisplayImage dip = new DisplayImage(result.img, new Point(0,0), new Point(i, j), (int)result.overlap);
//dip.Show();
}
}
Form1.sourceImages.Add(bestResult.img);
MinOverlap.map1.matched = true; // correct
MinOverlap.map2.matched = true; // correct
List <ColorfulContourMap> cmap = ColorfulContourMap.getAllContourMap(bestResult.img, Form1.sourceImages.Count - 1);
Form1.contourMaps.AddRange(cmap);
this.overlap += bestResult.overlap; // correct
refresh();
MessageBox.Show("Best Match Found.");
return(0);
}
else
{
Console.WriteLine("Failed");
return(1);
}
}
private void button1_Click(object sender, EventArgs e)
{
if (radioButton1.Checked)
{
pic1 = Form1.sourceImages[map1.imageIndex].Clone();
pic2 = Form1.sourceImages[map2.imageIndex].Clone();
map1.DrawTo(pic1);
map2.DrawTo(pic2);
edgeMatch = DNAUtil.partialMatch(DNA1, DNA2);
List <Point> pointToDraw1 = new List <Point>();
List <Point> pointToDraw2 = new List <Point>();
for (int i = edgeMatch.t11; i < edgeMatch.t12; i++)
{
pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y));
}
for (int i = edgeMatch.t21; i < edgeMatch.t22; i++)
{
pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
}
pic1.DrawPolyline(pointToDraw1.ToArray(), false, new Bgr(0, 255, 0), 2);
pic2.DrawPolyline(pointToDraw2.ToArray(), false, new Bgr(0, 255, 0), 2);
pictureBox1.Image = pic1.Resize(pictureBox1.Width, pictureBox1.Height, INTER.CV_INTER_LINEAR).ToBitmap();
pictureBox2.Image = pic2.Resize(pictureBox2.Width, pictureBox2.Height, INTER.CV_INTER_LINEAR).ToBitmap();
}
else
{
pic1 = Form1.sourceImages[map1.imageIndex].Clone();
pic2 = Form1.sourceImages[map2.imageIndex].Clone();
map1.DrawTo(pic1);
map2.DrawTo(pic2);
// add color matching algorithm here
edgeMatch = DNAUtil.partialColorMatch(DNA1, DNA2);
List <Point> pointToDraw1 = new List <Point>();
List <Point> pointToDraw2 = new List <Point>();
if (edgeMatch.t11 > edgeMatch.t12)
{
/*for(int i=edgeMatch.t11; i < DNA1.Count; i++)
* {
* pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y));
* }
* for(int i=0; i<=edgeMatch.t12; i++)
* {
* pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y));
* }*/
for (int i = edgeMatch.t12; i < edgeMatch.t11; i++)
{
pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y));
}
}
else
{
for (int i = edgeMatch.t11; i < edgeMatch.t12; i++)
{
pointToDraw1.Add(new Point((int)DNA1[i].x, (int)DNA1[i].y));
}
}
// the piece 2 should draw reversely
/*if (edgeMatch.t21 > edgeMatch.t22)
* {
* for (int i = edgeMatch.t22; i > -1; i--)
* {
* pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
* }
* for (int i = DNA2.Count-1; i >= edgeMatch.t21; i--)
* {
* pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
* }
*
* }
* else
* {
*
* for (int i = edgeMatch.t22; i >= edgeMatch.t21; i--)
* {
* pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
*
* }
* }*/
if (edgeMatch.t21 > edgeMatch.t22)
{
/*for (int i = edgeMatch.t21; i< DNA2.Count; i++)
* {
* pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
* }
* for (int i = 0; i <= edgeMatch.t22; i++)
* {
* pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
* }*/
for (int i = edgeMatch.t22; i < edgeMatch.t21; i++)
{
pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
}
}
else
{
for (int i = edgeMatch.t21; i < edgeMatch.t22; i++)
{
pointToDraw2.Add(new Point((int)DNA2[i].x, (int)DNA2[i].y));
}
}
pic1.DrawPolyline(pointToDraw1.ToArray(), false, new Bgr(0, 255, 0), 2);
pic2.DrawPolyline(pointToDraw2.ToArray(), false, new Bgr(0, 255, 0), 2);
pictureBox1.Image = pic1.Resize(pictureBox1.Width, pictureBox1.Height, INTER.CV_INTER_LINEAR).ToBitmap();
pictureBox2.Image = pic2.Resize(pictureBox2.Width, pictureBox2.Height, INTER.CV_INTER_LINEAR).ToBitmap();
}
}
