private Point pointTransform(Point p, ColorfulContourMap map)
{
Point result = new Point();
double requiredWidth = Form1.sourceImages[map.imageIndex].Width;
double requiredHeight = Form1.sourceImages[map.imageIndex].Height;
result.X = (int)((p.X + 0.0) / pictureBox1.Width * requiredWidth);
result.Y = (int)((p.Y + 0.0) / pictureBox1.Height * requiredHeight);
return(result);
}
public ColorfulContourMap Clone()
{
ColorfulContourMap cmap = new ColorfulContourMap();
cmap._points = _points.ToList();
cmap._polyPoints = _polyPoints.ToList();
cmap.Width = Width;
cmap.Height = Height;
cmap.Center = Center;
return(cmap);
}
private void button4_Click(object sender, EventArgs e)
{
Form1.sourceImages.Add(joined);
map1.matched = true;
map2.matched = true;
List <ColorfulContourMap> cmap = ColorfulContourMap.getAllContourMap(joined, Form1.sourceImages.Count - 1);
Form1.contourMaps.AddRange(cmap);
Close();
}
// static factory methods
// getting the contour with the max area in a picture
// the extraction algorithm is from the C++ code Line 440-534
public static ColorfulContourMap getMaxContourMap(Image <Bgr, byte> input, int index)
{
ColorfulContourMap result = new ColorfulContourMap();
Image <Gray, byte> gray = input.Convert <Gray, byte>();
gray = gray.SmoothGaussian(3).ThresholdBinaryInv(new Gray(245), new Gray(255)).MorphologyEx(null, CV_MORPH_OP.CV_MOP_CLOSE, 2);
List <Point> pointList = new List <Point>();
List <Point> polyPointList = new List <Point>();
List <ColorfulPoint> cps = new List <ColorfulPoint>();
List <ColorfulPoint> pcps = new List <ColorfulPoint>();
using (MemStorage storage1 = new MemStorage())
{
Image <Gray, Byte> temp = gray.Clone();
Contour <Point> contour = temp.FindContours(CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_NONE, RETR_TYPE.CV_RETR_EXTERNAL);
double area = Math.Abs(contour.Area);
Contour <Point> maxArea = contour;
contour = contour.HNext;
for (; contour != null; contour = contour.HNext)
{
double nextArea = Math.Abs(contour.Area);
if (area < nextArea)
{
area = nextArea;
maxArea = contour;
}
}
Contour <Point> poly = maxArea.ApproxPoly(1.0, storage1);
pointList = maxArea.ToList();
polyPointList = poly.ToList();
foreach (Point p in pointList)
{
ColorfulPoint cp = new ColorfulPoint {
X = p.X, Y = p.Y, color = extractPointColor(p, input)
};
cps.Add(cp);
}
foreach (Point p in polyPointList)
{
ColorfulPoint cp = new ColorfulPoint {
X = p.X, Y = p.Y, color = extractPointColor(p, input)
};
pcps.Add(cp);
}
result = new ColorfulContourMap(cps, pcps, index);
result.matched = false;
}
return(result);
}
private void button4_Click(object sender, EventArgs e)
{
if (Form1.sourceImages.Count != 0)
{
DisplayBestMatch bestMatchView = new DisplayBestMatch();
QueueView qv = new QueueView();
// extract the contour maps, send the result into queueview
int index = 0;
foreach (Image <Bgr, byte> image in Form1.sourceImages)
{
List <ColorfulContourMap> cmap;
if (blackSelect.Checked == true)
{
cmap = ColorfulContourMap.getAllContourMap(image, index, 0);
}
else
{
cmap = ColorfulContourMap.getAllContourMap(image, index, 1);
}
Form1.contourMaps.AddRange(cmap);
index++;
}
if (blackSelect.Checked)
{
Form1.BKG_WHITE = true;
}
else
{
Form1.BKG_WHITE = false;
}
Hide();
bestMatchView.Show();
qv.Show();
}
else
{
MessageBox.Show("Please input at least one image.");
}
}
// algorithm plot:
// 1. Try edge match
// Find max confidence
// Try put them together
// 2. Try color match
// Find max confidence
// Try put them together
// 3. If still fail, that is from other page
// 4. If success, find the best tweak
// 5. Join the images according to the final parameters
// the return value is how many pages should be in the original images
private int bestMatchTwo()
{
int pageCount = 0;
double maxConfidence = 0.0;
ColorfulContourMap map1 = new ColorfulContourMap();
ColorfulContourMap map2 = new ColorfulContourMap();
double overlap = 0.0;
bool matched = false;
// Double-thresholding algorithm (Worked for 5-piece precisely torned paper):
// search for all unmatched pairs, calculate the confidence level ( fixed the problem with different results caused by different ordering )
// reject all matches that has confidence less than 80 ( eliminate some of the false positives)
// calculate the intersection for each matching pair
// the pair with the lowest intersection is the best
// if the intersection of the lowest pair is still greater than 5000, there are no match for the given fragments
List <MatchMetricData> matchMetricData = new List <MatchMetricData>();
// search for all unmatched pairs, calculate the confidence level
// if the max confidence level is 0, that means the remaining pieces cannot be matched by turning angle
// for this case, use the color matching algorithm
// get the potential matching edges
foreach (ColorfulContourMap cmap in Form1.contourMaps)
{
if (cmap.matched == false)
{
foreach (ColorfulContourMap cmap2 in Form1.contourMaps)
{
if (cmap2.matched == false && cmap != cmap2)
{
Match match = DNAUtil.partialMatch(cmap.extractDNA(), cmap2.extractDNA());
double confidence = match.confidence;
matchMetricData.Add(new MatchMetricData {
map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match
});
if (confidence > maxConfidence)
{
maxConfidence = confidence;
map1 = cmap;
map2 = cmap2;
}
}
}
}
}
// if there are no pieces eligible for edge matching, the metric data should have all elements with confidence level 0
// at this time, calculate the metrics for color matching
// Double-thresholding algorithm works for color matching too
if (matchMetricData.Count == 0 || matchMetricData.OrderBy(o => o.confident).Last().confident == 0)
{
foreach (ColorfulContourMap cmap in Form1.contourMaps)
{
if (cmap.matched == false)
{
foreach (ColorfulContourMap cmap2 in Form1.contourMaps)
{
if (cmap2.matched == false && cmap != cmap2)
{
Match match = DNAUtil.partialColorMatch(cmap.extractDNA(), cmap2.extractDNA());
double confidence = match.confidence;
matchMetricData.Add(new MatchMetricData {
map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match
});
if (confidence > maxConfidence)
{
maxConfidence = confidence;
map1 = cmap;
map2 = cmap2;
}
}
}
}
}
}
// if there are still no matches, the process is done
// It will consider the rest of the pieces from another page
// 1st funnel: select the most potential matching edges
matchMetricData = matchMetricData.Where(o => o.confident > Constants.MIN_CONFIDENCE).OrderBy(o => o.confident).Reverse().ToList();
Console.WriteLine(maxConfidence);
Console.WriteLine(map1.imageIndex);
Console.WriteLine(map2.imageIndex);
// calculate the intersection for each matching pair
List <MatchMetricData> data2 = new List <MatchMetricData>();
foreach (MatchMetricData m in matchMetricData)
{
Image <Bgr, byte> pic1 = Form1.sourceImages[m.map1.imageIndex].Clone();
Image <Bgr, byte> pic2 = Form1.sourceImages[m.map2.imageIndex].Clone();
Point centroid1 = new Point();
Point centroid2 = new Point();
double angle = 0.0;
Match edgeMatch = m.match;
Transformation.transformation(m.dna1, m.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
angle = angle * 180 / Math.PI;
angle = -angle;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString());
Console.WriteLine(angle);
Image <Bgr, byte> mask1 = pic1.Clone();
Image <Bgr, byte> mask2 = pic2.Clone();
Image <Bgr, byte> joined = pic1.Clone();
Image <Bgr, byte> joined_mask = joined.Clone();
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(0, 0), Form1.BKG_WHITE);
data2.Add(new MatchMetricData {
map1 = m.map1, map2 = m.map2, overlap = result.overlap, dna1 = m.dna1, dna2 = m.dna2, match = m.match, confident = m.confident
});
}
if (data2.Count == 0)
{
MessageBox.Show("No match found");
return(1); // add 1 to full image found count
}
// the pair with highest confidence with a valid intersection is the best
// 2nd funnel: select only the matches that can actually match the picture together
MatchMetricData MinOverlap = data2.OrderBy(o => o.overlap).First();
if (MinOverlap.overlap < Constants.THRESHOLD)
{
Console.WriteLine("Map1 " + MinOverlap.map1.imageIndex);
Console.WriteLine("Map2 " + MinOverlap.map2.imageIndex);
Console.WriteLine("Overlap " + MinOverlap.overlap); // correct until this point
// add the resulting image into the queue
Image <Bgr, byte> pic1 = Form1.sourceImages[MinOverlap.map1.imageIndex].Clone();
Image <Bgr, byte> pic2 = Form1.sourceImages[MinOverlap.map2.imageIndex].Clone();
Point centroid1 = new Point();
Point centroid2 = new Point();
double angle = 0.0;
Match edgeMatch = MinOverlap.match;
Transformation.transformation(MinOverlap.dna1, MinOverlap.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
// correct until this point
angle = angle * 180 / Math.PI;
angle = -angle;
Image <Bgr, byte> mask1 = pic1.Clone();
Image <Bgr, byte> mask2 = pic2.Clone();
Image <Bgr, byte> joined = pic1.Clone();
Image <Bgr, byte> joined_mask = joined.Clone();
// tweak
ReturnColorImg bestResult = new ReturnColorImg();
double minOverlap = 999999;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString()); // correct until this point
// The tweaking is messing up with the result, I will just ignore this for a moment
// the tweaking now works for 1-button matching
// but the algorithm runs extremely slow ( still way faster than gluing the pieces :)
for (int i = -2; i < 3; i++)
{
for (int j = -2; j < 3; j++)
{
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(i, j), Form1.BKG_WHITE);
if (result.overlap < minOverlap && result.success) // if the overlap is 0, that means the transformation is failed
{
bestResult = result;
minOverlap = result.overlap;
}
//DisplayImage dip = new DisplayImage(result.img, new Point(0,0), new Point(i, j), (int)result.overlap);
//dip.Show();
}
}
Form1.sourceImages.Add(bestResult.img);
MinOverlap.map1.matched = true; // correct
MinOverlap.map2.matched = true; // correct
List <ColorfulContourMap> cmap = ColorfulContourMap.getAllContourMap(bestResult.img, Form1.sourceImages.Count - 1);
Form1.contourMaps.AddRange(cmap);
this.overlap += bestResult.overlap; // correct
refresh();
MessageBox.Show("Best Match Found.");
return(0);
}
else
{
Console.WriteLine("Failed");
return(1);
}
}
