// Only tweak the second image
private void button13_Click(object sender, EventArgs e)
{
// TODO : Tomorrow
// prompt to save the file
progressBar1.Value = 0;
SaveFileDialog sfd = new SaveFileDialog();
sfd.Filter = "PDF|*.pdf";
string filePath = "";
string tempDir = "";
int fileCount = 0;
if (sfd.ShowDialog() == DialogResult.OK)
{
button13.Enabled = false;
filePath = sfd.FileName;
Console.WriteLine(filePath);
// open PDF writer
WriteImagesToPDF writer = new WriteImagesToPDF(filePath);
// for each tweakable data in 5*5 for each image
for (int i = -2; i < 3; i++) // tweak x
{
for (int j = -2; j < 3; j++) // tweak y
{
// prepare images
pic1 = Form1.sourceImages[map1.imageIndex].Clone();
pic2 = Form1.sourceImages[map2.imageIndex].Clone();
Transformation.transformation(DNA1, DNA2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
angle = angle * 180 / Math.PI;
angle = -angle;
mask1 = pic1.Clone();
mask2 = pic2.Clone();
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(i, j));
if (result.success) // if tweakable
{
writer.AddImage(result.img, String.Format("X-tweak:{0}, Y-tweak:{1}, Overlap:{2}", i, j, result.overlap));
}
// progress the bar
if (progressBar1.Value > 95)
{
progressBar1.Value = 100;
}
else
{
progressBar1.Value += 4;
}
}
}
// write
writer.print();
// close PDF writer
MessageBox.Show("Your output file is ready.");
button13.Enabled = true;
}
}
private void button3_Click(object sender, EventArgs e)
{
pic1 = Form1.sourceImages[map1.imageIndex].Clone();
pic2 = Form1.sourceImages[map2.imageIndex].Clone();
Transformation.transformation(DNA1, DNA2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
angle = angle * 180 / Math.PI;
angle = -angle;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString());
Console.WriteLine(angle);
mask1 = pic1.Clone();
mask2 = pic2.Clone();
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, p1Tweak, p2Tweak, Form1.BKG_WHITE);
joined = result.img;
//pictureBox3.Image = result.img./*Resize(pictureBox1.Width, pictureBox1.Height, INTER.CV_INTER_LINEAR).*/ToBitmap();
confidence = edgeMatch.confidence;
overlap = result.overlap;
ConfidenceView.Text = confidence.ToString();
OverlapView.Text = overlap.ToString();
AddMatchHistory();
if (result.success)
{
DisplayImage dip = new DisplayImage(result.img, p1Tweak, p2Tweak, (int)overlap);
dip.Show();
}
else
{
MessageBox.Show("You cannot tweak further in that direction");
}
}
private void button14_Click(object sender, EventArgs e)
{
// TODO : Monday
// for each tweakble data in 5*5
double minOverlap = 999999;
Point tweak = new Point(0, 0);
for (int i = -2; i < 3; i++) // tweak x
{
for (int j = -2; j < 3; j++) // tweak y
{
// prepare images
pic1 = Form1.sourceImages[map1.imageIndex].Clone();
pic2 = Form1.sourceImages[map2.imageIndex].Clone();
Transformation.transformation(DNA1, DNA2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
angle = angle * 180 / Math.PI;
angle = -angle;
mask1 = pic1.Clone();
mask2 = pic2.Clone();
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(i, j));
if (result.success) // if tweakable
{
if (result.overlap < minOverlap)
{
minOverlap = result.overlap;
tweak.X = i;
tweak.Y = j;
}
}
// progress the bar
if (progressBar1.Value > 95)
{
progressBar1.Value = 100;
}
else
{
progressBar1.Value += 4;
}
}
}
p2Tweak = tweak;
label8.Text = p2Tweak.ToString();
}
private void button2_Click(object sender, EventArgs e)
{
pic1 = Form1.sourceImages[map1.imageIndex].Clone();
pic2 = Form1.sourceImages[map2.imageIndex].Clone();
Transformation.transformation(DNA1, DNA2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
angle = angle * 180 / Math.PI;
angle = -angle;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString());
Console.WriteLine(angle);
mask1 = pic1.Clone();
mask2 = pic2.Clone();
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle, p1Tweak, p2Tweak);
joined = result.img;
//pictureBox3.Image = result.img./*Resize(pictureBox1.Width, pictureBox1.Height, INTER.CV_INTER_LINEAR).*/ToBitmap();
confidence = edgeMatch.confidence;
overlap = result.overlap;
ConfidenceView.Text = confidence.ToString();
OverlapView.Text = overlap.ToString();
AddMatchHistory();
}
// used for transformation of color matched pieces
// used the framework of the transformation of edge matched pieces
// mode=0, white background, mode=1, black background
public static ReturnColorImg transformColor(Image <Bgr, Byte> img1, Image <Bgr, Byte> mask1, Image <Bgr, Byte> img2, Image <Bgr, Byte> mask2,
Image <Bgr, Byte> dst, Image <Bgr, Byte> dst_mask,
Point centroid1, Point centroid2, double angle, Point tweak1, Point tweak2, bool mode = true)
{
Image <Bgr, Byte> E = img2.Clone();
Image <Bgr, Byte> E_mask = mask2.Clone();//Don't ruin original images
double intersections = 0;
double x = centroid2.X;
double y = centroid2.Y;
double _x, _y, _y2;
double y2;
LineSegment2D centerLine = new LineSegment2D(new Point((int)x, (int)y), new Point(img2.Width - (int)x, img2.Height - (int)y));
//Rectangle r=new Rectangle((int)x,(int)y,2*(img2.Width-(int)x),2*(img2.Height-(int)y));
Image <Bgr, byte> ri = new Image <Bgr, byte>(2 * (img2.Width - (int)x), 2 * (img2.Height - (int)y), new Bgr(255, 255, 255));
Point oldc = new Point((int)x, (int)y);
bool success = false; // if the tweaking is not successful, return false
// inverse y axis
// y2 = -y;
// rotation of centeroid
x -= img2.Width / 2;
y -= img2.Height / 2; //shift origin to (w/2,h/2)
_x = x * Math.Cos(angle / (180 / Math.PI)) - y * Math.Sin(angle / (180 / Math.PI)); //rotate by theta
_y = x * Math.Sin(angle / (180 / Math.PI)) + y * Math.Cos(angle / (180 / Math.PI));
_x += img2.Width / 2;
_y += img2.Height / 2;//back to origin
//_x = x+img2.Width/2;
//_y = y+img2.Height/2;
// inverse y axis
//_y = -_y2;
centroid2.X = (int)_x;
centroid2.Y = (int)_y;
Point shift = new Point();
shift.X = centroid1.X - centroid2.X;
shift.Y = centroid1.Y - centroid2.Y;
E = E.Rotate(angle, new Bgr(255, 255, 255)); // actual rotation happens here
E_mask = E_mask.Rotate(angle, new Bgr(255, 255, 255));
//Find optimal size of canvas to hold both images and appropriate transformations
Point t1, t2;//transformation 1 and 2
t1 = new Point();
t2 = new Point();
int optimal_h = 0, optimal_w = 0;//of canvas(IplImage* dst)
switch (quadrant(shift))
{
case 1:
t1.X = 0;
t1.Y = 0;
t2 = shift;
optimal_h = Math.Max(img1.Height, img2.Height + shift.Y);
optimal_w = Math.Max(img1.Width, img2.Width + shift.X);
break;
case 2:
t1.X = -shift.X;
t1.Y = 0;
t2.X = 0;
t2.Y = shift.Y;
optimal_h = Math.Max(img1.Height, img2.Height + shift.Y);
optimal_w = Math.Max(img2.Width, img1.Width - shift.X);
break;
case 3:
t1.X = -shift.X;
t1.Y = -shift.Y;
t2.X = 0;
t2.Y = 0;
optimal_h = Math.Max(img1.Height - shift.Y, img2.Height);
optimal_w = Math.Max(img1.Width - shift.X, img2.Width);
break;
case 4:
t1.X = 0;
t1.Y = -shift.Y;
t2.X = shift.X;
t2.Y = 0;
optimal_h = Math.Max(img1.Height - shift.Y, img2.Height);
optimal_w = Math.Max(img2.Width + shift.X, img1.Width);
break;
}
// add tweak factor
t1.X += tweak1.X;
t1.Y += tweak1.Y;
t2.X += tweak2.X;
t2.Y += tweak2.Y;
//optimal_h = 1000;
//optimal_w = 1000;
dst = new Image <Bgr, byte>(optimal_w, optimal_h);
dst_mask = new Image <Bgr, byte>(optimal_w, optimal_h);
/*if (BKG_WHITE)
* cvSet(dst, cvScalar(255));//make it white
* else
* cvSet(dst, cvScalar(0));//make it black*/
if (mode)
{
dst.SetValue(255); // white background=255, black background=0
}
else
{
dst.SetValue(0); // white background=255, black background=0
}
dst_mask.SetZero();
//Direct access wrappers
/*BwImage canvas(dst);
* BwImage canvas_mask(dst_mask);
* BwImage image1(img1);
* BwImage image2(E);
* BwImage image1_msk(mask1);
* BwImage image2_msk(E_mask);*/
//Apply transformation to image1
/*t1.X = 0;
* t1.Y = 0;
* t2.X = 0;
* t2.Y = 0;*/
for (int i = 0; i < img1.Height; ++i)
{
for (int j = 0; j < img1.Width; ++j)
{
// if black background
if (mode)
{
if (mask1.Data[i, j, 0] != 255)
{
int i_new = i + t1.Y;
int j_new = j + t1.X;
try
{
dst.Data[i_new, j_new, 0] = img1.Data[i, j, 0];
dst.Data[i_new, j_new, 1] = img1.Data[i, j, 1];
dst.Data[i_new, j_new, 2] = img1.Data[i, j, 2];
dst_mask.Data[i_new, j_new, 0] = 255;
dst_mask.Data[i_new, j_new, 1] = 255;
dst_mask.Data[i_new, j_new, 2] = 255;
}
catch
{
//MessageBox.Show("You cannot tweak in that direction further");
success = false;
goto ret;
}
}
}
// if white background
else
{
if (mask1.Data[i, j, 0] != 0)
{
int i_new = i + t1.Y;
int j_new = j + t1.X;
try
{
dst.Data[i_new, j_new, 0] = img1.Data[i, j, 0];
dst.Data[i_new, j_new, 1] = img1.Data[i, j, 1];
dst.Data[i_new, j_new, 2] = img1.Data[i, j, 2];
dst_mask.Data[i_new, j_new, 0] = 0;
dst_mask.Data[i_new, j_new, 1] = 0;
dst_mask.Data[i_new, j_new, 2] = 0;
}
catch
{
//MessageBox.Show("You cannot tweak in that direction further");
success = false;
goto ret;
}
}
}
}
}
//Apply transformation to image2
for (int i = 0; i < img2.Height; ++i)
{
for (int j = 0; j < img2.Width; ++j)
{
// if black background
if (mode)
{
if (E_mask.Data[i, j, 0] != 255)
{
int i_new = i + t2.Y;
int j_new = j + t2.X;
try
{
if (dst_mask.Data[i_new, j_new, 0] != 0)
{
intersections++;
}
else
{
dst.Data[i_new, j_new, 0] = E.Data[i, j, 0];
dst.Data[i_new, j_new, 1] = E.Data[i, j, 1];
dst.Data[i_new, j_new, 2] = E.Data[i, j, 2];
dst_mask.Data[i_new, j_new, 0] = 255;
dst_mask.Data[i_new, j_new, 1] = 255;
dst_mask.Data[i_new, j_new, 2] = 255;
}
}
catch
{
//MessageBox.Show("You cannot tweak in that direction further");
success = false;
goto ret;
}
}
}
// else if white background
else
{
if (E_mask.Data[i, j, 0] != 0)
{
int i_new = i + t2.Y;
int j_new = j + t2.X;
try
{
if (dst_mask.Data[i_new, j_new, 0] != 0)
{
intersections++;
}
else
{
dst.Data[i_new, j_new, 0] = E.Data[i, j, 0];
dst.Data[i_new, j_new, 1] = E.Data[i, j, 1];
dst.Data[i_new, j_new, 2] = E.Data[i, j, 2];
dst_mask.Data[i_new, j_new, 0] = 0;
dst_mask.Data[i_new, j_new, 1] = 0;
dst_mask.Data[i_new, j_new, 2] = 0;
}
}
catch
{
//MessageBox.Show("You cannot tweak in that direction further");
success = false;
goto ret;
}
}
}
}
}
success = true;
/*cvReleaseImage(&E);
* cvReleaseImage(&E_mask);*/// should not need these two lines because of garbage collection
// threshold detection is meaningless for 2-piece case, always success
ret : if (intersections > Constants.THRESHOLD)
{
/*cvReleaseImage(&dst);//In case of failure in joining
* cvReleaseImage(&dst_mask);//release memory*/
ReturnColorImg img = new ReturnColorImg();
img.img = dst;
img.img_mask = dst_mask;
img.source1 = img1;
img.source2 = E_mask;
img.center1 = centroid1;
img.center2old = oldc;
img.center2new = centroid2;
img.centerLinee = centerLine;
img.returnbool = false; // for determining if the image is matched or not
img.translate1 = t1;
img.translate2 = t2;
img.overlap = intersections;
img.success = success; // for tweak only
return(img);
}
else
{
ReturnColorImg img = new ReturnColorImg();
img.img = dst;
img.img_mask = dst_mask;
img.source1 = img1;
img.source2 = E_mask;
img.center1 = centroid1;
img.center2old = oldc;
img.center2new = centroid2;
img.centerLinee = centerLine;
img.returnbool = true; // for determining if the image is matched or not
img.translate1 = t1;
img.translate2 = t2;
img.overlap = intersections;
img.success = success; // for tweak only
return(img);
}
}
// algorithm plot:
// 1. Try edge match
// Find max confidence
// Try put them together
// 2. Try color match
// Find max confidence
// Try put them together
// 3. If still fail, that is from other page
// 4. If success, find the best tweak
// 5. Join the images according to the final parameters
// the return value is how many pages should be in the original images
private int bestMatchTwo()
{
int pageCount = 0;
double maxConfidence = 0.0;
ColorfulContourMap map1 = new ColorfulContourMap();
ColorfulContourMap map2 = new ColorfulContourMap();
double overlap = 0.0;
bool matched = false;
// Double-thresholding algorithm (Worked for 5-piece precisely torned paper):
// search for all unmatched pairs, calculate the confidence level ( fixed the problem with different results caused by different ordering )
// reject all matches that has confidence less than 80 ( eliminate some of the false positives)
// calculate the intersection for each matching pair
// the pair with the lowest intersection is the best
// if the intersection of the lowest pair is still greater than 5000, there are no match for the given fragments
List <MatchMetricData> matchMetricData = new List <MatchMetricData>();
// search for all unmatched pairs, calculate the confidence level
// if the max confidence level is 0, that means the remaining pieces cannot be matched by turning angle
// for this case, use the color matching algorithm
// get the potential matching edges
foreach (ColorfulContourMap cmap in Form1.contourMaps)
{
if (cmap.matched == false)
{
foreach (ColorfulContourMap cmap2 in Form1.contourMaps)
{
if (cmap2.matched == false && cmap != cmap2)
{
Match match = DNAUtil.partialMatch(cmap.extractDNA(), cmap2.extractDNA());
double confidence = match.confidence;
matchMetricData.Add(new MatchMetricData {
map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match
});
if (confidence > maxConfidence)
{
maxConfidence = confidence;
map1 = cmap;
map2 = cmap2;
}
}
}
}
}
// if there are no pieces eligible for edge matching, the metric data should have all elements with confidence level 0
// at this time, calculate the metrics for color matching
// Double-thresholding algorithm works for color matching too
if (matchMetricData.Count == 0 || matchMetricData.OrderBy(o => o.confident).Last().confident == 0)
{
foreach (ColorfulContourMap cmap in Form1.contourMaps)
{
if (cmap.matched == false)
{
foreach (ColorfulContourMap cmap2 in Form1.contourMaps)
{
if (cmap2.matched == false && cmap != cmap2)
{
Match match = DNAUtil.partialColorMatch(cmap.extractDNA(), cmap2.extractDNA());
double confidence = match.confidence;
matchMetricData.Add(new MatchMetricData {
map1 = cmap, map2 = cmap2, confident = confidence, dna1 = cmap.extractDNA(), dna2 = cmap2.extractDNA(), match = match
});
if (confidence > maxConfidence)
{
maxConfidence = confidence;
map1 = cmap;
map2 = cmap2;
}
}
}
}
}
}
// if there are still no matches, the process is done
// It will consider the rest of the pieces from another page
// 1st funnel: select the most potential matching edges
matchMetricData = matchMetricData.Where(o => o.confident > Constants.MIN_CONFIDENCE).OrderBy(o => o.confident).Reverse().ToList();
Console.WriteLine(maxConfidence);
Console.WriteLine(map1.imageIndex);
Console.WriteLine(map2.imageIndex);
// calculate the intersection for each matching pair
List <MatchMetricData> data2 = new List <MatchMetricData>();
foreach (MatchMetricData m in matchMetricData)
{
Image <Bgr, byte> pic1 = Form1.sourceImages[m.map1.imageIndex].Clone();
Image <Bgr, byte> pic2 = Form1.sourceImages[m.map2.imageIndex].Clone();
Point centroid1 = new Point();
Point centroid2 = new Point();
double angle = 0.0;
Match edgeMatch = m.match;
Transformation.transformation(m.dna1, m.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
angle = angle * 180 / Math.PI;
angle = -angle;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString());
Console.WriteLine(angle);
Image <Bgr, byte> mask1 = pic1.Clone();
Image <Bgr, byte> mask2 = pic2.Clone();
Image <Bgr, byte> joined = pic1.Clone();
Image <Bgr, byte> joined_mask = joined.Clone();
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(0, 0), Form1.BKG_WHITE);
data2.Add(new MatchMetricData {
map1 = m.map1, map2 = m.map2, overlap = result.overlap, dna1 = m.dna1, dna2 = m.dna2, match = m.match, confident = m.confident
});
}
if (data2.Count == 0)
{
MessageBox.Show("No match found");
return(1); // add 1 to full image found count
}
// the pair with highest confidence with a valid intersection is the best
// 2nd funnel: select only the matches that can actually match the picture together
MatchMetricData MinOverlap = data2.OrderBy(o => o.overlap).First();
if (MinOverlap.overlap < Constants.THRESHOLD)
{
Console.WriteLine("Map1 " + MinOverlap.map1.imageIndex);
Console.WriteLine("Map2 " + MinOverlap.map2.imageIndex);
Console.WriteLine("Overlap " + MinOverlap.overlap); // correct until this point
// add the resulting image into the queue
Image <Bgr, byte> pic1 = Form1.sourceImages[MinOverlap.map1.imageIndex].Clone();
Image <Bgr, byte> pic2 = Form1.sourceImages[MinOverlap.map2.imageIndex].Clone();
Point centroid1 = new Point();
Point centroid2 = new Point();
double angle = 0.0;
Match edgeMatch = MinOverlap.match;
Transformation.transformation(MinOverlap.dna1, MinOverlap.dna2, ref edgeMatch, ref centroid1, ref centroid2, ref angle);
// correct until this point
angle = angle * 180 / Math.PI;
angle = -angle;
Image <Bgr, byte> mask1 = pic1.Clone();
Image <Bgr, byte> mask2 = pic2.Clone();
Image <Bgr, byte> joined = pic1.Clone();
Image <Bgr, byte> joined_mask = joined.Clone();
// tweak
ReturnColorImg bestResult = new ReturnColorImg();
double minOverlap = 999999;
Console.WriteLine(centroid1.ToString());
Console.WriteLine(centroid2.ToString()); // correct until this point
// The tweaking is messing up with the result, I will just ignore this for a moment
// the tweaking now works for 1-button matching
// but the algorithm runs extremely slow ( still way faster than gluing the pieces :)
for (int i = -2; i < 3; i++)
{
for (int j = -2; j < 3; j++)
{
ReturnColorImg result = Transformation.transformColor(pic1, mask1, pic2, mask2, joined, joined_mask, centroid1, centroid2, -angle + 180, new Point(0, 0), new Point(i, j), Form1.BKG_WHITE);
if (result.overlap < minOverlap && result.success) // if the overlap is 0, that means the transformation is failed
{
bestResult = result;
minOverlap = result.overlap;
}
//DisplayImage dip = new DisplayImage(result.img, new Point(0,0), new Point(i, j), (int)result.overlap);
//dip.Show();
}
}
Form1.sourceImages.Add(bestResult.img);
MinOverlap.map1.matched = true; // correct
MinOverlap.map2.matched = true; // correct
List <ColorfulContourMap> cmap = ColorfulContourMap.getAllContourMap(bestResult.img, Form1.sourceImages.Count - 1);
Form1.contourMaps.AddRange(cmap);
this.overlap += bestResult.overlap; // correct
refresh();
MessageBox.Show("Best Match Found.");
return(0);
}
else
{
Console.WriteLine("Failed");
return(1);
}
}
