///''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
public List <PossiblePlate> detectPlatesInScene(Mat imgOriginalScene)
{
List <PossiblePlate> listOfPossiblePlates = new List <PossiblePlate>();
//this will be the return value
Mat imgGrayscaleScene = new Mat();
Mat imgThreshScene = new Mat();
Mat imgContours = new Mat(imgOriginalScene.Size, DepthType.Cv8U, 3);
Random random = new Random();
CvInvoke.DestroyAllWindows();
// show steps '''''''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
CvInvoke.Imshow("0", imgOriginalScene);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
Preprocess.preprocess(imgOriginalScene, ref imgGrayscaleScene, ref imgThreshScene);
//preprocess to get grayscale and threshold images
// show steps '''''''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
CvInvoke.Imshow("1a", imgGrayscaleScene);
CvInvoke.Imshow("1b", imgThreshScene);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
//find all possible chars in the scene,
//this function first finds all contours, then only includes contours that could be chars (without comparison to other chars yet)
List <PossibleChar> listOfPossibleCharsInScene = findPossibleCharsInScene(imgThreshScene);
// show steps '''''''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
frm.txtInfo.AppendText("step 2 - listOfPossibleCharsInScene.Count = " + listOfPossibleCharsInScene.Count.ToString() + "\r\n");
//131 with MCLRNF1 image
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
foreach (PossibleChar possibleChar in listOfPossibleCharsInScene)
{
contours.Push(possibleChar.contour);
}
CvInvoke.DrawContours(imgContours, contours, -1, SCALAR_WHITE);
CvInvoke.Imshow("2b", imgContours);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
//given a list of all possible chars, find groups of matching chars
//in the next steps each group of matching chars will attempt to be recognized as a plate
List <List <PossibleChar> > listOfListsOfMatchingCharsInScene = detectChars.findListOfListsOfMatchingChars(listOfPossibleCharsInScene);
// show steps '''''''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
frm.txtInfo.AppendText("step 3 - listOfListsOfMatchingCharsInScene.Count = " + listOfListsOfMatchingCharsInScene.Count.ToString() + "\r\n");
//13 with MCLRNF1 image
imgContours = new Mat(imgOriginalScene.Size, DepthType.Cv8U, 3);
foreach (List <PossibleChar> listOfMatchingChars in listOfListsOfMatchingCharsInScene)
{
object intRandomBlue = random.Next(0, 256);
object intRandomGreen = random.Next(0, 256);
object intRandomRed = random.Next(0, 256);
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
foreach (PossibleChar matchingChar in listOfMatchingChars)
{
contours.Push(matchingChar.contour);
}
CvInvoke.DrawContours(imgContours, contours, -1, new MCvScalar(Convert.ToDouble(intRandomBlue), Convert.ToDouble(intRandomGreen), Convert.ToDouble(intRandomRed)));
}
CvInvoke.Imshow("3", imgContours);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
//for each group of matching chars
foreach (List <PossibleChar> listOfMatchingChars in listOfListsOfMatchingCharsInScene)
{
PossiblePlate possiblePlate = extractPlate(imgOriginalScene, listOfMatchingChars);
//attempt to extract plate
//if plate was found
if (((possiblePlate.imgPlate != null)))
{
listOfPossiblePlates.Add(possiblePlate);
//add to list of possible plates
}
}
frm.txtInfo.AppendText("\r\n" + listOfPossiblePlates.Count.ToString() + " possible plates found" + "\r\n");
//update text box with # of plates found
// show steps '''''''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
frm.txtInfo.AppendText("\r\n");
CvInvoke.Imshow("4a", imgContours);
for (int i = 0; i <= listOfPossiblePlates.Count - 1; i++)
{
PointF[] ptfRectPoints = new PointF[5];
ptfRectPoints = listOfPossiblePlates[i].rrLocationOfPlateInScene.GetVertices();
Point pt0 = new Point(Convert.ToInt32(ptfRectPoints[0].X), Convert.ToInt32(ptfRectPoints[0].Y));
Point pt1 = new Point(Convert.ToInt32(ptfRectPoints[1].X), Convert.ToInt32(ptfRectPoints[1].Y));
Point pt2 = new Point(Convert.ToInt32(ptfRectPoints[2].X), Convert.ToInt32(ptfRectPoints[2].Y));
Point pt3 = new Point(Convert.ToInt32(ptfRectPoints[3].X), Convert.ToInt32(ptfRectPoints[3].Y));
CvInvoke.Line(imgContours, pt0, pt1, SCALAR_RED, 2);
CvInvoke.Line(imgContours, pt1, pt2, SCALAR_RED, 2);
CvInvoke.Line(imgContours, pt2, pt3, SCALAR_RED, 2);
CvInvoke.Line(imgContours, pt3, pt0, SCALAR_RED, 2);
CvInvoke.Imshow("4a", imgContours);
frm.txtInfo.AppendText("possible plate " + i.ToString() + ", click on any image and press a key to continue . . ." + "\r\n");
CvInvoke.Imshow("4b", listOfPossiblePlates[i].imgPlate);
CvInvoke.WaitKey(0);
}
frm.txtInfo.AppendText("\r\n" + "plate detection complete, click on any image and press a key to begin char recognition . . ." + "\r\n" + "\r\n");
CvInvoke.WaitKey(0);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
return(listOfPossiblePlates);
}
///''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
public List <PossiblePlate> detectCharsInPlates(List <PossiblePlate> listOfPossiblePlates)
{
int intPlateCounter = 0;
//this is only for showing steps
Mat imgContours = default(Mat);
Random random = new Random();
//this is only for showing steps
//if list of possible plates is null,
if ((listOfPossiblePlates == null))
{
return(listOfPossiblePlates);
//return
//if list of possible plates has zero plates
}
else if ((listOfPossiblePlates.Count == 0))
{
return(listOfPossiblePlates);
//return
}
//at this point we can be sure list of possible plates has at least one plate
// for each possible plate, this is a big for loop that takes up most of the function
foreach (PossiblePlate possiblePlate in listOfPossiblePlates)
{
Preprocess.preprocess(possiblePlate.imgPlate, ref possiblePlate.imgGrayscale, ref possiblePlate.imgThresh);
//preprocess to get grayscale and threshold images
// show steps '''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
CvInvoke.Imshow("5a", possiblePlate.imgPlate);
CvInvoke.Imshow("5b", possiblePlate.imgGrayscale);
CvInvoke.Imshow("5c", possiblePlate.imgThresh);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
CvInvoke.Resize(possiblePlate.imgThresh, possiblePlate.imgThresh, new Size(), 1.6, 1.6);
//upscale size by 60% for better viewing and character recognition
CvInvoke.Threshold(possiblePlate.imgThresh, possiblePlate.imgThresh, 0.0, 255.0, ThresholdType.Binary | ThresholdType.Otsu);
//threshold again to eliminate any gray areas
// show steps '''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
CvInvoke.Imshow("5d", possiblePlate.imgThresh);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
//find all possible chars in the plate,
//this function first finds all contours, then only includes contours that could be chars (without comparison to other chars yet)
List <PossibleChar> listOfPossibleCharsInPlate = findPossibleCharsInPlate(possiblePlate.imgGrayscale, possiblePlate.imgThresh);
// show steps '''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
imgContours = new Mat(possiblePlate.imgThresh.Size, DepthType.Cv8U, 3);
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
foreach (PossibleChar possibleChar in listOfPossibleCharsInPlate)
{
contours.Push(possibleChar.contour);
}
CvInvoke.DrawContours(imgContours, contours, -1, SCALAR_WHITE);
CvInvoke.Imshow("6", imgContours);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
//given a list of all possible chars, find groups of matching chars within the plate
List <List <PossibleChar> > listOfListsOfMatchingCharsInPlate = findListOfListsOfMatchingChars(listOfPossibleCharsInPlate);
// show steps '''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
imgContours = new Mat(possiblePlate.imgThresh.Size, DepthType.Cv8U, 3);
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
foreach (List <PossibleChar> listOfMatchingChars in listOfListsOfMatchingCharsInPlate)
{
object intRandomBlue = random.Next(0, 256);
object intRandomGreen = random.Next(0, 256);
object intRandomRed = random.Next(0, 256);
foreach (PossibleChar matchingChar in listOfMatchingChars)
{
contours.Push(matchingChar.contour);
}
CvInvoke.DrawContours(imgContours, contours, -1, new MCvScalar(Convert.ToDouble(intRandomBlue), Convert.ToDouble(intRandomGreen), Convert.ToDouble(intRandomRed)));
}
CvInvoke.Imshow("7", imgContours);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
//if no matching chars were found
if ((listOfListsOfMatchingCharsInPlate == null))
{
// show steps '''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
frm.txtInfo.AppendText("chars found in plate number " + intPlateCounter.ToString() + " = (none), click on any image and press a key to continue . . ." + "\r\n");
intPlateCounter = intPlateCounter + 1;
CvInvoke.DestroyWindow("8");
CvInvoke.DestroyWindow("9");
CvInvoke.DestroyWindow("10");
CvInvoke.WaitKey(0);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
possiblePlate.strChars = "";
//set plate string member variable to empty string
continue;
//and jump back to top of big for loop
}
else if ((listOfListsOfMatchingCharsInPlate.Count == 0))
{
// show steps '''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
frm.txtInfo.AppendText("chars found in plate number " + intPlateCounter.ToString() + " = (none), click on any image and press a key to continue . . ." + "\r\n");
intPlateCounter = intPlateCounter + 1;
CvInvoke.DestroyWindow("8");
CvInvoke.DestroyWindow("9");
CvInvoke.DestroyWindow("10");
CvInvoke.WaitKey(0);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
possiblePlate.strChars = "";
//set plate string member variable to empty string
continue;
//and jump back to top of big for loop
}
//for each group of chars within the plate
for (int i = 0; i <= listOfListsOfMatchingCharsInPlate.Count - 1; i++)
{
//sort chars from left to right
listOfListsOfMatchingCharsInPlate[i].Sort((oneChar, otherChar) => oneChar.boundingRect.X.CompareTo(otherChar.boundingRect.X));
//remove inner overlapping chars
listOfListsOfMatchingCharsInPlate[i] = removeInnerOverlappingChars(listOfListsOfMatchingCharsInPlate[i]);
}
// show steps '''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
imgContours = new Mat(possiblePlate.imgThresh.Size, DepthType.Cv8U, 3);
foreach (List <PossibleChar> listOfMatchingChars in listOfListsOfMatchingCharsInPlate)
{
object intRandomBlue = random.Next(0, 256);
object intRandomGreen = random.Next(0, 256);
object intRandomRed = random.Next(0, 256);
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
foreach (PossibleChar matchingChar in listOfMatchingChars)
{
contours.Push(matchingChar.contour);
}
CvInvoke.DrawContours(imgContours, contours, -1, new MCvScalar(Convert.ToDouble(intRandomBlue), Convert.ToDouble(intRandomGreen), Convert.ToDouble(intRandomRed)));
}
CvInvoke.Imshow("8", imgContours);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
//within each possible plate, suppose the longest list of potential matching chars is the actual list of chars
int intLenOfLongestListOfChars = 0;
int intIndexOfLongestListOfChars = 0;
//loop through all the lists of matching chars, get the index of the one with the most chars
for (int i = 0; i <= listOfListsOfMatchingCharsInPlate.Count - 1; i++)
{
if ((listOfListsOfMatchingCharsInPlate[i].Count > intLenOfLongestListOfChars))
{
intLenOfLongestListOfChars = listOfListsOfMatchingCharsInPlate[i].Count;
intIndexOfLongestListOfChars = i;
}
}
//suppose that the longest list of matching chars within the plate is the actual list of chars
List <PossibleChar> longestListOfMatchingCharsInPlate = listOfListsOfMatchingCharsInPlate[intIndexOfLongestListOfChars];
// show steps '''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
imgContours = new Mat(possiblePlate.imgThresh.Size, DepthType.Cv8U, 3);
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
foreach (PossibleChar matchingChar in longestListOfMatchingCharsInPlate)
{
contours.Push(matchingChar.contour);
}
CvInvoke.DrawContours(imgContours, contours, -1, SCALAR_WHITE);
CvInvoke.Imshow("9", imgContours);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
possiblePlate.strChars = recognizeCharsInPlate(possiblePlate.imgThresh, longestListOfMatchingCharsInPlate);
//perform char recognition on the longest list of matching chars in the plate
// show steps '''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
frm.txtInfo.AppendText("chars found in plate number " + intPlateCounter.ToString() + " = " + possiblePlate.strChars + ", click on any image and press a key to continue . . ." + "\r\n");
intPlateCounter = intPlateCounter + 1;
CvInvoke.WaitKey(0);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
}
//end for each possible plate big for loop that takes up most of the function
// show steps '''''''''''''''''''''''''''''''''
if ((frm.cbShowSteps.Checked == true))
{
frm.txtInfo.AppendText("\r\n" + "char detection complete, click on any image and press a key to continue . . ." + "\r\n");
CvInvoke.WaitKey(0);
}
// show steps '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
return(listOfPossiblePlates);
}
private void btnOpenTrainingImage_Click(object sender, EventArgs e)
{
DialogResult drChosenFile = default(DialogResult);
drChosenFile = ofdOpenFile.ShowDialog();
//open file dialog
//if user chose Cancel or filename is blank . . .
if ((drChosenFile != DialogResult.OK | string.IsNullOrEmpty(ofdOpenFile.FileName)))
{
lblChosenFile.Text = "file not chosen";
//show error message on label
return;
//and exit function
}
Mat imgTrainingNumbers = default(Mat);
try
{
imgTrainingNumbers = CvInvoke.Imread(ofdOpenFile.FileName, LoadImageType.Color);
//if error occurred
}
catch (Exception ex)
{
lblChosenFile.Text = "unable to open image, error: " + ex.Message;
//show error message on label
return;
//and exit function
}
//if image could not be opened
if ((imgTrainingNumbers == null))
{
lblChosenFile.Text = "unable to open image";
//show error message on label
return;
//and exit function
}
lblChosenFile.Text = ofdOpenFile.FileName;
//update label with file name
Mat imgGrayscale = new Mat();
//
Mat imgBlurred = new Mat();
//declare various images
Mat imgThresh = new Mat();
//
Mat imgThreshCopy = new Mat();
//
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
Matrix<float> mtxClassifications = default(Matrix<float>);
Matrix<float> mtxTrainingImages = default(Matrix<float>);
Mat matTrainingImagesAsFlattenedFloats = new Mat();
//possible chars we are interested in are digits 0 through 9 and capital letters A through Z, put these in list intValidChars
List<int> intValidChars = new List<int>{
(int)'0', (int)'1', (int)'2', (int)'3', (int)'4', (int)'5', (int)'6', (int)'7', (int)'8', (int)'9',
(int)'A', (int)'B', (int)'C', (int)'D', (int)'E', (int)'F', (int)'G', (int)'H', (int)'I', (int)'J',
(int)'K', (int)'L', (int)'M', (int)'N', (int)'O', (int)'P', (int)'Q', (int)'R', (int)'S', (int)'T',
(int)'U', (int)'V', (int)'W', (int)'X', (int)'Y', (int)'Z' };
Preprocess.preprocess(imgTrainingNumbers, ref imgGrayscale, ref imgThresh);
CvInvoke.Imshow("imgThresh", imgThresh);
//show threshold image for reference
imgThreshCopy = imgThresh.Clone();
//make a copy of the thresh image, this in necessary b/c findContours modifies the image
//get external countours only
CvInvoke.FindContours(imgThreshCopy, contours, null, RetrType.External, ChainApproxMethod.ChainApproxSimple);
int intNumberOfTrainingSamples = contours.Size;
mtxClassifications = new Matrix<float>(intNumberOfTrainingSamples, 1);
//this is our classifications data structure
//this is our training images data structure, note we will have to perform some conversions to write to this later
mtxTrainingImages = new Matrix<float>(intNumberOfTrainingSamples, RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT);
//this keeps track of which row we are on in both classifications and training images,
int intTrainingDataRowToAdd = 0;
//note that each sample will correspond to one row in
//both the classifications XML file and the training images XML file
//for each contour
for (int i = 0; i <= contours.Size - 1; i++)
{
//if contour is big enough to consider
if ((CvInvoke.ContourArea(contours[i]) > MIN_CONTOUR_AREA))
{
Rectangle boundingRect = CvInvoke.BoundingRectangle(contours[i]);
//get the bounding rect
CvInvoke.Rectangle(imgTrainingNumbers, boundingRect, new MCvScalar(0.0, 0.0, 255.0), 2);
//draw red rectangle around each contour as we ask user for input
Mat imgROItoBeCloned = new Mat(imgThresh, boundingRect);
//get ROI image of current char
Mat imgROI = imgROItoBeCloned.Clone();
//make a copy so we do not change the ROI area of the original image
Mat imgROIResized = new Mat();
//resize image, this is necessary for recognition and storage
CvInvoke.Resize(imgROI, imgROIResized, new Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT));
CvInvoke.Imshow("imgROI", imgROI);
//show ROI image for reference
CvInvoke.Imshow("imgROIResized", imgROIResized);
//show resized ROI image for reference
CvInvoke.Imshow("imgTrainingNumbers", imgTrainingNumbers);
//show training numbers image, this will now have red rectangles drawn on it
int intChar = CvInvoke.WaitKey(0);
//get key press
//if esc key was pressed
if ((intChar == 27))
{
CvInvoke.DestroyAllWindows();
return;
//exit the function
//else if the char is in the list of chars we are looking for . . .
}
else if ((intValidChars.Contains(intChar)))
{
mtxClassifications[intTrainingDataRowToAdd, 0] = Convert.ToSingle(intChar);
//write classification char to classifications Matrix
//now add the training image (some conversion is necessary first) . . .
//note that we have to covert the images to Matrix(Of Single) type, this is necessary to pass into the KNearest object call to train
Matrix<float> mtxTemp = new Matrix<float>(imgROIResized.Size);
Matrix<float> mtxTempReshaped = new Matrix<float>(1, RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT);
imgROIResized.ConvertTo(mtxTemp, DepthType.Cv32F);
//convert Image to a Matrix of Singles with the same dimensions
//flatten Matrix into one row by RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT number of columns
for (int intRow = 0; intRow <= RESIZED_IMAGE_HEIGHT - 1; intRow++)
{
for (int intCol = 0; intCol <= RESIZED_IMAGE_WIDTH - 1; intCol++)
{
mtxTempReshaped[0, (intRow * RESIZED_IMAGE_WIDTH) + intCol] = mtxTemp[intRow, intCol];
}
}
//write flattened Matrix into one row of training images Matrix
for (int intCol = 0; intCol <= (RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT) - 1; intCol++)
{
mtxTrainingImages[intTrainingDataRowToAdd, intCol] = mtxTempReshaped[0, intCol];
}
intTrainingDataRowToAdd = intTrainingDataRowToAdd + 1;
//increment which row, i.e. sample we are on
}
}
}
txtInfo.Text = txtInfo.Text + "training complete !!\n";
//save classifications to file '''''''''''''''''''''''''''''''''''''''''''''''''''''
XmlSerializer xmlSerializer = new XmlSerializer(mtxClassifications.GetType());
StreamWriter streamWriter = default(StreamWriter);
try
{
streamWriter = new StreamWriter("classifications.xml");
//attempt to open classifications file
//if error is encountered, show error and return
}
catch (Exception ex)
{
txtInfo.Text = "\n" + txtInfo.Text + "unable to open //classifications.xml//, error:\n";
txtInfo.Text = txtInfo.Text + ex.Message + "\n";
return;
}
xmlSerializer.Serialize(streamWriter, mtxClassifications);
streamWriter.Close();
//save training images to file '''''''''''''''''''''''''''''''''''''''''''''''''''''
xmlSerializer = new XmlSerializer(mtxTrainingImages.GetType());
try
{
streamWriter = new StreamWriter("images.xml");
//attempt to open images file
//if error is encountered, show error and return
}
catch (Exception ex)
{
txtInfo.Text = "\n" + txtInfo.Text + "unable to open //images.xml//, error:\n";
txtInfo.Text = txtInfo.Text + ex.Message + "\n";
return;
}
xmlSerializer.Serialize(streamWriter, mtxTrainingImages);
streamWriter.Close();
txtInfo.Text = "\n" + txtInfo.Text + "file writing done\n";
MessageBox.Show("Training complete, file writing done !!");
}
