public List <TextBox> GetTextBoxes(Mat src, ScaleParam scale, float boxScoreThresh, float boxThresh, float unClipRatio)
{
Mat srcResize = new Mat();
CvInvoke.Resize(src, srcResize, new Size(scale.DstWidth, scale.DstHeight));
Tensor <float> inputTensors = OcrUtils.SubstractMeanNormalize(srcResize, MeanValues, NormValues);
var inputs = new List <NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor(inputNames[0], inputTensors)
};
try
{
using (IDisposableReadOnlyCollection <DisposableNamedOnnxValue> results = dbNet.Run(inputs))
{
var resultsArray = results.ToArray();
Console.WriteLine(resultsArray);
var textBoxes = GetTextBoxes(resultsArray, srcResize.Rows, srcResize.Cols, scale, boxScoreThresh, boxThresh, unClipRatio);
return(textBoxes);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message + ex.StackTrace);
}
return(null);
}
public OcrResult Detect(string img, int padding, int maxSideLen, float boxScoreThresh, float boxThresh,
float unClipRatio, bool doAngle, bool mostAngle)
{
Mat originSrc = CvInvoke.Imread(img, ImreadModes.Color);//default : BGR
int originMaxSide = Math.Max(originSrc.Cols, originSrc.Rows);
int resize;
if (maxSideLen <= 0 || maxSideLen > originMaxSide)
{
resize = originMaxSide;
}
else
{
resize = maxSideLen;
}
resize += 2 * padding;
Rectangle paddingRect = new Rectangle(padding, padding, originSrc.Cols, originSrc.Rows);
Mat paddingSrc = OcrUtils.MakePadding(originSrc, padding);
ScaleParam scale = ScaleParam.GetScaleParam(paddingSrc, resize);
return(DetectOnce(paddingSrc, paddingRect, scale, boxScoreThresh, boxThresh, unClipRatio, doAngle, mostAngle));
}
public OcrResult Detect(string img, int padding, int imgResize, float boxScoreThresh, float boxThresh,
float unClipRatio, bool doAngle, bool mostAngle)
{
Mat brgSrc = CvInvoke.Imread(img, ImreadModes.Color);//default : BGR
Mat originSrc = new Mat();
CvInvoke.CvtColor(brgSrc, originSrc, ColorConversion.Bgr2Rgb);// convert to RGB
Rectangle originRect = new Rectangle(padding, padding, originSrc.Cols, originSrc.Rows);
Mat paddingSrc = OcrUtils.MakePadding(originSrc, padding);
int resize;
if (imgResize <= 0)
{
resize = Math.Max(paddingSrc.Cols, paddingSrc.Rows);
}
else
{
resize = imgResize;
}
ScaleParam scale = ScaleParam.GetScaleParam(paddingSrc, resize);
return(DetectOnce(paddingSrc, originRect, scale, boxScoreThresh, boxThresh, unClipRatio, doAngle, mostAngle));
}
private OcrResult DetectOnce(Mat src, Rectangle originRect, ScaleParam scale, float boxScoreThresh, float boxThresh,
float unClipRatio, bool doAngle, bool mostAngle)
{
Mat textBoxPaddingImg = src.Clone();
int thickness = OcrUtils.GetThickness(src);
Console.WriteLine("=====Start detect=====");
var startTicks = DateTime.Now.Ticks;
Console.WriteLine("---------- step: dbNet getTextBoxes ----------");
var textBoxes = dbNet.GetTextBoxes(src, scale, boxScoreThresh, boxThresh, unClipRatio);
var dbNetTime = (DateTime.Now.Ticks - startTicks) / 10000F;
Console.WriteLine($"TextBoxesSize({textBoxes.Count})");
textBoxes.ForEach(x => Console.WriteLine(x));
//Console.WriteLine($"dbNetTime({dbNetTime}ms)");
Console.WriteLine("---------- step: drawTextBoxes ----------");
OcrUtils.DrawTextBoxes(textBoxPaddingImg, textBoxes, thickness);
//CvInvoke.Imshow("ResultPadding", textBoxPaddingImg);
//---------- getPartImages ----------
List <Mat> partImages = OcrUtils.GetPartImages(src, textBoxes);
if (isPartImg)
{
for (int i = 0; i < partImages.Count; i++)
{
CvInvoke.Imshow($"PartImg({i})", partImages[i]);
}
}
Console.WriteLine("---------- step: angleNet getAngles ----------");
List <Angle> angles = angleNet.GetAngles(partImages, doAngle, mostAngle);
//angles.ForEach(x => Console.WriteLine(x));
//Rotate partImgs
for (int i = 0; i < partImages.Count; ++i)
{
if (angles[i].Index == 1)
{
partImages[i] = OcrUtils.MatRotateClockWise180(partImages[i]);
}
if (isDebugImg)
{
CvInvoke.Imshow($"DebugImg({i})", partImages[i]);
}
}
Console.WriteLine("---------- step: crnnNet getTextLines ----------");
List <TextLine> textLines = crnnNet.GetTextLines(partImages);
//textLines.ForEach(x => Console.WriteLine(x));
List <TextBlock> textBlocks = new List <TextBlock>();
for (int i = 0; i < textLines.Count; ++i)
{
TextBlock textBlock = new TextBlock();
textBlock.BoxPoints = textBoxes[i].Points;
textBlock.BoxScore = textBoxes[i].Score;
textBlock.AngleIndex = angles[i].Index;
textBlock.AngleScore = angles[i].Score;
textBlock.AngleTime = angles[i].Time;
textBlock.Text = textLines[i].Text;
textBlock.CharScores = textLines[i].CharScores;
textBlock.CrnnTime = textLines[i].Time;
textBlock.BlockTime = angles[i].Time + textLines[i].Time;
textBlocks.Add(textBlock);
}
//textBlocks.ForEach(x => Console.WriteLine(x));
var endTicks = DateTime.Now.Ticks;
var fullDetectTime = (endTicks - startTicks) / 10000F;
//Console.WriteLine($"fullDetectTime({fullDetectTime}ms)");
//cropped to original size
Mat boxImg = new Mat(textBoxPaddingImg, originRect);
StringBuilder strRes = new StringBuilder();
textBlocks.ForEach(x => strRes.AppendLine(x.Text));
OcrResult ocrResult = new OcrResult();
ocrResult.TextBlocks = textBlocks;
ocrResult.DbNetTime = dbNetTime;
ocrResult.BoxImg = boxImg;
ocrResult.DetectTime = fullDetectTime;
ocrResult.StrRes = strRes.ToString();
return(ocrResult);
}
private static List <TextBox> GetTextBoxes(DisposableNamedOnnxValue[] outputTensor, int rows, int cols, ScaleParam s, float boxScoreThresh, float boxThresh, float unClipRatio)
{
float maxSideThresh = 3.0f;//长边门限
List <TextBox> rsBoxes = new List <TextBox>();
//-----Data preparation-----
float[] predData = outputTensor[0].AsEnumerable <float>().ToArray();
List <byte> cbufData = new List <byte>();
foreach (float data in predData)
{
var val = data * 255;
cbufData.Add(Convert.ToByte(val));
}
Mat predMat = new Mat(rows, cols, DepthType.Cv32F, 1);
predMat.SetTo(predData);
Mat cbufMat = new Mat(rows, cols, DepthType.Cv8U, 1);
cbufMat.SetTo(cbufData.ToArray());
//-----boxThresh-----
Mat thresholdMat = new Mat();
CvInvoke.Threshold(cbufMat, thresholdMat, boxThresh * 255.0, 255.0, ThresholdType.Binary);
//-----dilate-----
Mat dilateMat = new Mat();
Mat dilateElement = CvInvoke.GetStructuringElement(ElementShape.Rectangle, new Size(2, 2), new Point(-1, -1));
CvInvoke.Dilate(thresholdMat, dilateMat, dilateElement, new Point(-1, -1), 1, BorderType.Default, new MCvScalar(128, 128, 128));
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
CvInvoke.FindContours(dilateMat, contours, null, RetrType.List, ChainApproxMethod.ChainApproxSimple);
for (int i = 0; i < contours.Size; i++)
{
if (contours[i].Size <= 2)
{
continue;
}
float maxSide = 0;
List <PointF> minBox = GetMiniBox(contours[i], out maxSide);
if (maxSide < maxSideThresh)
{
continue;
}
double score = GetScore(contours[i], predMat);
if (score < boxScoreThresh)
{
continue;
}
List <Point> clipBox = Unclip(minBox, unClipRatio);
if (clipBox == null)
{
continue;
}
List <PointF> clipMinBox = GetMiniBox(clipBox, out maxSide);
if (maxSide < maxSideThresh + 2)
{
continue;
}
List <Point> finalPoints = new List <Point>();
foreach (var item in clipMinBox)
{
int x = (int)(item.X / s.ScaleWidth);
int ptx = Math.Min(Math.Max(x, 0), s.SrcWidth);
int y = (int)(item.Y / s.ScaleHeight);
int pty = Math.Min(Math.Max(y, 0), s.SrcHeight);
Point dstPt = new Point(ptx, pty);
finalPoints.Add(dstPt);
}
TextBox textBox = new TextBox();
textBox.Score = (float)score;
textBox.Points = finalPoints;
rsBoxes.Add(textBox);
}
rsBoxes.Reverse();
return(rsBoxes);
}
private static List <TextBox> GetTextBoxes(DisposableNamedOnnxValue[] outputTensor, int rows, int cols, ScaleParam s, float boxScoreThresh, float boxThresh, float unClipRatio)
{
float minArea = 3.0f;
List <TextBox> rsBoxes = new List <TextBox>();
float[] outputData = outputTensor[0].AsEnumerable <float>().ToArray();
List <byte> norf = new List <byte>();
foreach (float data in outputData)
{
int val = data > boxThresh ? 255 : 0;
norf.Add((byte)val);
}
Mat fMapMat = new Mat(rows, cols, DepthType.Cv32F, 1);
fMapMat.SetTo(outputData);
Console.WriteLine(fMapMat);
Mat norfMapMat = new Mat(rows, cols, DepthType.Cv8U, 1);
norfMapMat.SetTo(norf.ToArray());
Console.WriteLine(norfMapMat);
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
CvInvoke.FindContours(norfMapMat, contours, null, RetrType.List, ChainApproxMethod.ChainApproxSimple);
for (int i = 0; i < contours.Size; i++)
{
float minEdgeSize = 0;
List <PointF> box = GetMiniBox(contours[i], out minEdgeSize);
if (minEdgeSize < minArea)
{
continue;
}
double score = GetScore(contours[i], fMapMat);
if (score < boxScoreThresh)
{
continue;
}
List <Point> newBox = Unclip(box, unClipRatio);
if (newBox == null)
{
continue;
}
List <PointF> minBox = GetMiniBox(newBox, out minEdgeSize);
if (minEdgeSize < minArea + 2)
{
continue;
}
List <Point> finalPoints = new List <Point>();
foreach (var item in minBox)
{
int x = (int)(item.X / s.ScaleWidth);
int ptx = Math.Min(Math.Max(x, 0), s.SrcWidth);
int y = (int)(item.Y / s.ScaleHeight);
int pty = Math.Min(Math.Max(y, 0), s.SrcHeight);
Point dstPt = new Point(ptx, pty);
finalPoints.Add(dstPt);
}
TextBox textBox = new TextBox();
textBox.Score = (float)score;
textBox.Points = finalPoints;
rsBoxes.Add(textBox);
}
rsBoxes.Reverse();
return(rsBoxes);
}
