public TensorFlowDnnFaceDetector(string modelFile)
{
var modelDirectory = Path.GetDirectoryName(modelFile) !;
var configFile = Path.Combine(modelDirectory, "opencv_face_detector.pbtxt");
Net = CvDnn.ReadNetFromTensorflow(modelFile, configFile);
// Net.SetPreferableTarget(Net.Target.OPENCL_FP16);
Net.SetPreferableBackend(Net.Backend.OPENCV);
}
public void LoadMnistTrainingDataFromFile_NetRecognizesAnImageOfA9Correctly()
{
var img_of_9 = Image(Path.Combine("Dnn", "MNIST_9.png"), ImreadModes.Grayscale);
var img9DataBlob = CvDnn.BlobFromImage(img_of_9, 1f / 255.0f);
var modelPath = Path.Combine("_data", "model", "MNISTTest_tensorflow.pb");
var res = -1;
using (var tfGraph = CvDnn.ReadNetFromTensorflow(modelPath))
{
tfGraph.SetInput(img9DataBlob);
using (var prob = tfGraph.Forward())
res = GetResultClass(prob);
}
Assert.True(res == 9);
}
public void LoadMnistTrainingDataFromStream_NetRecognizesAnImageOfA5Correctly()
{
var img_of_5 = Image(Path.Combine("Dnn", "MNIST_5.png"), ImreadModes.Grayscale);
var img5DataBlob = CvDnn.BlobFromImage(img_of_5, 1f / 255.0f);
var modelPath = Path.Combine("_data", "model", "MNISTTest_tensorflow.pb");
var res = -1;
using (var stream = new FileStream(modelPath, FileMode.Open))
{
using (var tfGraph = CvDnn.ReadNetFromTensorflow(stream))
{
tfGraph.SetInput(img5DataBlob);
using (var prob = tfGraph.Forward())
res = GetResultClass(prob);
}
}
Assert.True(res == 5);
}
private void buttonTest_Click(object sender, EventArgs e)
{
Mat orgMat = new Mat(AppDomain.CurrentDomain.BaseDirectory + @"TestImage\z7.png", ImreadModes.AnyColor);
orgMat = orgMat.CvtColor(ColorConversionCodes.BGR2GRAY);
orgMat = orgMat.CvtColor(ColorConversionCodes.GRAY2BGR);
Cv2.ImShow("img", orgMat);
using (Net net = CvDnn.ReadNetFromTensorflow(AppDomain.CurrentDomain.BaseDirectory + "all_freezed_vgg19_tf115.pb"))
{
int classId1;
double classProb1;
List <CharProbClass> probList;
var inputBlob = CvDnn.BlobFromImage(orgMat, 1, new Size(80, 80), new Scalar(104, 117, 123));
net.SetInput(inputBlob);
var prob = net.Forward();
GetMaxClass(prob, out classId1, out classProb1, out probList);
Debug.Print($"ClassID:{GetClassText(classId1)}, classProb:{classProb1}");
}
}
public ScreenDetection(Form mainForm, PictureBox outPictureBox)
{
this.mainForm = mainForm;
this.outPictureBox = outPictureBox;
this.detectionNet = CvDnn.ReadNetFromTensorflow(modelFile, configFile);
//this.detectionNet = CvDnn.ReadNetFromTensorflow(modelFile);
//this.detectionNet.SetPreferableBackend(Backend.DEFAULT);
//this.detectionNet.SetPreferableTarget(Target.CPU);
this.detectionNet.SetPreferableBackend(Backend.CUDA);
this.detectionNet.SetPreferableTarget(Target.CUDA);
this.labelNames = File.ReadAllLines(labelFile)
.Select(line => line.Split('\n').Last())
.ToArray();
//设置初始检测区域
rawDetectionRect = new DetectionRect();
//缺省使用使用全屏作为工作屏幕
rawDetectionRect.setValues(0, 0, 900, 760);
SetSrcScreenRect(rawDetectionRect);
}
private void LoadImage(string filePath)
{
_textSegments.Clear();
List <Mat> croppedChars = new List <Mat>();
byte[] image = File.ReadAllBytes(filePath);
Mat originalMat = Mat.FromImageData(image, ImreadModes.AnyColor);
Mat displayMat = originalMat.Clone();
_imageSize = originalMat.Size();
Dictionary <string, object> options = new Dictionary <string, object>
{
{ "recognize_granularity", "small" },
{ "detect_direction", "true" },
{ "vertexes_location", "true" },
{ "probability", "true" },
{ "detect_language", "true" }
};
JObject resultJson = _client.Accurate(image, options); // OCR accurate
//Debug.Print(resultJson.ToString());
dynamic ocrResult = JsonConvert.DeserializeObject <dynamic>(resultJson.ToString());
int wordCount = ocrResult.words_result_num;
for (int i = 0; i < wordCount; i++)
{
dynamic chars = ocrResult.words_result[i].chars;
for (int j = 0; j < chars.Count; j++)
{
TextSegmentData segmentData = new TextSegmentData
{
TextLine = ocrResult.words_result[i].words,
TextLineWidth = ocrResult.words_result[i].location.width,
TextLineTop = ocrResult.words_result[i].location.top,
TextLineLeft = ocrResult.words_result[i].location.left,
TextLineHeight = ocrResult.words_result[i].location.height,
TextChar = chars[j]["char"],
TextCharWidth = chars[j].location.width,
TextCharTop = chars[j].location.top,
TextCharLeft = chars[j].location.left,
TextCharHeight = chars[j].location.height
};
segmentData.IsCJK = segmentData.TextChar.Any(x => x.IsChinese());
Debug.Print($"Text: {segmentData.TextChar}, IsCJK? {segmentData.IsCJK}, W {segmentData.TextCharWidth}, H {segmentData.TextCharHeight}, T {segmentData.TextCharTop}, L {segmentData.TextCharLeft} ");
if (segmentData.IsCJK)
{
Rect cropCharRect = new Rect(
segmentData.TextCharLeft,
segmentData.TextCharTop,
GetSizeSafe((int)(segmentData.TextCharWidth * 1.5), segmentData.TextCharLeft, originalMat.Width),
GetSizeSafe((int)(segmentData.TextCharHeight * 1.2), segmentData.TextCharTop, originalMat.Height)
);
//displayMat.Rectangle(cropCharRect, Scalar.RandomColor(), 2, LineTypes.AntiAlias); // mark every word
Mat croppedChar = new Mat(originalMat, cropCharRect);
croppedChars.Add(croppedChar);
segmentData.TextCharCroppedMat = croppedChar.Clone();
Rect cropTextRect = new Rect(segmentData.TextLineLeft, segmentData.TextLineTop, segmentData.TextLineWidth, segmentData.TextLineHeight);
Mat croppedLine = new Mat(originalMat, cropTextRect);
segmentData.TextLineCroppedMat = croppedLine.Clone();
//croppedChar.SaveImage("!" + DateTime.Now.Ticks + ".png");
}
_textSegments.Add(segmentData);
}
}
int netInputWidth = 80;
int netInputHeight = 80;
using (Net net = CvDnn.ReadNetFromTensorflow(AppDomain.CurrentDomain.BaseDirectory + "all_freezed_vgg19_tf115.pb"))
{
foreach (TextSegmentData sgData in _textSegments.Where(x => x.IsCJK).ToArray())
{
// preprocess
//sgData.TextCharCroppedMat.SaveImage("!" + DateTime.Now.Ticks + ".png");
Mat greyText = sgData.TextCharCroppedMat.CvtColor(ColorConversionCodes.BGR2GRAY);
//Mat textAfterThreshold = new Mat();
//Cv2.Threshold(greyText, textAfterThreshold, 0, 255, ThresholdTypes.Binary | ThresholdTypes.Otsu);
//Mat textAfterMorph = new Mat();
//Mat kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(1, 1));
//Cv2.MorphologyEx(textAfterThreshold, textAfterMorph, MorphTypes.Open, kernel);
//Cv2.MorphologyEx(textAfterMorph, textAfterMorph, MorphTypes.Close, kernel);
// resize
double scaleW = netInputWidth / (double)sgData.TextCharCroppedMat.Width;
double scaleH = netInputHeight / (double)sgData.TextCharCroppedMat.Height;
double scale = scaleW < scaleH ? scaleW : scaleH;
Mat resizedText = new Mat();
Cv2.Resize(greyText, resizedText, new Size(0, 0), scale, scale, InterpolationFlags.Cubic);
int padTop = 0;
int padBottom = 0;
int padLeft = 0;
int padRight = 0;
if (resizedText.Width < netInputWidth)
{
padLeft = (netInputWidth - resizedText.Width) / 2;
if ((netInputWidth - resizedText.Width) % 2 > 0)
{
padRight = padLeft + 1;
}
else
{
padRight = padLeft;
}
}
else if (resizedText.Height < netInputHeight)
{
padTop = (netInputHeight - resizedText.Height) / 2;
if ((netInputHeight - resizedText.Height) % 2 > 0)
{
padBottom = padTop + 1;
}
else
{
padBottom = padTop;
}
}
resizedText = resizedText.CopyMakeBorder(padTop, padBottom, padLeft, padRight, BorderTypes.Constant, Scalar.White);
resizedText = resizedText.CvtColor(ColorConversionCodes.GRAY2BGR); // inferring needs BGR input instead of gray
//Cv2.ImShow("" + Guid.NewGuid(), resizedText);
//resizedText.SaveImage("!" + DateTime.Now.Ticks + ".png");
int classId1;
double classProb1;
List <CharProbClass> probList;
var inputBlob = CvDnn.BlobFromImage(resizedText, 1, new Size(netInputWidth, netInputHeight), new Scalar(104, 117, 123));
net.SetInput(inputBlob);
var prob = net.Forward();
GetMaxClass(prob, out classId1, out classProb1, out probList);
sgData.ClassLable = GetClassText(classId1);
sgData.ClassProb = classProb1;
sgData.ProbClassList = probList;
Debug.Print($"Char:{sgData.TextChar}, ClassID:{GetClassText(classId1)}, classProb:{classProb1}");
}
}
// done image processing, calculating
var groupedTextLines = _textSegments.Where(x => x.IsCJK).GroupBy(
x => new
{
x.TextLineWidth,
x.TextLineTop,
x.TextLineLeft,
x.TextLineHeight
}).ToArray();
foreach (var textLine in groupedTextLines)
{
Dictionary <string, double> fontProbDict = new Dictionary <string, double>();
foreach (TextSegmentData segmentData in textLine)
{
if (!fontProbDict.ContainsKey(segmentData.ClassLable))
{
fontProbDict.Add(segmentData.ClassLable, segmentData.ClassProb);
}
else if (segmentData.ClassProb > fontProbDict[segmentData.ClassLable])
{
fontProbDict[segmentData.ClassLable] += segmentData.ClassProb;
}
}
var orderedFontProb = fontProbDict.OrderByDescending(x => x.Value).ToArray();
Debug.Print($"Text Line: {textLine.FirstOrDefault()?.TextLine}, Font Name: {orderedFontProb[0].Key}");
foreach (TextSegmentData data in textLine)
{
data.TextLineFont = orderedFontProb[0].Key;
data.ProbClassList = textLine.ToList().FirstOrDefault()?.ProbClassList;
}
Rect textLineRect = new Rect((int)textLine.FirstOrDefault()?.TextLineLeft, (int)textLine.FirstOrDefault()?.TextLineTop, (int)textLine.FirstOrDefault()?.TextLineWidth, (int)textLine.FirstOrDefault()?.TextLineHeight);
displayMat.Rectangle(textLineRect, Scalar.RandomColor(), 2, LineTypes.AntiAlias);
}
pictureBoxOriginal.Image = displayMat.ToBitmap();
}
public async Task <IEnumerable <Face> > ProcessAsync(string inputFilename)
{
if (!File.Exists(inputFilename))
{
throw new FileNotFoundException(nameof(inputFilename));
}
// Read sample image
using var frame = Cv2.ImRead(inputFilename);
var frameHeight = frame.Rows;
var frameWidth = frame.Cols;
using var faceNet = CvDnn.ReadNetFromTensorflow(TensorflowWeightFile, TensorflowConfigFile);
using var blob = CvDnn.BlobFromImage(frame, 1.0, new Size(300, 300),
new Scalar(104, 117, 123), false, false);
faceNet.SetInput(blob, "data");
using var detection = faceNet.Forward("detection_out");
using var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F, detection.Ptr(0));
var list = new List <ConfidenceBox>(detectionMat.Rows);
for (var i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
var x1 = (int)(detectionMat.At <float>(i, 3) * frameWidth);
var y1 = (int)(detectionMat.At <float>(i, 4) * frameHeight);
var x2 = (int)(detectionMat.At <float>(i, 5) * frameWidth);
var y2 = (int)(detectionMat.At <float>(i, 6) * frameHeight);
list.Add(new ConfidenceBox(new Point(x1, y1), new Point(x2, y2), confidence));
}
var orderedFaces = list.OrderByDescending(x => x.Confidence).Where(x => x.Confidence > 0.3).ToList();
var origFilename = new FileInfo(inputFilename).Name;
var faces = orderedFaces;
// foreach (var face in faces)
// {
// FaceBoxer.Draw(frame, face.P1, face.P2, face.Confidence);
// }
// var outputFilename = Path.Combine(outputDirectory, $"{origFilename}_{Name}.jpg");
// Cv2.ImWrite(outputFilename, frame);
// for (var i = 0; i < orderedList.Count; i++)
// {
// var box = orderedList[i];
// FaceBoxer.Draw(frame, box.P1, box.P2, box.Confidence);
// var outputFilename = Path.Combine(outputDirectory, $"{origFilename}_{Name}_{i+1}.jpg");
// Cv2.ImWrite(outputFilename, frame);
// }
await Task.Yield();
return(list.Select(x => new Face
{
Position = new RectangleDto
{
Top = x.P1.Y,
Right = x.P2.X,
Left = x.P1.X,
Bottom = x.P2.Y,
}.ToRectangle(),
Confidence = x.Confidence,
}));
}