/// <summary>
/// Finds largest connected binary component from a 2D 8-bit grayscale image
/// </summary>
/// <param name="image">Input image.</param>
/// <param name="threshold">Threshold value.</param>
/// <returns></returns>
public static Mat LargestBWObject(Mat image, double threshold)
{
//Binarize the inputimage
Mat bw = image.Threshold(threshold, 255.0, ThresholdTypes.Binary);
//Empty label image
OutputArray labelArray = OutputArray.Create(new Mat(bw.Height, bw.Width, MatType.CV_16SC1));
//Get connected components
bw.ConnectedComponents(labelArray, PixelConnectivity.Connectivity8);
//Get label min and max
double min, max;
labelArray.GetMat().MinMaxIdx(out min, out max);
//Iterate over the labels and select largest connected component
Scalar curmax = 0;
Mat output = new Mat();
for (int k = 1; k < max + 1; k++)
{
Mat _tmp = selectGS(labelArray.GetMat(), (double)k);
Scalar _sum = _tmp.Sum();
if (_sum.Val0 > curmax.Val0)
{
curmax = _sum;
output = _tmp;
}
}
return(output);
}
private void ColorHistShow(Mat image)
{
Mat[] planes;
Mat[] hist;
int[] channels = { 0 };
int[] size = { 256 };
Rangef[] range = { new Rangef(0.0F, 256.0F) };
OpenCvSharp.Scalar[] color =
new[] {
new OpenCvSharp.Scalar(255, 0, 0),
new OpenCvSharp.Scalar(0, 255, 0),
new OpenCvSharp.Scalar(0, 0, 255),
};
Cv2.Split(image, out planes);
hist = new Mat[planes.Length];
for (int i = 0; i < planes.Length; i++)
{
hist[i] = new Mat();
Cv2.CalcHist(new Mat[] { planes[i] }, channels, null, OutputArray.Create(hist[i]), 1, size, range);
}
int hist_width = 512; int hist_height = 400;
int bin_width = (int)Math.Round((double)hist_width / 256);
Mat histImage = new Mat(hist_height, hist_width, MatType.CV_8UC3, new OpenCvSharp.Scalar(0, 0, 0));
for (int i = 0; i < hist.Length; i++)
{
Cv2.Normalize(hist[i], hist[i], 0, histImage.Rows, NormTypes.MinMax, -1, null);
}
for (int i = 1; i < 256; i++)
{
for (int j = 0; j < color.Length; j++)
{
Cv2.Line(histImage,
new OpenCvSharp.Point(bin_width * (i - 1), hist_height - (int)Math.Round(hist[j].At <float>(i - 1))),
new OpenCvSharp.Point(bin_width * (i), hist_height - (int)Math.Round(hist[j].At <float>(i))),
color[j], 2, LineTypes.Link8, 0);
}
}
for (int i = 0; i < planes.Length; i++)
{
planes[i].Dispose();
}
for (int i = 0; i < hist.Length; i++)
{
hist[i].Dispose();
}
Cv2.NamedWindow("Color Histogram", WindowFlags.AutoSize);
Cv2.ImShow("Color Histogram", histImage);
}
private Mat[] FindContours()
{
var hierarchyIndex = OutputArray.Create(new Mat());
var threshold = MakeThreshold();
Cv2.FindContours(threshold, out Mat[] contours, hierarchyIndex, RetrievalModes.List, ContourApproximationModes.ApproxSimple);
return(contours);
}
public Mat Detect(Mat mat)
{
using (var gray = new Mat())
{
Cv2.CvtColor(InputArray.Create(mat), OutputArray.Create(gray), ColorConversionCodes.BGR2GRAY);
cascadeClassifier.DetectMultiScale(gray, Convert.ToDouble(scaleFactor), minimumNeighbors, 0,
new OpenCvSharp.Size(minSizeWidth, minSizeHeight))
.ToList().ForEach(r => mat.Rectangle(new OpenCvSharp.Rect(r.X, r.Y, r.Width, r.Height), Scalar.Red));
return(mat);
}
}
public override void OnUpdate(float dt)
{
base.OnUpdate(dt);
if (capture.Grab())
{
var image = Mat.Zeros(capture.FrameHeight, capture.FrameWidth, MatType.CV_8UC3);
var output = OutputArray.Create(image);
capture.Read(output);
Cv2.ImWrite(tempImagePath, image);
}
}
public List <Point2f> UndistortPoint(List <Point2f> src)
{
if (_init)
{
List <Point2f> dst = new List <Point2f>();
Cv2.UndistortPoints(InputArray.Create(src), OutputArray.Create(dst), _cameraMatrix, _distCoeffs);
return(dst);
}
else
{
return(src);
}
}
public override void Apply(Mat input)
{
_start = DateTime.Now;
Input = input;
if (IsActive)
{
OpenCvSharp.Cv2.CvtColor(InputArray.Create(input), OutputArray.Create(Output), ColorConversionCodes.BGR2HSV);
}
else
{
OpenCvSharp.Cv2.CopyTo(input, Output);
}
base.Apply(input);
}
static void Main(string[] args)
{
string filename = "qrTest2.jpg";
Image input = Image.FromFile(filename);
List <Tag> result = Detector.detectTags(input, showProcessedInput: false);
Mat dst = BitmapConverter.ToMat((Bitmap)input);
foreach (Tag tag in result)
{
System.Drawing.Point[] _points = tag.Polygon;
List <OpenCvSharp.Point> points = new List <OpenCvSharp.Point>();
// Convertimos a array de puntos compatible con OpenCV
foreach (System.Drawing.Point _point in _points)
{
OpenCvSharp.Point point = new OpenCvSharp.Point(_point.X, _point.Y);
points.Add(point);
}
List <OpenCvSharp.Point> hull = new List <OpenCvSharp.Point>();
// Si los puntos no forman un cuadrado hayamos la envolvente convexa
if (points.Count > 4)
{
Cv2.ConvexHull(InputArray.Create(points), OutputArray.Create(hull));
}
else
{
hull = points;
}
// La pintamos sobre la imagen
int n = hull.Count;
for (int j = 0; j < n; j++)
{
Cv2.Line(dst, hull[j], hull[(j + 1) % n], new Scalar(0, 0, 255), 6);
}
string text = Encoding.ASCII.GetString(tag.Data);
Cv2.PutText(dst, text, points[2], HersheyFonts.HersheyDuplex, 2, new Scalar(0, 0, 255), 4);
}
// Redimensionamos y mostramos el resultado
Mat resizedDst = new Mat();
Cv2.Resize(dst, resizedDst, new OpenCvSharp.Size(1080, 720));
using (new Window("Result", WindowMode.FreeRatio, resizedDst))
{
Cv2.WaitKey();
}
}
/// <summary>
/// Run thresholding to byte array
/// </summary>
private void Threshold1()
{
const int T = 3;
const int Max = 5;
byte[] input = { 1, 2, 3, 4, 5, };
List <byte> output = new List <byte>();
Cv2.Threshold(InputArray.Create(input), OutputArray.Create(output),
T, Max, ThresholdType.Binary);
Console.WriteLine("Threshold: {0}", T);
Console.WriteLine("input: {0}", String.Join(",", input));
Console.WriteLine("output: {0}", String.Join(",", output));
}
/// <summary>
/// Run thresholding to short array
/// </summary>
private void Threshold2()
{
const int T = 150;
const int Max = 250;
short[] input = { 50, 100, 150, 200, 250, };
List <short> output = new List <short>();
Cv2.Threshold(InputArray.Create(input), OutputArray.Create(output),
T, Max, ThresholdType.Binary);
Console.WriteLine("Threshold: {0}", T);
Console.WriteLine("input: {0}", String.Join(",", input));
Console.WriteLine("output: {0}", String.Join(",", output));
}
/// <summary>
/// Template match
/// </summary>
/// <param name="wantBitmap">Want match bitmap</param>
/// <param name="bitmap">target bitmap</param>
/// <param name="templateMatch">template match option</param>
/// <param name="cancellationToken">cancellationToken</param>
/// <returns>Target bitmap location</returns>
private async Task <Rectangle?> TemplateMatchLocation(Bitmap wantBitmap, Bitmap bitmap,
TemplateMatch templateMatch, CancellationToken cancellationToken)
{
return(await Task.Run(() =>
{
try
{
cancellationToken.ThrowIfCancellationRequested();
using var srcMat = bitmap.ToMat();
using var dstMat = wantBitmap.ToMat();
using var outArray = OutputArray.Create(srcMat);
cancellationToken.ThrowIfCancellationRequested();
Cv2.MatchTemplate(srcMat, dstMat, outArray, templateMatch.TemplateMatchModel);
cancellationToken.ThrowIfCancellationRequested();
Cv2.MinMaxLoc(InputArray.Create(outArray.GetMat() !), out _,
out var maxValue, out _, out var point);
if (maxValue >= templateMatch.Threshold && maxValue <= 1d)
{
var rectangle =
new Rectangle?(new Rectangle(point.X, point.Y, wantBitmap.Width, wantBitmap.Height));
WindowsApi.WriteLog(
$"{nameof(TemplateMatchLocation)} match success, {nameof(TemplateMatch.TemplateMatchModel)}:{templateMatch.TemplateMatchModel}, {nameof(TemplateMatch.Threshold)}:{templateMatch.Threshold}, {nameof(maxValue)}:{maxValue}, {rectangle}");
return rectangle;
}
else
{
WindowsApi.WriteLog(
$"{nameof(TemplateMatchLocation)} match failed, {nameof(TemplateMatch.TemplateMatchModel)}:{templateMatch.TemplateMatchModel}, {nameof(TemplateMatch.Threshold)}:{templateMatch.Threshold}, {nameof(maxValue)}:{maxValue}");
}
}
catch (Exception ex)
{
WindowsApi.WriteLog(
$"{nameof(TemplateMatchLocation)} {nameof(TemplateMatch.TemplateMatchModel)}:{templateMatch.TemplateMatchModel}, {nameof(TemplateMatch.Threshold)}:{templateMatch.Threshold}, ErrorMessage:{ex.Message}");
}
return null;
}, cancellationToken));
}
public bool PlMask(Mat plImage, int threshold, out Mat edgeMask)
{
bool result = false;
edgeMask = new Mat();
try
{
Mat gray = new Mat();
Cv2.CvtColor(plImage, gray, ColorConversionCodes.BGR2GRAY);
Cv2.Threshold(gray, edgeMask, threshold, 255, ThresholdTypes.Binary);
Cv2.ImShow("edgeMask", edgeMask);
Cv2.WaitKey(0);
Mat[] contours;
var hierarchy = new List <Point>();
Cv2.FindContours(edgeMask, out contours, OutputArray.Create(hierarchy), RetrievalModes.External,
ContourApproximationModes.ApproxSimple);
var hulls = new List <Mat>();
for (int j = 0; j < contours.Length; j++)
{
Mat hull = new Mat();
Cv2.ConvexHull(contours[j], hull);
hulls.Add(hull);
}
edgeMask = Mat.Zeros(edgeMask.Size(), MatType.CV_8UC1);
Cv2.DrawContours(edgeMask, hulls, -1, Scalar.White);
Cv2.ImShow("edgeMask", edgeMask);
Cv2.WaitKey(0);
Cv2.DestroyAllWindows();
result = true;
}
catch (Exception ex)
{
}
return(result);
}
private static string CreateBase64HtmlImage(Mat mat)
{
byte[] buffer;
Mat mediumMat = new Mat();
Cv2.Resize(InputArray.Create(mat), OutputArray.Create(mediumMat), new Size(mat.Width / 2, mat.Height / 2));
Cv2.ImEncode(".jpeg", InputArray.Create(mediumMat), out buffer);
string b64 = Convert.ToBase64String(buffer);
// Create thumbnail
Mat smallMat = new Mat();
Cv2.Resize(InputArray.Create(mat), OutputArray.Create(smallMat), new Size(mat.Width / 6, mat.Height / 6));
Cv2.ImEncode(".jpeg", InputArray.Create(smallMat), out buffer);
string b64Small = Convert.ToBase64String(buffer);
return($"<a class=\"thumbnail\" href=\"#thumb\"><img src=\"data:image/jpeg;base64, {b64Small}\" class=\"thumbnail-image\" border=\"0\" /><span><img src=\"data:image/jpeg;base64, {b64}\" /><br></span></a>");
}
/// <summary>
/// Run thresholding to struct array
/// </summary>
private void Threshold3()
{
const double T = 2000;
const double Max = 5000;
// threshold does not support Point (int)
Point2f[] input =
{
new Point2f(1000, 1500),
new Point2f(2000, 2001),
new Point2f(500, 5000),
};
List <Point2f> output = new List <Point2f>();
Cv2.Threshold(InputArray.Create(input), OutputArray.Create(output),
T, Max, ThresholdType.Binary);
Console.WriteLine("Threshold: {0}", T);
Console.WriteLine("input: {0}", String.Join(",", input));
Console.WriteLine("output: {0}", String.Join(",", output));
}
public void ByNormalArray()
{
// x + y = 10
// 2x + 3y = 26
// (x=4, y=6)
double[,] a = { { 1, 1 },
{ 2, 3 } };
double[] y = { 10, 26 };
List <double> x = new List <double>();
Cv2.Solve(
InputArray.Create(a), InputArray.Create(y),
OutputArray.Create(x),
DecompTypes.LU);
Console.WriteLine("X1 = {0}, X2 = {1}", x[0], x[1]);
Assert.That(x[0], Is.EqualTo(4).Within(1e-6));
Assert.That(x[1], Is.EqualTo(6).Within(1e-6));
}
public void ByNormalArray()
{
// x + y = 10
// 2x + 3y = 26
// (x=4, y=6)
double[,] a = { { 1, 1 },
{ 2, 3 } };
double[] y = { 10, 26 };
List <double> x = new List <double>();
Cv2.Solve(
InputArray.Create(a), InputArray.Create(y),
OutputArray.Create(x),
DecompTypes.LU);
testOutputHelper.WriteLine("X1 = {0}, X2 = {1}", x[0], x[1]);
Assert.Equal(4, x[0], 6);
Assert.Equal(6, x[1], 6);
}
/// <summary>
/// Solve equation AX = Y
/// </summary>
private void ByNormalArray()
{
// x + y = 10
// 2x + 3y = 26
// (x=4, y=6)
double[,] a = { { 1, 1 },
{ 2, 3 } };
double[] y = { 10, 26 };
var x = new List <double>();
Cv2.Solve(
InputArray.Create(a), InputArray.Create(y),
OutputArray.Create(x),
DecompTypes.LU);
Console.WriteLine("ByNormalArray:");
Console.WriteLine("X1 = {0}, X2 = {1}", x[0], x[1]);
}
public List <Rect> DetectLetters(Mat input)
{
Mat mask = Mat.Zeros(input.Size(), MatType.CV_8UC1);
Mat[] contours;
var bounds = new List <Rect>();
var hierarchy = new List <Vec4i>();
input.FindContours(out contours, OutputArray.Create(hierarchy), ContourRetrieval.CComp, ContourChain.ApproxSimple, new Point(0, 0));
for (var idx = 0; idx >= 0; idx = hierarchy[idx][0])
{
var rect = Cv2.BoundingRect(contours[idx]);
Cv2.DrawContours(mask, contours, idx, new Scalar(255, 255, 255));
var maskRoi = new Mat(mask, rect);
var fill = (double)maskRoi.CountNonZero() / (double)(rect.Width * rect.Height);
if (fill > _minFill && rect.Height > 8 && rect.Width > 8)
{
bounds.Add(rect);
}
}
return(bounds);
}
//Capture a rendered texture frame and register the checkerboard pattern data
public void RegisterCurrentCalib()
{
corners.Clear();
obj.Clear();
//imagePoints.Clear();
//objPoints.Clear();
bool b = false;
//find the corners and populate the data for one sqaure
b = Cv2.FindChessboardCorners(mat, boardSize, OutputArray.Create(corners),
ChessboardFlags.AdaptiveThresh | ChessboardFlags.NormalizeImage | ChessboardFlags.FastCheck);
if (!b)
{
return;
}
Cv2.CornerSubPix(grayMat, corners, new Size(5, 5), new Size(-1, -1), TermCriteria.Both(30, 0.1));
Debug.Log(b);
// for debug draw the found squares
Cv2.DrawChessboardCorners(mat, boardSize, corners, b);
for (int i = 0; i < boardSize.Height; i++)
{
for (int j = 0; j < boardSize.Width; j++)
{
//add the space coordinates of the squares. Z = 0 since its a flat plane.
obj.Add(new Point3f((float)j * squareSizeMeters, (float)i * squareSizeMeters, 0));
if (b)
{
//register the data per square
CornerPoints.Add(corners);
objPoints.Add(obj);
}
}
}
}
private void ClickThread(object args)
{
//初始化
ClickThreadArguments cta = new ClickThreadArguments();
ClickThreadInfoSave ctis = new ClickThreadInfoSave();
void UpdateAndGetClickThreadArguments()
{
cta = (ClickThreadArguments)this.Invoke((Func <object>)(() =>
{
clickThreadArguments.screen = Screen.AllScreens[int.Parse(Config.Get(Config.Name_ScreenIndex))];
clickThreadArguments.templatePath = Config.Get(Config.Name_TemplatePath);
clickThreadArguments.threshold = byte.Parse(Config.Get(Config.Name_Threshold));
clickThreadArguments.delay = int.Parse(Config.Get(Config.Name_Delay));
return(this.clickThreadArguments);
}));
}
DateTime begin = new DateTime(0);
DateTime timer = new DateTime(0);
void WaitTimer()
{
TimeSpan subResult;
TimeSpan delayTime;
while (true)
{
//更新配置
UpdateAndGetClickThreadArguments();
delayTime = new TimeSpan(0, 0, 0, 0, cta.delay);
//计算时差
subResult = DateTime.Now.Subtract(timer);
StatusStrip_Set_toolStripStatusLabel_TotalRunningTime(DateTime.Now.Subtract(begin));
if (subResult < delayTime)
{
StatusStrip_Set_toolStripProgressBar_Timer_Value((int)(100 * (subResult.TotalMilliseconds / delayTime.TotalMilliseconds))); //UI
StatusStrip_Set_toolStripLabel_Timer_Value((int)subResult.TotalMilliseconds, cta.delay); //UI
}
else
{
StatusStrip_Set_toolStripProgressBar_Timer_Value(100);
StatusStrip_Set_toolStripLabel_Timer_Value((int)delayTime.TotalMilliseconds, cta.delay);//UI
break;
}
//防卡停顿
Thread.Sleep(100);
}
}
ScreenShots screenShots = new ScreenShots();
Mat screenShot = new Mat();
Mat template = new Mat();
Mat result = new Mat();
//恢复线程状态
if (args != null)
{
ctis = (ClickThreadInfoSave)args;
begin = DateTime.Now.Subtract(ctis.totalRunningTime);
timer = DateTime.Now.Subtract(ctis.timer);
}
else
{
ctis.timer = new TimeSpan(0);
ctis.totalRunningTime = new TimeSpan(0);
ctis.lastLocation = new System.Drawing.Point(0, 0);
ctis.lastValue = 0;
ctis.clickCount = 0;
ctis.scanedCount = 0;
begin = DateTime.Now;
}
try
{
if (args != null)
{
WaitTimer();
}
while (true)
{
//获取配置
UpdateAndGetClickThreadArguments();
//开始计时
timer = DateTime.Now;
StatusStrip_Set_toolStripProgressBar_Timer_Value(0); //UI
StatusStrip_Set_toolStripLabel_Timer_Value(0, cta.delay); //UI
//获取并保存截屏
screenShots.ScreenShotDefault(cta.screen);
screenShots.LastScreenShot.Save(Config.ScreenShotPath);
//初始化图片
screenShot = new Mat(Config.ScreenShotPath);
template = new Mat(Config.Get(Config.Name_TemplatePath));
result = new Mat();//Mat(new OpenCvSharp.Size(screenShots.LastScreenShot.Width, screenShots.LastScreenShot.Height), MatType.CV_32SC1)
//比对
Cv2.MatchTemplate(InputArray.Create(screenShot), InputArray.Create(template), OutputArray.Create(result), TemplateMatchModes.CCoeffNormed, null);
//查找最佳匹配
OpenCvSharp.Point maxLoc = new OpenCvSharp.Point(0, 0);
double maxVal = 0;
Cv2.MinMaxLoc(InputArray.Create(result), out _, out maxVal, out _, out maxLoc);
//记录结果值
ctis.lastValue = (byte)(maxVal * 255);
StatusStrip_Set_toolStripLabel_LastValue_Value(ctis.lastValue);//UI
//记录扫描
ctis.scanedCount++;
StatusStrip_Set_toolStripStatusLabel_ScanedCount(ctis.scanedCount);//UI
//检查阀值
if (maxVal >= cta.threshold / (double)255)
{
//计算实际中心位置
int realX = cta.screen.WorkingArea.X + maxLoc.X + template.Width / 2;
int realY = cta.screen.WorkingArea.Y + maxLoc.Y + template.Height / 2;
//模拟点击
SimClick(realX, realY);
StatusStrip_Set_toolStripLabel_LastLocation_Value(realX, realY);//UI
//记录点击
ctis.lastLocation = new System.Drawing.Point(realX, realY);
ctis.clickCount++;
StatusStrip_Set_toolStripStatusLabel_ClickCount(ctis.clickCount);//UI
//检查是否完成退出计数
if (Program.ProgramArguments.ExitClickCount > 0 && Program.ProgramArguments.ExitClickCount <= ctis.clickCount)
{
this.Invoke((Action)(() =>
{
Program.Exit(0);
}));
}
}
//清理内存
GC.Collect();
//延时
WaitTimer();
}
}
catch (ThreadAbortException taEx)
{
//处理线程终止
//清理
result.Dispose();
template.Dispose();
screenShot.Dispose();
screenShots.Clear();
if (taEx.ExceptionState != null && (string)taEx.ExceptionState == "pause")
{
if (timer != new DateTime(0))
{
ctis.timer = DateTime.Now.Subtract(timer);
}
if (begin != new DateTime(0))
{
ctis.totalRunningTime = DateTime.Now.Subtract(begin);
}
this.BeginInvoke((Action)(() =>
{
lastClickThreadInfoSave = ctis;
}));
}
//else
//{
// //清除状态栏信息
// StatusStrip_Set_toolStripProgressBar_Timer_Value(0);//UI
// StatusStrip_Set_toolStripLabel_Timer_Value(0, cta.delay);//UI
//}
//阻塞UI，不可使用，请在调用终止代码后方添加状态调整
//this.BeginInvoke((Action)(() =>
//{
// SetThreadStatus(false);//UI
//}));
}
}
private Bitmap CreateObjectMaskNew(Bitmap image, out Mat image_mask,
out double mask_length, out double mask_area, out double mask_width, out double mask_height,
out double mask_pvheight, double kThresh, double hThresh, double canny1, double canny2,
out Mat image_mask_spc, out double mask2_area, int brightAreaThreshold = -1, int darkAreaThreshold = -1)
{
Bitmap dst = null;
image_mask = null;
image_mask_spc = null;
mask_length = mask_area = mask_width = mask_height = mask_pvheight = mask2_area = 0;
try
{
Mat src = BitmapConverter.ToMat(image);
Mat src_kirsch = BitmapConverter.ToMat(image.KirschFilter());
Mat kirsch_gray = new Mat();
Cv2.CvtColor(src_kirsch, kirsch_gray, ColorConversionCodes.RGB2GRAY);
Mat kirsch_threshold = new Mat();
Cv2.Threshold(kirsch_gray, kirsch_threshold, kThresh, 255, ThresholdTypes.Binary);
Mat[] contours;
List <OpenCvSharp.Point> hierarchy;
List <Mat> hulls;
Mat morph_element = Cv2.GetStructuringElement(MorphShapes.Ellipse, new OpenCvSharp.Size(2, 2), new OpenCvSharp.Point(1, 1));
#region morphology
Mat kirsch_threshold_copy = new Mat();
kirsch_threshold.CopyTo(kirsch_threshold_copy);
int hullCount = 0, numLoops = 0;
do
{
numLoops++;
Mat kirsch_morph = kirsch_threshold_copy.MorphologyEx(MorphTypes.Gradient, morph_element);
hierarchy = new List <OpenCvSharp.Point>();
Cv2.FindContours(kirsch_morph, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
hulls = new List <Mat>();
for (int j = 0; j < contours.Length; j++)
{
Mat hull = new Mat();
Cv2.ConvexHull(contours[j], hull);
hulls.Add(hull);
}
Mat drawing = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(drawing, hulls, -1, Scalar.White);
if (hulls.Count != hullCount && numLoops < 100)
{
hullCount = hulls.Count;
kirsch_threshold_copy = drawing;
}
else
{
break;
}
} while (true);
#endregion
if (numLoops >= 100)
{
throw new Exception("Could not find hull");
}
#region bestHull
//try and filter out dust near to stone
double largestArea = hulls.Max(m => Cv2.ContourArea(m));
var bestHulls = hulls.Where(m => Cv2.ContourArea(m) == largestArea).ToList();
Mat hulls_mask = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(hulls_mask, bestHulls, -1, Scalar.White, -1);
//hulls_mask is the convex hull of outline, now look for clefts
Cv2.Threshold(kirsch_gray, kirsch_threshold, hThresh, 255, ThresholdTypes.Binary);
Mat kirsch_mask = Mat.Zeros(kirsch_threshold.Size(), kirsch_threshold.Type());
kirsch_threshold.CopyTo(kirsch_mask, hulls_mask);
Mat kirsch_mask_canny = new Mat();
Cv2.Canny(kirsch_mask, kirsch_mask_canny, canny1, canny2, 3);
morph_element = Cv2.GetStructuringElement(MorphShapes.Ellipse, new OpenCvSharp.Size(5, 5), new OpenCvSharp.Point(2, 2));
Mat kirsch_filled = new Mat();
Cv2.Dilate(kirsch_mask_canny, kirsch_filled, morph_element);
Cv2.Dilate(kirsch_filled, kirsch_filled, morph_element);
Cv2.Erode(kirsch_filled, kirsch_filled, morph_element);
Cv2.Erode(kirsch_filled, kirsch_filled, morph_element);
hierarchy = new List <OpenCvSharp.Point>();;
Cv2.FindContours(kirsch_filled, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
#endregion
hulls_mask = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(hulls_mask, contours, -1, Scalar.White, -1);
Cv2.Erode(hulls_mask, hulls_mask, morph_element);
Cv2.Erode(hulls_mask, hulls_mask, morph_element);
image_mask = hulls_mask;
//remove bright areas
if ((brightAreaThreshold > -1) || (darkAreaThreshold > -1))
{
Mat src_mask = new Mat();
Mat hulls_mask_spc = hulls_mask.Clone();
src.CopyTo(src_mask, hulls_mask_spc);
Mat gray = new Mat();
Cv2.CvtColor(src_mask, gray, ColorConversionCodes.BGR2GRAY);
if (brightAreaThreshold > -1)
{
Mat bright = new Mat();
Cv2.Threshold(gray, bright, brightAreaThreshold, 255, ThresholdTypes.BinaryInv);
Cv2.ImWrite(@"C:\gColorFancy\Image\bright.jpg", bright);
Mat t = new Mat();
hulls_mask_spc.CopyTo(t, bright);
hulls_mask_spc = t.Clone();
}
if (darkAreaThreshold > -1)
{
Mat dark = new Mat();
Cv2.Threshold(gray, dark, darkAreaThreshold, 255, ThresholdTypes.Binary);
Cv2.ImWrite(@"C:\gColorFancy\Image\dark.jpg", dark);
Mat t = new Mat();
hulls_mask_spc.CopyTo(t, dark);
hulls_mask_spc = t.Clone();
}
image_mask_spc = hulls_mask_spc;
var hierarchy2 = new List <OpenCvSharp.Point>();;
Cv2.FindContours(hulls_mask_spc, out contours, OutputArray.Create(hierarchy2),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
largestArea = contours.Max(m => Cv2.ContourArea(m));
Mat finalHullSpc = contours.Where(m => Cv2.ContourArea(m) == largestArea).ToList()[0];
if (ConvexHullOnMask)
{
Mat hull = new Mat();
Cv2.ConvexHull(finalHullSpc, hull);
Mat polySpc = new Mat();
Cv2.ApproxPolyDP(hull, polySpc, 3, true);
mask2_area = Cv2.ContourArea(polySpc);
}
else
{
mask2_area = largestArea;
}
}
///////////////////////////
hierarchy = new List <OpenCvSharp.Point>();;
Cv2.FindContours(hulls_mask, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
largestArea = contours.Max(m => Cv2.ContourArea(m));
Mat finalHull = contours.Where(m => Cv2.ContourArea(m) == largestArea).ToList()[0];
if (ConvexHullOnMask)
{
var hull = new Mat();
Cv2.ConvexHull(finalHull, hull);
finalHull = hull;
}
List <Mat> finalHulls = new List <Mat>();
finalHulls.Add(finalHull);
Cv2.DrawContours(src, finalHulls, -1, new Scalar(128, 0, 128, 255), 3);
#region bounding
Mat poly = new Mat();
Cv2.ApproxPolyDP(finalHull, poly, 3, true);
Rect boundaryRect = Cv2.BoundingRect(poly);
mask_width = boundaryRect.Width;
mask_height = boundaryRect.Height;
if (ConvexHullOnMask)
{
mask_area = Cv2.ContourArea(poly);
}
else
{
mask_area = largestArea;
}
mask_length = Cv2.ArcLength(finalHull, true);
List <OpenCvSharp.Point> finalPoints = new List <OpenCvSharp.Point>();
int m1Count = (finalHull.Rows % 2 > 0) ? finalHull.Rows + 1 : finalHull.Rows;
OpenCvSharp.Point[] p1 = new OpenCvSharp.Point[m1Count];
finalHull.GetArray(0, 0, p1);
Array.Resize(ref p1, finalHull.Rows);
finalPoints.AddRange(p1.ToList());
double y_min = boundaryRect.Bottom;
double y_x_min = finalPoints.Where(p => p.X == boundaryRect.Left).ToList()[0].Y;
double y_x_max = finalPoints.Where(p => p.X == boundaryRect.Right).ToList()[0].Y;
mask_pvheight = ((double)y_x_max + (double)y_x_min) / 2 - (double)y_min;
#endregion
//dst = BitmapConverter.ToBitmap(src);
using (var ms = src.ToMemoryStream())
{
dst = (Bitmap)Image.FromStream(ms);
}
try
{
if (saveMaskDataPath.Length > 0)
{
//StringBuilder sb = new StringBuilder();
//sb.AppendLine("mask_length,mask_area,mask_width,mask_height,mask_pvheight");
//sb.AppendLine(mask_length + "," + mask_area + "," + mask_width + "," + mask_height + "," + mask_pvheight);
image_mask.SaveImage(saveMaskDataPath + @"\image_mask.jpg");
if (image_mask_spc != null)
{
image_mask_spc.SaveImage(saveMaskDataPath + @"\image_mask_spc.jpg");
}
BitmapConverter.ToMat(image).SaveImage(saveMaskDataPath + @"\src.jpg");
//File.WriteAllText(saveMaskDataPath + @"\mask_vals.csv", sb.ToString());
//File.AppendAllText(saveMaskDataPath + @"\exception.txt", DateTime.Now + ":" + av.Message);
//File.AppendAllText(saveMaskDataPath + @"\exception.txt", DateTime.Now + ":" + av.StackTrace);
//File.AppendAllText(saveMaskDataPath + @"\exception.txt", DateTime.Now + ":" + av.Source);
}
}
catch
{
}
}
catch (Exception ex)
{
dst = null;
}
return(dst);
}
public static int ExtractTables(Image img)
{
//Delete this after
Bitmap bit;
List <Rectangle> AreasOfInterest = new List <Rectangle>();
Bitmap bitmap = (Bitmap)img;
Mat srcImg = OCS.Extensions.BitmapConverter.ToMat(bitmap);
if (srcImg.Data == null)
{
throw new NullReferenceException("Image has nothing?");
}
//Resize into smaller size
Mat rsz = new Mat();
OpenCvSharp.Size size = new OpenCvSharp.Size(4000, 2828);
Cv2.Resize(srcImg, rsz, size);
// Convert to greyscale if it has more than one channel
// else just leave it alone
Mat grey = new Mat();
Cv2.CvtColor(rsz, grey, ColorConversionCodes.BGR2GRAY);
#if IMG_DEBUG
Cv2.ImShow("grey", grey);
Cv2.WaitKey(0);
#endif
//Apply adaptive thresholding to get negative
Mat bw = new Mat();
Cv2.AdaptiveThreshold(~grey, bw, 255, AdaptiveThresholdTypes.MeanC, ThresholdTypes.Binary, 15, -2);
bit = OCS.Extensions.BitmapConverter.ToBitmap(bw);
bit.Save("bw.tiff", System.Drawing.Imaging.ImageFormat.Tiff);
// Create two new masks cloned from bw.
Mat horizontal = bw.Clone();
Mat vertical = bw.Clone();
// adjust this for number of lines
int scale = 10;
/////////////////////////
/////////////////////////
/////////////////////////
// Specify size on horizontal axis
int horizontalsize = horizontal.Cols / scale;
// Create structure element for extracting horizontal lines through morphology operations
//Mat horizontalStructure = getStructuringElement(MORPH_RECT, Size(horizontalsize, 1));
Mat horizontalStructure = Cv2.GetStructuringElement(MorphShapes.Rect, new OCS.Size(horizontalsize, 1));
// Apply morphology operations
//erode(horizontal, horizontal, horizontalStructure, Point(-1, -1));
Cv2.Erode(horizontal, horizontal, horizontalStructure, new OCS.Point(-1, -1));
//dilate(horizontal, horizontal, horizontalStructure, Point(-1, -1));
Cv2.Dilate(horizontal, horizontal, horizontalStructure, new OCS.Point(-1, -1));
// dilate(horizontal, horizontal, horizontalStructure, Point(-1, -1)); // expand horizontal lines
// Show extracted horizontal lines
#if IMG_DEBUG
Cv2.ImShow("horizontal", horizontal);
Cv2.WaitKey(0);
#endif
bit = OCS.Extensions.BitmapConverter.ToBitmap(horizontal);
bit.Save("horizontal.tiff", System.Drawing.Imaging.ImageFormat.Tiff);
// Specify size on vertical axis
int verticalsize = vertical.Rows / scale;
// Create structure element for extracting vertical lines through morphology operations
//Mat verticalStructure = getStructuringElement(MORPH_RECT, Size(1, verticalsize));
Mat verticalStructure = Cv2.GetStructuringElement(MorphShapes.Rect, new OCS.Size(1, verticalsize));
// Apply morphology operations
//erode(vertical, vertical, verticalStructure, Point(-1, -1));
Cv2.Erode(vertical, vertical, verticalStructure, new OCS.Point(-1, -1));
//dilate(vertical, vertical, verticalStructure, Point(-1, -1));
Cv2.Dilate(vertical, vertical, verticalStructure, new OCS.Point(-1, -1));
// Show extracted vertical lines
#if IMG_DEBUG
Cv2.ImShow("vertical", vertical);
Cv2.WaitKey(0);
#endif
bit = OCS.Extensions.BitmapConverter.ToBitmap(vertical);
bit.Save("vertical.tiff", System.Drawing.Imaging.ImageFormat.Tiff);
// create a mask which includes the tables
Mat mask = horizontal + vertical;
#if IMG_DEBUG
Cv2.ImShow("mask", mask);
Cv2.WaitKey(0);
#endif
// find the joints between the lines of the tables, we will use this information in order to descriminate tables from pictures (tables will contain more than 4 joints while a picture only 4 (i.e. at the corners))
Mat joints = new Mat();
//bitwise_and(horizontal, vertical, joints);
Cv2.BitwiseAnd(horizontal, vertical, joints);
//Cv2.ImShow("joints", joints);
bit = OCS.Extensions.BitmapConverter.ToBitmap(joints);
bit.Save("joints.tiff", System.Drawing.Imaging.ImageFormat.Tiff);
#if IMG_DEBUG
Cv2.ImShow("a", joints);
Cv2.WaitKey(0);
#endif
//Thread.Sleep(2000);
// Find external contours from the mask, which most probably will belong to tables or to images
//vector<Vec4i> hierarchy;
//std::vector<std::vector<cv::Point>> contours;
OCS.HierarchyIndex[] hierarchy;
//List<List<OCS.Point>> contours = new List<List<OCS.Point>>;
OCS.Point[][] contours;
//cv::findContours(mask, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
Cv2.FindContours(mask, out contours, out hierarchy, OCS.RetrievalModes.External, OCS.ContourApproximationModes.ApproxSimple, new OCS.Point(0, 0));
//////vector<vector<Point>> contours_poly(contours.size() );
//////vector<Rect> boundRect(contours.size() );
//////vector<Mat> rois;
List <List <OCS.Point> > contours_poly = new List <List <OCS.Point> >(contours.Length);
List <OCS.Rect> boundRect = new List <OCS.Rect>(contours.Length);
List <Mat> rois = new List <Mat>();
////for (size_t i = 0; i < contours.size(); i++)
////{
//// // find the area of each contour
//// double area = contourArea(contours[i]);
///
//// // // filter individual lines of blobs that might exist and they do not represent a table
//// if (area < 100) // value is randomly chosen, you will need to find that by yourself with trial and error procedure
//// continue;
//// approxPolyDP(Mat(contours[i]), contours_poly[i], 3, true);
//// boundRect[i] = boundingRect(Mat(contours_poly[i]));
//// // find the number of joints that each table has
//// Mat roi = joints(boundRect[i]);
//// vector<vector<Point>> joints_contours;
//// findContours(roi, joints_contours, RETR_CCOMP, CHAIN_APPROX_SIMPLE);
//// // if the number is not more than 5 then most likely it not a table
//// if (joints_contours.size() <= 4)
//// continue;
//// rois.push_back(rsz(boundRect[i]).clone());
//// //drawContours( rsz, contours, i, Scalar(0, 0, 255), CV_FILLED, 8, vector<Vec4i>(), 0, Point() );
//// rectangle(rsz, boundRect[i].tl(), boundRect[i].br(), Scalar(0, 255, 0), 1, 8, 0);
////}
for (int i = 0; i < contours.Length; i++)
{
double area = Cv2.ContourArea(contours[i]);
if (area < 100.0)
{
// Skip because its not likely such a small area is a cell
continue;
}
// contours_poly is null at runtime. so we create a new entry and exit array
contours_poly.Add(new List <OCS.Point>());
OutputArray contour_poly_output = OutputArray.Create(contours_poly[i]);
InputArray contour_poly_input = InputArray.Create(contours[i]);
Cv2.ApproxPolyDP(InputArray.Create(contours[i]), contour_poly_output, 0.0, true);
Rect boundingRect = Cv2.BoundingRect(InputArray.Create(contours_poly[i]));
boundRect.Add(boundingRect);
//boundRect[i] = Cv2.BoundingRect(InputArray.Create(contours_poly[i]));
//OCS.Mat roi = Cv2.joints()
}
#if IMG_DEBUG
Cv2.NamedWindow("Output", WindowMode.KeepRatio);
Cv2.Rectangle(rsz, boundRect.ElementAt(0), Scalar.Red, 10);
Cv2.ImShow("Output", rsz);
Cv2.WaitKey(0);
Cv2.DestroyAllWindows();
#endif
////for (size_t i = 0; i < rois.size(); ++i)
////{
//// /* Now you can do whatever post process you want
//// * with the data within the rectangles/tables. */
//// imshow("roi", rois[i]);
//// waitKey();
////}
return(boundRect.Count);
}
public bool Test()
{
var plMask = new PLMask();
Mat whiteLight = Cv2.ImRead(@"P:\Projects\N3 Imaging\Images\09042018_jewerly\WhiteLight\whitelight.bmp");
//get phos contours
var rootDir = @"P:\Projects\N3 Imaging\Images\09042018_jewerly\Phos";
var files = Directory.GetFiles(rootDir, "*.bmp", SearchOption.TopDirectoryOnly);
Mat combinedMask = null;
foreach (var file in files)
{
var pl = Cv2.ImRead(file);
Mat mask;
var res = plMask.PlMask(pl, 20, out mask);
if (combinedMask == null)
{
combinedMask = Mat.Zeros(mask.Size(), mask.Type());
}
Cv2.Add(mask, combinedMask, combinedMask);
}
Mat element = Cv2.GetStructuringElement(MorphShapes.Ellipse,
new OpenCvSharp.Size(9, 9),
new OpenCvSharp.Point(2, 2));
Cv2.Dilate(combinedMask, combinedMask, element);
//find contours on this mask
Mat[] contours;
var hierarchy = new List <Point>();
Cv2.FindContours(combinedMask, out contours, OutputArray.Create(hierarchy), RetrievalModes.External,
ContourApproximationModes.ApproxSimple);
//remove small size contours
List <Mat> phosContours = new List <Mat>();
foreach (var contour in contours)
{
if (Cv2.ContourArea(contour) > 400)
{
phosContours.Add(contour);
}
}
//Mat phosMask = Mat.Zeros(combinedMask.Size(), MatType.CV_8UC1);
//Cv2.DrawContours(phosMask, phosContours, -1, Scalar.White, -1);//filled contours
//get centers of contours
for (int i = 0; i < phosContours.Count; i++)
{
var c = phosContours[i];
var m = c.Moments(true);
var x = m.M10 / m.M00;
var y = m.M01 / m.M00;
Cv2.DrawContours(whiteLight, phosContours, i, new Scalar(255, 0, 0), 4);
Cv2.PutText(whiteLight, "P", new Point(x, y), HersheyFonts.HersheySimplex, 1, new Scalar(0, 0, 255), 4);
}
//get fl contours
rootDir = @"P:\Projects\N3 Imaging\Images\09042018_jewerly\SW Fl";
files = Directory.GetFiles(rootDir, "*.bmp", SearchOption.TopDirectoryOnly);
Mat combinedFlMask = null;
foreach (var file in files)
{
var pl = Cv2.ImRead(file);
Mat mask;
var res = plMask.PlMask(pl, 20, out mask);
if (combinedFlMask == null)
{
combinedFlMask = Mat.Zeros(mask.Size(), mask.Type());
}
Cv2.Add(mask, combinedFlMask, combinedFlMask);
}
Cv2.ImShow("combinedFlMask", combinedFlMask);
Cv2.WaitKey(0);
Cv2.DestroyWindow("combinedFlMask");
element = Cv2.GetStructuringElement(MorphShapes.Ellipse,
new OpenCvSharp.Size(9, 9),
new OpenCvSharp.Point(2, 2));
Cv2.Dilate(combinedFlMask, combinedFlMask, element);
//find contours on this mask
Cv2.FindContours(combinedFlMask, out contours, OutputArray.Create(hierarchy), RetrievalModes.External,
ContourApproximationModes.ApproxSimple);
//remove small size contours
List <Mat> flContours = new List <Mat>();
foreach (var contour in contours)
{
if (Cv2.ContourArea(contour) > 400)
{
flContours.Add(contour);
}
}
//Mat flMask = Mat.Zeros(combinedFlMask.Size(), MatType.CV_8UC1);
//Cv2.DrawContours(flMask, flContours, -1, Scalar.White, -1);//filled contours
//check for intersection with phos contours
//if no intersection then label them
for (int i = 0; i < flContours.Count; i++)
{
Mat blankFl = Mat.Zeros(whiteLight.Size(), MatType.CV_8UC1);
Cv2.DrawContours(blankFl, flContours, i, Scalar.White, -1);
bool phos = false;
for (int j = 0; j < phosContours.Count; j++)
{
Mat blankPhos = Mat.Zeros(whiteLight.Size(), MatType.CV_8UC1);
Cv2.DrawContours(blankPhos, phosContours, j, Scalar.White, -1);
Mat intersection = new Mat();
Cv2.BitwiseAnd(blankFl, blankPhos, intersection);
//Cv2.ImShow("blankFl", blankFl);
//Cv2.ImShow("blankPhos", blankPhos);
//Cv2.ImShow("intersection", intersection);
//Cv2.WaitKey();
if (intersection.Sum()[0] > 0)
{
phos = true;
break;
}
}
if (!phos)
{
var c = flContours[i];
var m = c.Moments(true);
var x = m.M10 / m.M00;
var y = m.M01 / m.M00;
Cv2.DrawContours(whiteLight, flContours, i, new Scalar(255, 0, 0), 4);
Cv2.PutText(whiteLight, "F", new Point(x, y), HersheyFonts.HersheySimplex, 1, new Scalar(0, 0, 255), 4);
}
}
Cv2.ImShow("whiteLight", whiteLight);
Cv2.WaitKey();
Cv2.DestroyAllWindows();
return(true);
}
private void CreateObjectMask(ref System.Drawing.Bitmap image, out System.Drawing.Bitmap mask)
{
double kThresh = 125;
double hThresh = 55;
double canny1 = 25;
double canny2 = 75;
mask = null;
Mat src = BitmapConverter.ToMat(image);
Mat src_kirsch = BitmapConverter.ToMat(image.KirschFilter());
Mat kirsch_gray = new Mat();
Cv2.CvtColor(src_kirsch, kirsch_gray, ColorConversionCodes.RGB2GRAY);
Cv2.ImWrite(@"C:\temp\kirsch_gray.jpg", kirsch_gray);
Mat kirsch_threshold = new Mat();
Cv2.Threshold(kirsch_gray, kirsch_threshold, kThresh, 255, ThresholdTypes.Binary);
Cv2.ImWrite(@"C:\temp\kirsch_threshold.jpg", kirsch_threshold);
Mat[] contours;
List <OpenCvSharp.Point> hierarchy;
List <Mat> hulls;
Mat morph_element = Cv2.GetStructuringElement(MorphShapes.Ellipse, new OpenCvSharp.Size(2, 2), new OpenCvSharp.Point(1, 1));
#region morphology
Mat kirsch_threshold_copy = new Mat();
kirsch_threshold.CopyTo(kirsch_threshold_copy);
int hullCount = 0, numLoops = 0;
do
{
numLoops++;
Mat kirsch_morph = kirsch_threshold_copy.MorphologyEx(MorphTypes.Gradient, morph_element);
Cv2.ImWrite(@"C:\temp\kirsch_morph" + numLoops + ".jpg", kirsch_morph);
hierarchy = new List <OpenCvSharp.Point>();
Cv2.FindContours(kirsch_morph, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
hulls = new List <Mat>();
for (int j = 0; j < contours.Length; j++)
{
Mat hull = new Mat();
Cv2.ConvexHull(contours[j], hull);
hulls.Add(hull);
}
Mat drawing = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(drawing, hulls, -1, Scalar.White);
Cv2.ImWrite(@"C:\temp\drawing" + numLoops + ".jpg", drawing);
if (hulls.Count != hullCount && numLoops < 100)
{
hullCount = hulls.Count;
kirsch_threshold_copy = drawing;
}
else
{
break;
}
} while (true);
#endregion
if (numLoops >= 100)
{
throw new Exception("Could not find hull");
}
#region bestHull
//try and filter out dust near to stone
double largestArea = hulls.Max(m => Cv2.ContourArea(m));
var bestHulls = hulls.Where(m => Cv2.ContourArea(m) == largestArea).ToList();
Mat hulls_mask = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(hulls_mask, bestHulls, -1, Scalar.White, -1);
Cv2.ImWrite(@"C:\temp\hulls_mask.jpg", hulls_mask);
//hulls_mask is the convex hull of outline, now look for clefts
Cv2.Threshold(kirsch_gray, kirsch_threshold, hThresh, 255, ThresholdTypes.Binary);
Mat kirsch_mask = Mat.Zeros(kirsch_threshold.Size(), kirsch_threshold.Type());
kirsch_threshold.CopyTo(kirsch_mask, hulls_mask);
Cv2.ImWrite(@"C:\temp\kirsch_mask.jpg", kirsch_mask);
Mat kirsch_mask_canny = new Mat();
Cv2.Canny(kirsch_mask, kirsch_mask_canny, canny1, canny2, 3);
Cv2.ImWrite(@"C:\temp\kirsch_mask_canny.jpg", kirsch_mask_canny);
morph_element = Cv2.GetStructuringElement(MorphShapes.Ellipse, new OpenCvSharp.Size(5, 5), new OpenCvSharp.Point(2, 2));
Mat kirsch_filled = new Mat();
Cv2.Dilate(kirsch_mask_canny, kirsch_filled, morph_element);
Cv2.Dilate(kirsch_filled, kirsch_filled, morph_element);
Cv2.Erode(kirsch_filled, kirsch_filled, morph_element);
Cv2.Erode(kirsch_filled, kirsch_filled, morph_element);
Cv2.ImWrite(@"C:\temp\kirsch_filled.jpg", kirsch_filled);
hierarchy = new List <OpenCvSharp.Point>();;
Cv2.FindContours(kirsch_filled, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
#endregion
hulls_mask = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(hulls_mask, contours, -1, Scalar.White, -1);
Cv2.ImWrite(@"C:\temp\hulls_mask_final.jpg", hulls_mask);
Cv2.Erode(hulls_mask, hulls_mask, morph_element);
Cv2.Erode(hulls_mask, hulls_mask, morph_element);
Mat fImage = new Mat();
src.CopyTo(fImage, hulls_mask);
Cv2.ImWrite(@"C:\temp\fImage.png", fImage);
mask = BitmapConverter.ToBitmap(fImage);
// contrast calculation
double contrast, L, a, b, H, C;
CalculateContrast(ref src, ref hulls_mask, out L, out a, out b, out contrast, out H, out C, ContrastType.RMS);
Console.WriteLine("contrast: {0}, L: {1}, a: {2}, b: {3}, C: {4}, H: {5}", contrast, L, a, b, C, H);
//hulls_mask = null;
CalcHistogram(ref src, ref hulls_mask, out mask);
}
private Bitmap CreateObjectMaskMelee(Bitmap image, out Mat image_mask,
out double mask_length, out double mask_area, out double mask_width, out double mask_height,
out double mask_pvheight, bool useKthresholdLab = false)
{
Bitmap dst = null;
image_mask = null;
mask_length = mask_area = mask_width = mask_height = mask_pvheight = 0;
try
{
Mat src = BitmapConverter.ToMat(image);
Mat src_kirsch = BitmapConverter.ToMat(image.KirschFilter());
Mat kirsch_gray = new Mat();
Cv2.CvtColor(src_kirsch, kirsch_gray, ColorConversionCodes.RGB2GRAY);
Mat kirsch_threshold = new Mat();
if (!useKthresholdLab)
{
Cv2.Threshold(kirsch_gray, kirsch_threshold, kThreshold, 255, ThresholdTypes.Binary);
}
else
{
Cv2.Threshold(kirsch_gray, kirsch_threshold, kThresholdLab, 255, ThresholdTypes.Binary);
}
Mat[] contours;
List <OpenCvSharp.Point> hierarchy;
List <Mat> hulls;
Mat morph_element = Cv2.GetStructuringElement(MorphShapes.Ellipse, new OpenCvSharp.Size(2, 2), new OpenCvSharp.Point(1, 1));
#region morphology
Mat kirsch_threshold_copy = new Mat();
kirsch_threshold.CopyTo(kirsch_threshold_copy);
int hullCount = 0, numLoops = 0;
do
{
numLoops++;
Mat kirsch_morph = kirsch_threshold_copy.MorphologyEx(MorphTypes.Gradient, morph_element);
hierarchy = new List <OpenCvSharp.Point>();
Cv2.FindContours(kirsch_morph, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
hulls = new List <Mat>();
for (int j = 0; j < contours.Length; j++)
{
Mat hull = new Mat();
Cv2.ConvexHull(contours[j], hull);
hulls.Add(hull);
}
Mat drawing = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(drawing, hulls, -1, Scalar.White);
if (hulls.Count != hullCount && numLoops < 100)
{
hullCount = hulls.Count;
kirsch_threshold_copy = drawing;
}
else
{
break;
}
} while (true);
#endregion
if (numLoops >= 100)
{
throw new Exception("Could not find hull");
}
#region bestHull
//try and filter out dust near to stone
double largestArea = hulls.Max(m => Cv2.ContourArea(m));
var bestHulls = hulls.Where(m => Cv2.ContourArea(m) == largestArea).ToList();
Mat hulls_mask = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(hulls_mask, bestHulls, -1, Scalar.White, -1);
//hulls_mask is the convex hull of main outline excluding nearby dust
Cv2.Threshold(kirsch_gray, kirsch_threshold, hullThreshold, 255, ThresholdTypes.Binary);
Mat kirsch_mask = Mat.Zeros(kirsch_threshold.Size(), kirsch_threshold.Type());
kirsch_threshold.CopyTo(kirsch_mask, hulls_mask);
#endregion
hierarchy = new List <OpenCvSharp.Point>();;
Cv2.FindContours(kirsch_mask, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External, ContourApproximationModes.ApproxSimple, new OpenCvSharp.Point(0, 0));
List <OpenCvSharp.Point> points = new List <OpenCvSharp.Point>();
foreach (Mat contour in contours)
{
int m2Count = (contour.Rows % 2 > 0) ? contour.Rows + 1 : contour.Rows;
OpenCvSharp.Point[] p2 = new OpenCvSharp.Point[m2Count];
contour.GetArray(0, 0, p2);
Array.Resize(ref p2, contour.Rows);
points.AddRange(p2.ToList());
}
Mat finalHull = new Mat();
Cv2.ConvexHull(InputArray.Create(points), finalHull);
List <Mat> finalHulls = new List <Mat>();
finalHulls.Add(finalHull);
Cv2.DrawContours(src, finalHulls, -1, new Scalar(128, 0, 128, 255), 2);
hulls_mask = Mat.Zeros(src.Size(), MatType.CV_8UC1);
Cv2.DrawContours(hulls_mask, finalHulls, -1, Scalar.White, -1);
image_mask = hulls_mask;
#region bounding
Mat poly = new Mat();
Cv2.ApproxPolyDP(finalHull, poly, 3, true);
Rect boundaryRect = Cv2.BoundingRect(poly);
mask_width = boundaryRect.Width;
mask_height = boundaryRect.Height;
mask_area = Cv2.ContourArea(poly);
mask_length = Cv2.ArcLength(finalHull, true);
List <OpenCvSharp.Point> finalPoints = new List <OpenCvSharp.Point>();
int m1Count = (finalHull.Rows % 2 > 0) ? finalHull.Rows + 1 : finalHull.Rows;
OpenCvSharp.Point[] p1 = new OpenCvSharp.Point[m1Count];
finalHull.GetArray(0, 0, p1);
Array.Resize(ref p1, finalHull.Rows);
finalPoints.AddRange(p1.ToList());
double y_min = boundaryRect.Bottom;
double y_x_min = finalPoints.Where(p => p.X == boundaryRect.Left).ToList()[0].Y;
double y_x_max = finalPoints.Where(p => p.X == boundaryRect.Right).ToList()[0].Y;
mask_pvheight = ((double)y_x_max + (double)y_x_min) / 2 - (double)y_min;
#endregion
//dst = BitmapConverter.ToBitmap(src);
using (var ms = src.ToMemoryStream())
{
dst = (Bitmap)Image.FromStream(ms);
}
try
{
if (saveMaskDataPath.Length > 0)
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("mask_length,mask_area,mask_width,mask_height,mask_pvheight");
sb.AppendLine(mask_length + "," + mask_area + "," + mask_width + "," + mask_height + "," + mask_pvheight);
image_mask.SaveImage(saveMaskDataPath + @"\image_mask.jpg");
File.WriteAllText(saveMaskDataPath + @"\mask_vals.csv", sb.ToString());
}
}
catch
{
}
}
catch
{
dst = null;
}
return(dst);
}
private void buttonSelectImage1_Click(object sender, EventArgs e)
{
OpenFileDialog openFileDialog1 = new OpenFileDialog();
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
try
{
List <OpenCvSharp.Point> rois = new List <OpenCvSharp.Point>(new OpenCvSharp.Point[] {
new OpenCvSharp.Point(35, 90), new OpenCvSharp.Point(400, 90), new OpenCvSharp.Point(850, 90), new OpenCvSharp.Point(1290, 90),
new OpenCvSharp.Point(35, 400), new OpenCvSharp.Point(400, 400), new OpenCvSharp.Point(850, 400), new OpenCvSharp.Point(1290, 400),
new OpenCvSharp.Point(35, 800), new OpenCvSharp.Point(400, 800), new OpenCvSharp.Point(850, 800), new OpenCvSharp.Point(1290, 800),
new OpenCvSharp.Point(35, 1230), new OpenCvSharp.Point(400, 1230), new OpenCvSharp.Point(850, 1230), new OpenCvSharp.Point(1290, 1230)
});
var files = Directory.GetFiles(Path.GetDirectoryName(openFileDialog1.FileName), "*.png").ToList();
if (files != null && files.Count > 0)
{
int index = files.IndexOf(openFileDialog1.FileName);
//using (var win = new Window())
//using (var win1 = new Window())
{
while (true)
{
ListBox listBoxTemp = new ListBox();
listBoxTemp.Items.Clear();
Debug.WriteLine(files[index]);
Mat img = new Mat(files[index]);//load color image
Mat gray = new Mat(), laplacian = new Mat(), sobel = new Mat(), canny = new Mat();
Cv2.CvtColor(img, gray, ColorConversionCodes.BGR2GRAY);
//Cv2.GaussianBlur(gray, gray, new OpenCvSharp.Size(3, 3), 0);
Cv2.Canny(gray, canny, 100, 300, 3);
Cv2.NamedWindow("img", WindowMode.Normal);
Cv2.NamedWindow("hulls", WindowMode.Normal);
Cv2.ImShow("img", img);
Mat[] contours1;
var hierarchy1 = new List <OpenCvSharp.Point>();
Cv2.FindContours(canny, out contours1, OutputArray.Create(hierarchy1),
RetrievalModes.External,
ContourApproximationModes.ApproxSimple);
var hulls1 = new List <Mat>();
for (int j = 0; j < contours1.Length; j++)
{
Mat hull = new Mat();
Cv2.ConvexHull(contours1[j], hull);
if (hull.ContourArea() > 100)
{
hulls1.Add(hull);
}
}
if (hulls1.Count > 0)
{
Mat drawing = Mat.Zeros(canny.Size(), MatType.CV_8UC1);
Cv2.DrawContours(drawing, hulls1, -1, Scalar.White, -1);
Cv2.ImShow("hulls", drawing);
}
goto wait;
Cv2.Laplacian(gray, laplacian, MatType.CV_32F);
foreach (var pt in rois)
{
Rect roi = new Rect(pt, new OpenCvSharp.Size(200, 200));
Mat original = img.Clone(roi);
Mat temp = laplacian.Clone(roi);
MatOfFloat matf = new MatOfFloat(temp);
var indexer = matf.GetIndexer();
List <double> sums = new List <double>();
double sum = 0;
for (int y = 0; y < 25; y++)
{
for (int x = 0; x < 25; x++)
{
float val = indexer[y, x];
if (val >= 1)
{
sum += val;
}
}
}
sums.Add(sum);
sum = 0;
for (int y = 0; y < 25; y++)
{
for (int x = temp.Width - 25; x < temp.Width; x++)
{
float val = indexer[y, x];
if (val >= 1)
{
sum += val;
}
}
}
sums.Add(sum);
sum = 0;
for (int y = temp.Height - 25; y < temp.Height; y++)
{
for (int x = temp.Width - 25; x < temp.Width; x++)
{
float val = indexer[y, x];
if (val >= 1)
{
sum += val;
}
}
}
sums.Add(sum);
sum = 0;
for (int y = temp.Height - 25; y < temp.Height; y++)
{
for (int x = 0; x < 25; x++)
{
float val = indexer[y, x];
if (val >= 1)
{
sum += val;
}
}
}
sums.Add(sum);
sum = 0;
for (int y = 88; y < 113; y++)
{
for (int x = 88; x < 113; x++)
{
float val = indexer[y, x];
if (val >= 1)
{
sum += val;
}
}
}
sums.Add(sum);
double avg = sums.Average();
double stdev = CalculateStdDev(sums);
int upper = (int)(avg + 1.5 * stdev);
int lower = (int)(avg - 1.5 * stdev);
listBoxTemp.Items.Add(pt.ToString() + ": " + String.Join(",", sums.ToArray())
+ "," + upper + "," + lower);
if (sums[4] > upper || sums[4] < lower)
{
//Cv2.ImShow(pt.ToString(), temp);
temp.ConvertTo(temp, MatType.CV_8UC1);
Mat [] contours;
var hierarchy = new List <OpenCvSharp.Point>();
Cv2.FindContours(temp, out contours, OutputArray.Create(hierarchy),
RetrievalModes.External,
ContourApproximationModes.ApproxSimple);
var hulls = new List <Mat>();
for (int j = 0; j < contours.Length; j++)
{
Mat hull = new Mat();
Cv2.ConvexHull(contours[j], hull);
if (hull.ContourArea() > 1000)
{
hulls.Add(hull);
}
}
if (hulls.Count > 0)
{
Mat drawing = Mat.Zeros(temp.Size(), MatType.CV_8UC1);
Cv2.DrawContours(drawing, contours, -1, Scalar.White, -1);
Cv2.ImShow(pt.ToString() + " hulls", drawing);
/*
* List<OpenCvSharp.Point> points = new List<OpenCvSharp.Point>();
* foreach (Mat hull in hulls)
* {
* int m2Count = (hull.Rows % 2 > 0) ? hull.Rows + 1 : hull.Rows;
* OpenCvSharp.Point[] p2 = new OpenCvSharp.Point[m2Count];
* hull.GetArray(0, 0, p2);
* Array.Resize(ref p2, hull.Rows);
*
* points.AddRange(p2.ToList());
* }
* Mat finalHull = new Mat();
* Cv2.ConvexHull(InputArray.Create(points), finalHull);
*
* if (finalHull.ContourArea() > 1000)
* {
* List<Mat> finalHulls = new List<Mat>();
* finalHulls.Add(finalHull);
* Mat mask = Mat.Zeros(temp.Size(), MatType.CV_8UC1);
* Cv2.DrawContours(mask, finalHulls, -1, Scalar.White, -1);
*
* Cv2.ImShow(pt.ToString(), temp);
* Cv2.ImShow(pt.ToString() + " original", original);
* Cv2.ImShow(pt.ToString() + " mask", mask);
* //Mat masked = Mat.Zeros(original.Size(), original.Type()) ;
* //masked.CopyTo(original, mask);
* //Cv2.ImShow(pt.ToString() + " masked", masked);
* }
*/
}
}
}
wait:
int c = Cv2.WaitKey(0);
if (c == 'n' && index < files.Count - 1)
{
index++;
}
else if (c == 'p' && index > 0)
{
index--;
}
else if (c == 27)
{
break;
}
else
{
index = 0;
}
Window.DestroyAllWindows();
}
}
}
}
catch (Exception ex)
{
MessageBox.Show("Error: " + ex.Message);
}
}
}
void Start()
{
//thread = new Thread(new ThreadStart(ThreadMainFunc));
//thread = new Thread(ThreadMainFunc);
//thread.IsBackground = true;
//thread = Loom.RunAsync(ThreadMainFunc);
//thread.Start();
//textTex = Resources.Load<Texture2D>("test-IMG_0204-text");
//rawImageRI.texture = textTex;
//TextAsset binary = (TextAsset)AssetDatabase.LoadAssetAtPath("Assets/img1.bytes", typeof(TextAsset));
//inputTex = Resources.Load<Texture2D>("test-IMG_0204");
//rawImageRI.texture = inputTex;
//Debug.Log("inputTex.width: "+inputTex.width);
//Debug.Log("inputTex.height: "+inputTex.height);
//tex.LoadImage(binary.bytes);
//Texture2D tex = (Texture2D)AssetDatabase.LoadAssetAtPath("Assets/img.PNG", typeof(Texture2D));
//StartCoroutine(GetTextImg());
//Size texSize = new Size(tex.width, tex.height);
//Mat mat = new Mat(texSize, CvType.CV_8UC4);
//Utils.texture2DToMat(tex, mat);
//inputImg = Cv2.ImRead(imgPath);
//Cv2.ImShow("inputImg", inputImg);
//tex = Unity.MatToTexture(inputImg);
//rawImageRI.texture = tex;
/*inputTex = new Texture2D(2,2);
* string imgPath = "../signboard-rectangle/test-IMG_0204.PNG";
* byte [] binaryImageData = File.ReadAllBytes(imgPath);
* inputTex.LoadImage(binaryImageData);*/
//inputTex = Resources.Load<Texture2D>("forAddText");
//必要 防止記憶體爆炸
Texture2D inputTex1 = TextureGray.ToGray("1");
Texture2D inputTex2 = TextureGray.ToGray("2");
//Texture2D inputTex3 = TextureGray.ToGray("3");
//Debug.Log("inputTex.width: "+inputTex.width);
//Debug.Log("inputTex.height: "+inputTex.height);
//rawImageRI.texture = inputTex;
OpenCvSharp.Mat inputImg1 = Unity.TextureToMat(inputTex1);
OpenCvSharp.Mat inputImg2 = Unity.TextureToMat(inputTex2);
//OpenCvSharp.Mat inputImg3 = Unity.TextureToMat(inputTex3);
//OpenCvSharp.Mat inputImg2 = Unity.TextureToMat(inputTex);
//Cv2.ImShow("img", inputImg);
InputArray img1 = InputArray.Create(inputImg1);
InputArray img2 = InputArray.Create(inputImg2);
//InputArray img3 = InputArray.Create(inputImg3);
//Debug.Log("inputImg: "+inputImg.ToString());
//InputArray mask = null;
//OpenCvSharp.KeyPoint[] kp1 = null;
des1 = OutputArray.Create(inputImg1);
des2 = OutputArray.Create(inputImg2);
//des3 = OutputArray.Create(inputImg3);
//Debug.Log("des1: "+des1);
//Initiate SIFT detector and extractor
//siftDetect = FeatureDetector.create(3);
//siftExtract = DescriptorExtractor.create(1);
sift = SIFT.Create();
//surf = SURF.Create((double)100);
//orb = OpenCvSharp.ORB.Create();
//brief = OpenCvSharp.XFeatures2D.BriefDescriptorExtractor.Create();
//if image too large will cause app Terminated due to memory error
kp1 = sift.Detect(inputImg1);
kp2 = sift.Detect(inputImg2);
//kp3 = sift.Detect(inputImg3);
//kp1 = surf.Detect(inputImg);
//kp1 = orb.Detect(inputImg);
//kp1 = brief.Detect(inputImg);
//Cv2.ImShow("img", inputImg); ok
sift.Compute(img1, ref kp1, des1);
sift.Compute(img2, ref kp2, des2);
//sift.Compute(img3, ref kp3, des3);
//surf.Compute(img1, ref kp1, des1);
//orb.Compute(img1, ref kp1, des1);
//brief.Compute(img1, ref kp1, des1);
//Cv2.ImShow("img", inputImg); 亂碼圖
//Cv2.ImShow("img", inputImg2); ok
//foreach (OpenCvSharp.KeyPoint kp in kp1)
// Debug.Log("kp: "+kp.ToString());
//用flannbased的話unity會掛掉
descriptorMatcher = OpenCvSharp.DescriptorMatcher.Create("BruteForce");
//sift.DetectAndCompute(img1, mask, out kp1, des1);
//MatOfKeyPoint kp1 = new MatOfKeyPoint();
//Mat des1 = new Mat();
//siftDetect.detect(inputImg, kp1);
//siftExtract.compute(inputImg, kp1, des1);
//StartCoroutine(OpenCamera());//開啟攝影機鏡頭
//StartCoroutine(CalculateHomography());
/*Texture2D sourceTex = ScreenCapture.CaptureScreenshotAsTexture();
* Color[] pix = sourceTex.GetPixels((int)rectBotLeft.x, (int)rectBotLeft.y, width, height);
* tex = new Texture2D(width, height);
* tex.SetPixels(pix);
* tex.Apply();
*
* tex = TextureGray.ToGray(tex);
*
* mat = Unity.TextureToMat(tex);
*
* InputArray img2 = InputArray.Create(mat);
* desCam = OutputArray.Create(mat);
*
* kpCam = sift.Detect(mat);*/
}
IEnumerator CalculateHomography()
//void CalculateHomography()
{
//Debug.Log("CalculateHomography1");
//myCam.Pause();
yield return(new WaitForEndOfFrame());
//yield return new WaitForSeconds((float)0.5);
//程式開始後至少要等0.3秒才會出現影像畫面，不然算sift一開始就會記憶體爆掉
//input camera image
/*Texture2D sourceTex = ScreenCapture.CaptureScreenshotAsTexture();
* Color[] pix = sourceTex.GetPixels((int)rectBotLeft.x, (int)rectBotLeft.y, width, height);
* Texture2D tex = new Texture2D(width, height);
* tex.SetPixels(pix);
* tex.Apply();*/
//Debug.Log("CalculateHomography2");
//rawimage position at (0,0)，start from bottom left
int xStart = (int)(Screen.width - rawImageRT.rect.width) / 2;
int yStart = (int)(Screen.height - rawImageRT.rect.height) / 2;
/*Debug.Log("xStart: "+xStart);
* Debug.Log("yStart: "+yStart);
* Debug.Log("Screen.width: "+Screen.width);
* Debug.Log("Screen.height: "+Screen.height);
* Debug.Log("rawImageRT.rect.width: "+rawImageRT.rect.width);
* Debug.Log("rawImageRT.rect.height: "+rawImageRT.rect.height);*/
//get sign image with text
Texture2D sourceTex = ScreenCapture.CaptureScreenshotAsTexture();
//rawImageRI.texture = sourceTex;
//Color[] pix = sourceTex.GetPixels((int)rectBotLeft.x, (int)rectBotLeft.y, width, height);
Color[] pix = sourceTex.GetPixels(xStart, yStart, (int)rawImageRT.rect.width, (int)rawImageRT.rect.height);
tex = new Texture2D((int)rawImageRT.rect.width, (int)rawImageRT.rect.height);
tex.SetPixels(pix);
tex.Apply();
//Debug.Log("tex.width: "+tex.width);
//Debug.Log("tex.height: "+tex.height);
//input fixed image
/*Texture2D tex = new Texture2D(2,2);
* string imgPath = "../signboard-rectangle/test-199-fast-628.jpg";
* byte [] binaryImageData = File.ReadAllBytes(imgPath);
* tex.LoadImage(binaryImageData);*/
//scale texture to make it smaller
TextureScale.Bilinear(tex, tex.width / 2, tex.height / 2);
//必要 防止記憶體爆炸
tex = TextureGray.ToGray(tex);
//rawImageRI.texture = tex;
mat = Unity.TextureToMat(tex);
Destroy(sourceTex);
Destroy(tex);
//Cv2.ImShow("img", mat); ok
//OpenCvSharp.Mat mat = Cv2.ImRead(imgPath, ImreadModes.Unchanged);
//Debug.Log("mat: "+mat.ToString());
//string imgPath = "../signboard-rectangle/test-199-fast-628.jpg";
//OpenCvSharp.Mat mat = Cv2.ImRead(imgPath);
InputArray imgCam = InputArray.Create(mat);
desCam = OutputArray.Create(mat);
//Cv2.ImShow("img", mat); ok
//OpenCvSharp.Mat mat2 = mat;
//sift = SIFT.Create();
//System.Diagnostics.Stopwatch time = new System.Diagnostics.Stopwatch();
//time.Start ();
//卡卡
OpenCvSharp.KeyPoint[] kpCam = sift.Detect(mat);
//OpenCvSharp.KeyPoint[] kpCam = surf.Detect(mat);
//OpenCvSharp.KeyPoint[] kpCam = orb.Detect(mat);
//OpenCvSharp.KeyPoint[] kpCam = brief.Detect(mat);
//time.Stop();
//Debug.Log("執行 " + time.Elapsed.TotalSeconds + " 秒");
//myCam.Pause();
//rawImageRI.texture = tex;
//Cv2.ImShow("img", mat); ok
//Cv2.ImShow("img", mat2); ok
sift.Compute(imgCam, ref kpCam, desCam);
//surf.Compute(img2, ref kpCam, desCam);
//orb.Compute(img2, ref kpCam, desCam);
//brief.Compute(img2, ref kpCam, desCam);
//Cv2.ImShow("img", mat);
//Cv2.ImShow("img", mat2); 爆炸
OpenCvSharp.Mat desCammat = desCam.GetMat();
//Debug.Log("desCammat: "+desCammat);
//if (!M) 如果還沒計算出homography M {
//desFirstCatch = desCam;
//OutputArray descriptors_object = des1;
OpenCvSharp.Mat des1mat = des1.GetMat();
OpenCvSharp.Mat des2mat = des2.GetMat();
//OpenCvSharp.Mat des3mat = des3.GetMat();
//Debug.Log("des1mat: "+des1mat);
OpenCvSharp.DMatch[] dmatch1 = descriptorMatcher.Match(des1mat, desCammat);
OpenCvSharp.DMatch[] dmatch2 = descriptorMatcher.Match(des2mat, desCammat);
//OpenCvSharp.DMatch[] dmatch3 = descriptorMatcher.Match(des3mat, desCammat);
//Debug.Log("damtch1[0]: "+dmatch1[0].ToString());
//}
//else {
//OpenCvSharp.Mat desFirstCatchmat = desFirstCatch.GetMat();
// OpenCvSharp.DMatch[] dmatch = descriptorMatcher.Match(desFirstCatchmat, desCammat);
// OutputArray descriptors_object = desFirstCatch;
//}
double max_dist1 = 0;
double min_dist1 = 100;
double max_dist2 = 0;
double min_dist2 = 100;
//double max_dist3 = 0;
//double min_dist3 = 100;
//Cv2.ImShow("img", mat); 爆炸
//Quick calculation of max and min distances between keypoints
foreach (OpenCvSharp.DMatch d in dmatch1)
{
double dist = d.Distance;
if (dist < min_dist1)
{
min_dist1 = dist;
}
if (dist > max_dist1)
{
max_dist1 = dist;
}
}
foreach (OpenCvSharp.DMatch d in dmatch2)
{
double dist = d.Distance;
if (dist < min_dist2)
{
min_dist2 = dist;
}
if (dist > max_dist2)
{
max_dist2 = dist;
}
}
/*foreach (OpenCvSharp.DMatch d in dmatch3){
* double dist = d.Distance;
* if( dist < min_dist3 ) min_dist3 = dist;
* if( dist > max_dist3 ) max_dist3 = dist;
* }*/
//Draw only "good" matches (i.e. whose distance is less than 3*min_dist )
List <OpenCvSharp.DMatch> goodMatch1 = new List <OpenCvSharp.DMatch>();
foreach (OpenCvSharp.DMatch d in dmatch1)
{
if (d.Distance < 3 * min_dist1)
{
goodMatch1.Add(d);
}
}
List <OpenCvSharp.DMatch> goodMatch2 = new List <OpenCvSharp.DMatch>();
foreach (OpenCvSharp.DMatch d in dmatch2)
{
if (d.Distance < 3 * min_dist2)
{
goodMatch2.Add(d);
}
}
/*List<OpenCvSharp.DMatch> goodMatch3 = new List<OpenCvSharp.DMatch>();
* foreach (OpenCvSharp.DMatch d in dmatch3){
* if( d.Distance < 3*min_dist3 )
* goodMatch3.Add(d);
* }*/
List <OpenCvSharp.Point2f> srcPts1 = new List <OpenCvSharp.Point2f>();
List <OpenCvSharp.Point2f> dstPts1 = new List <OpenCvSharp.Point2f>();
foreach (OpenCvSharp.DMatch d in goodMatch1)
{
//-- Get the keypoints from the good matches
srcPts1.Add(kp1[d.QueryIdx].Pt);
dstPts1.Add(kpCam[d.TrainIdx].Pt);
//Debug.Log("kp1[d.QueryIdx].Pt: "+kp1[d.QueryIdx].Pt);
}
List <OpenCvSharp.Point2f> srcPts2 = new List <OpenCvSharp.Point2f>();
List <OpenCvSharp.Point2f> dstPts2 = new List <OpenCvSharp.Point2f>();
foreach (OpenCvSharp.DMatch d in goodMatch2)
{
//-- Get the keypoints from the good matches
srcPts2.Add(kp2[d.QueryIdx].Pt);
dstPts2.Add(kpCam[d.TrainIdx].Pt);
//Debug.Log("kp1[d.QueryIdx].Pt: "+kp1[d.QueryIdx].Pt);
}
/*List<OpenCvSharp.Point2f> srcPts3 = new List<OpenCvSharp.Point2f>();
* List<OpenCvSharp.Point2f> dstPts3 = new List<OpenCvSharp.Point2f>();
* foreach (OpenCvSharp.DMatch d in goodMatch3){
* //-- Get the keypoints from the good matches
* srcPts3.Add(kp3[d.QueryIdx].Pt);
* dstPts3.Add(kpCam[d.TrainIdx].Pt);
* //Debug.Log("kp1[d.QueryIdx].Pt: "+kp1[d.QueryIdx].Pt);
* }*/
//jump to next iteration if less than certain number of keypoints matched
if (srcPts1.Count < 200 && srcPts2.Count < 200)
{
yield break;
}
if (srcPts1.Count >= srcPts2.Count)
{
srcPts = new List <OpenCvSharp.Point2f>(srcPts1);
dstPts = new List <OpenCvSharp.Point2f>(dstPts1);
text1.enabled = true;
text2.enabled = false;
num1++;
//text3.enabled = false;
}
/*else if(srcPts2.Count >= srcPts1.Count && srcPts2.Count >= srcPts3.Count){
* srcPts = new List<OpenCvSharp.Point2f>(srcPts2);
* dstPts = new List<OpenCvSharp.Point2f>(dstPts2);
* text2.enabled = true;
* text1.enabled = false;
* text3.enabled = false;
* }*/
else
{
srcPts = new List <OpenCvSharp.Point2f>(srcPts2);
dstPts = new List <OpenCvSharp.Point2f>(dstPts2);
text2.enabled = true;
text1.enabled = false;
num2++;
//text2.enabled = false;
}
if (num1 > num2 + 10)
{
text1.enabled = true;
text2.enabled = false;
}
if (num2 > num1 + 10)
{
text2.enabled = true;
text1.enabled = false;
}
if (num1 > 60 || num2 > 60)
{
num1 = 0;
num2 = 0;
}
//OpenCvSharp.Mat mat2 = mat;
//Cv2.DrawKeypoints(mat, kpCam, mat2);
//Cv2.ImShow("img", mat); 亂碼圖
//Texture2D tex2 = new Texture2D(8, 8);
//tex2 = Unity.MatToTexture(mat);
//rawImageRI.texture = tex2;
//myCam.Pause();
//Cv2.ImShow("img", mat2); 亂碼圖
Texture2D emptyTex = new Texture2D(8, 8);
OpenCvSharp.Mat outputImg = Unity.TextureToMat(emptyTex);
//Debug.Log("outputImg: "+outputImg.ToString());
InputArray srcArr = InputArray.Create <OpenCvSharp.Point2f>(srcPts);
InputArray dstArr = InputArray.Create <OpenCvSharp.Point2f>(dstPts);
OutputArray mask = OutputArray.Create(outputImg);
OpenCvSharp.Mat M = Cv2.FindHomography(srcArr, dstArr, HomographyMethods.Ransac, 5, mask);
OpenCVForUnity.Mat transMat = new OpenCVForUnity.Mat(3, 3, CvType.CV_32FC1);
transMat.put(0, 0, M.Get <double>(0, 0), M.Get <double>(0, 1), M.Get <double>(0, 2),
M.Get <double>(1, 0), M.Get <double>(1, 1), M.Get <double>(1, 2),
M.Get <double>(2, 0), M.Get <double>(2, 1), M.Get <double>(2, 2));
//Debug.Log("transMat: "+transMat.dump());
//Debug.Log("mask: "+mask);
//OpenCvSharp.Mat maskMat = mask.GetMat();
//Debug.Log("maskMat: "+maskMat.ToString());
//maskMoB = new OpenCvSharp.MatOfByte(maskMat);
//-- Get the corners from the image_1 ( the object to be "detected" )
/*OpenCvSharp.Point2f[] obj_corners = new OpenCvSharp.Point2f[4];
* obj_corners[0] = new OpenCvSharp.Point2f(0, 0);
* obj_corners[1] = new OpenCvSharp.Point2f(inputTex.width, 0);
* obj_corners[2] = new OpenCvSharp.Point2f(inputTex.width, inputTex.height);
* obj_corners[3] = new OpenCvSharp.Point2f(0, inputTex.height);
*
* //OpenCvSharp.Point2f[] scene_corners = new OpenCvSharp.Point2f[4];
* //scene_corners = Cv2.PerspectiveTransform(obj_corners, M);
*
* //if (!M) 如果還沒計算出homography M {
* //Cv2.DrawMatches(inputImg, kp1, mat, kpCam, goodMatch, outputImg, OpenCvSharp.Scalar.All(-1),
* //OpenCvSharp.Scalar.All(-1), maskMoB.ToArray(), DrawMatchesFlags.NotDrawSinglePoints);
* //else {
*
* //Texture2D outputTex = Unity.MatToTexture(outputImg);
* //rawImageRI.texture = outputTex;
*
* //-- Draw lines between the corners (the mapped object in the scene - image_2 )
* //Cv2.Line(outputImg, scene_corners[0] + obj_corners[1], scene_corners[1] + obj_corners[1], OpenCvSharp.Scalar.LightBlue, 4);
* //Cv2.Line(outputImg, scene_corners[1] + obj_corners[1], scene_corners[2] + obj_corners[1], OpenCvSharp.Scalar.LightBlue, 4);
* //Cv2.Line(outputImg, scene_corners[2] + obj_corners[1], scene_corners[3] + obj_corners[1], OpenCvSharp.Scalar.LightBlue, 4);
* //Cv2.Line(outputImg, scene_corners[3] + obj_corners[1], scene_corners[0] + obj_corners[1], OpenCvSharp.Scalar.LightBlue, 4);
*
* //OpenCvSharp.Mat outimg = Unity.TextureToMat(emptyTex);
* //inputImg = Unity.TextureToMat(emptyTex);
* //Cv2.DrawKeypoints(mat, kpCam, outimg, OpenCvSharp.Scalar.LightBlue);
*
* //show image with text after homography
* /*string imgPath2 = "../signboard-rectangle/test-IMG_0204-text.PNG";
* textTex = new Texture2D(2,2);
* byte [] binaryImageData2 = File.ReadAllBytes(imgPath2);
* textTex.LoadImage(binaryImageData2);
* rawImageRI.texture = textTex;*/
/*OpenCVForUnity.Mat inputTextImg = new OpenCVForUnity.Mat(new OpenCVForUnity.Size(textTex.width, textTex.height), CvType.CV_8UC4);
* Utils.texture2DToMat(textTex, inputTextImg);
* OpenCVForUnity.Mat outputTextImg = new OpenCVForUnity.Mat(new OpenCVForUnity.Size(textTex.width, textTex.height), CvType.CV_8UC4);
*
* Imgproc.warpPerspective(inputTextImg, outputTextImg, transMat, new OpenCVForUnity.Size(textTex.width, textTex.height));
*
* Texture2D outputTex = new Texture2D((int)textTex.width, (int)textTex.height, TextureFormat.RGB24, false);
* Utils.matToTexture2D(outputTextImg, outputTex);*/
//TextureScale.Bilinear(outputTex, outputTex.width/5, outputTex.height/5);
//rawImageRI.texture = outputTex;
//text.enabled = true;
/*Vector3 scale;
* scale.x = new Vector4((float)M.Get<double>(0,0), (float)M.Get<double>(1,0), (float)M.Get<double>(2,0), 0).magnitude;
* scale.y = new Vector4((float)M.Get<double>(0,1), (float)M.Get<double>(1,1), (float)M.Get<double>(2,1), 0).magnitude;
* scale.z = new Vector4((float)M.Get<double>(0,2), (float)M.Get<double>(1,2), (float)M.Get<double>(2,2), 0).magnitude;
*
* Vector3 forward;
* forward.x = (float)M.Get<double>(0,2);
* forward.y = (float)M.Get<double>(1,2);
* forward.z = (float)M.Get<double>(2,2);
*
* Vector3 upwards;
* upwards.x = (float)M.Get<double>(0,1);
* upwards.y = (float)M.Get<double>(1,1);
* upwards.z = (float)M.Get<double>(2,1);
*
* //textRT.localScale = scale;
* //textRT.rotation = Quaternion.LookRotation(forward, upwards);*/
Matrix4x4 matrix = new Matrix4x4();
/*matrix.SetRow(0, new Vector4((float)M.Get<double>(0,0), (float)M.Get<double>(0,1), (float)M.Get<double>(0,2),0));
* matrix.SetRow(1, new Vector4((float)M.Get<double>(1,0), (float)M.Get<double>(1,1), (float)M.Get<double>(1,2),0));
* matrix.SetRow(2, new Vector4(0,0,1,0));
* matrix.SetRow(3, new Vector4(0,0,0,1));*/
//inverse效果還行
matrix.SetRow(0, new Vector4((float)M.Get <double>(0, 0), (float)M.Get <double>(0, 1), 0, (float)M.Get <double>(0, 2)));
matrix.SetRow(1, new Vector4((float)M.Get <double>(1, 0), (float)M.Get <double>(1, 1), 0, (float)M.Get <double>(1, 2)));
matrix.SetRow(2, new Vector4(0, 0, 1, 0));
matrix.SetRow(3, new Vector4(0, 0, 0, 1));
Matrix4x4 inverse = matrix.inverse;
//textRT.localScale = matrix.lossyScale;
//textRT.rotation = matrix.rotation; //rotation跟eulerangles效果一樣
textRT1.rotation = inverse.rotation;
textRT2.rotation = inverse.rotation;
//textRT3.rotation = inverse.rotation;
Destroy(emptyTex);
//calculate euler angle
/*double angleX = Math.Asin(-M.Get<double>(2,1));
* double angleY = Math.Atan2(M.Get<double>(2,0), M.Get<double>(2,2));
* double angleZ = Math.Atan2(M.Get<double>(0,1), M.Get<double>(1,1));
* //textRT.eulerAngles = new Vector3((float)angleX, (float)angleY, (float)angleZ);
* //Debug.Log("textRT.eulerAngles: "+textRT.eulerAngles.ToString());
*
* //calculate quaternion
* double w = Math.Sqrt(1 + M.Get<double>(0,0) + M.Get<double>(1,1) + M.Get<double>(2,2))/2;
* double w4 = w*4;
* double qx = (M.Get<double>(2,1) - M.Get<double>(1,2))/w4 ;
* double qy = (M.Get<double>(0,2) - M.Get<double>(2,0))/w4 ;
* double qz = (M.Get<double>(1,0) - M.Get<double>(0,1))/w4 ;
* //textRT.rotation = new Quaternion((float)qx, (float)qy, (float)qz, 1);
*
* double tr = M.Get<double>(0,0) + M.Get<double>(1,1) + M.Get<double>(2,2);
* Debug.Log("tr: "+tr);*/
//Cv2.ImShow("img", mat);
//myCam.Pause();
}
private void buttonSelectImage_Click2(object sender, EventArgs e)
{
OpenFileDialog openFileDialog1 = new OpenFileDialog();
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
try
{
List <OpenCvSharp.Point> rois = new List <OpenCvSharp.Point>(new OpenCvSharp.Point[] {
new OpenCvSharp.Point(35, 90), new OpenCvSharp.Point(400, 90), new OpenCvSharp.Point(850, 90), new OpenCvSharp.Point(1290, 90),
new OpenCvSharp.Point(35, 400), new OpenCvSharp.Point(400, 400), new OpenCvSharp.Point(850, 400), new OpenCvSharp.Point(1290, 400),
new OpenCvSharp.Point(35, 800), new OpenCvSharp.Point(400, 800), new OpenCvSharp.Point(850, 800), new OpenCvSharp.Point(1290, 800),
new OpenCvSharp.Point(35, 1230), new OpenCvSharp.Point(400, 1230), new OpenCvSharp.Point(850, 1230), new OpenCvSharp.Point(1290, 1230)
});
var files = Directory.GetFiles(Path.GetDirectoryName(openFileDialog1.FileName), "*.png").ToList();
if (files != null && files.Count > 0)
{
int index = files.IndexOf(openFileDialog1.FileName);
//using (var win = new Window())
//using (var win1 = new Window())
{
while (true)
{
Debug.WriteLine(files[index]);
Mat img = new Mat(files[index]);//load color image
Mat gray = new Mat(), laplacian = new Mat(), sobel = new Mat(), canny = new Mat();
Cv2.CvtColor(img, gray, ColorConversionCodes.BGR2GRAY);
//Cv2.GaussianBlur(gray, gray, new OpenCvSharp.Size(3, 3), 0);
Cv2.NamedWindow("img", WindowMode.Normal);
Cv2.ImShow("img", img);
foreach (var pt in rois)
{
Rect roi = new Rect(pt, new OpenCvSharp.Size(200, 200));
Mat original = img.Clone(roi);
Mat gray2 = gray.Clone(roi);
Cv2.Canny(gray2, canny, 100, 300, 3);
Mat[] contours1;
var hierarchy1 = new List <OpenCvSharp.Point>();
Cv2.FindContours(canny, out contours1, OutputArray.Create(hierarchy1),
RetrievalModes.External,
ContourApproximationModes.ApproxSimple);
var hulls1 = new List <Mat>();
for (int j = 0; j < contours1.Length; j++)
{
Mat hull = new Mat();
Cv2.ConvexHull(contours1[j], hull);
if (hull.ContourArea() > 100)
{
hulls1.Add(hull);
}
}
if (hulls1.Count > 0)
{
Mat drawing = Mat.Zeros(canny.Size(), MatType.CV_8UC1);
Cv2.DrawContours(drawing, hulls1, -1, Scalar.White, -1);
//Cv2.NamedWindow(pt.ToString() + " hulls", WindowMode.Normal);
//Cv2.ImShow(pt.ToString() + " hulls", drawing);
List <OpenCvSharp.Point> points = new List <OpenCvSharp.Point>();
foreach (Mat hull in hulls1)
{
int m2Count = (hull.Rows % 2 > 0) ? hull.Rows + 1 : hull.Rows;
OpenCvSharp.Point[] p2 = new OpenCvSharp.Point[m2Count];
hull.GetArray(0, 0, p2);
Array.Resize(ref p2, hull.Rows);
points.AddRange(p2.ToList());
}
Mat finalHull = new Mat();
Cv2.ConvexHull(InputArray.Create(points), finalHull);
if (finalHull.ContourArea() > 1000)
{
List <Mat> finalHulls = new List <Mat>();
finalHulls.Add(finalHull);
Mat mask = Mat.Zeros(drawing.Size(), MatType.CV_8UC1);
Cv2.DrawContours(mask, finalHulls, -1, Scalar.White, -1);
//Cv2.ImShow(pt.ToString() + " mask", mask);
//Cv2.ImShow(pt.ToString() + " original", original);
Mat masked = Mat.Zeros(original.Size(), original.Type());
original.CopyTo(masked, mask);
Cv2.ImShow(pt.ToString() + " masked", masked);
}
}
}
int c = Cv2.WaitKey(0);
if (c == 'n' && index < files.Count - 1)
{
index++;
}
else if (c == 'p' && index > 0)
{
index--;
}
else if (c == 27)
{
break;
}
else
{
index = 0;
}
Window.DestroyAllWindows();
}
}
}
}
catch (Exception ex)
{
MessageBox.Show("Error: " + ex.Message);
}
}
}
private void стартToolStripMenuItem_Click(object sender, EventArgs e)
{
try
{
labelstrip.Text = "Получение изображения";
// берем изображение из PictureBox и переводим bitmap в Mat
Bitmap image1b = (Bitmap)objectPictureBox.Image;
Bitmap image2b = (Bitmap)scenePictureBox.Image;
Mat image1 = BitmapConverter.ToMat(image1b);
Mat image2 = BitmapConverter.ToMat(image2b);
//создаем SURF переменную
SURF surf = new SURF(form.getThreshold(), form.getOctaves(), form.getOctavesLayer(), form.getDecriptors(), false);
labelstrip.Text = "Нахождение особых точек и их дескриптеров";
//создаем по 2 переменных для записи ключевых точек и дескриптеров
Mat descriptors1 = new Mat();
Mat descriptors2 = new Mat();
KeyPoint[] points1, points2;
//находим особые точки
points1 = surf.Detect(image1);
points2 = surf.Detect(image2);
//нахождение дескрипторов точек
surf.Compute(image1, ref points1, descriptors1);
surf.Compute(image2, ref points2, descriptors2);
//матчим массивы дескрипторов
FlannBasedMatcher matcher = new FlannBasedMatcher();
DMatch[] matches;
matches = matcher.Match(descriptors1, descriptors2);
//Вычисление максимального и минимального расстояния среди всех дескрипторов
double max_dist = 0; double min_dist = 100;
for (int i = 0; i < descriptors1.Rows; i++)
{
double dist = matches[i].Distance;
if (dist < min_dist)
{
min_dist = dist;
}
if (dist > max_dist)
{
max_dist = dist;
}
}
labelstrip.Text = "Отбор точек";
// Отобрать только хорошие матчи, расстояние меньше чем 3 * min_dist
List <DMatch> good_matches = new List <DMatch>();
for (int i = 0; i < matches.Length; i++)
{
if (matches[i].Distance < form.getMinDinst() * min_dist)
{
good_matches.Add(matches[i]);
}
}
Mat image3 = new Mat();
Cv2.DrawMatches(image1, points1, image2, points2, good_matches, image3, Scalar.RandomColor(), Scalar.RandomColor(), null, DrawMatchesFlags.NotDrawSinglePoints);
labelstrip.Text = "Локализация объекта";
//Использование гомографии
// Локализация объектов
Point2f[] vector = new Point2f[good_matches.Count];
Point2d[] vector1 = new Point2d[good_matches.Count];
Point2d[] vector2 = new Point2d[good_matches.Count];
for (int i = 0; i < good_matches.Count; i++)
{
vector[i] = points1[good_matches[i].QueryIdx].Pt;
vector1[i].X = vector[i].X;
vector1[i].Y = vector[i].Y;
vector[i] = points2[good_matches[i].TrainIdx].Pt;
vector2[i].X = vector[i].X;
vector2[i].Y = vector[i].Y;
}
Mat H = Cv2.FindHomography(vector1, vector2, HomographyMethod.Ransac);
//Получить "углы" изображения с целевым объектом
Point2d[] vector3 = new Point2d[4];
vector3[0].X = 0; vector3[0].Y = 0;
vector3[1].X = image1.Cols; vector3[1].Y = 0;
vector3[2].X = image1.Cols; vector3[2].Y = image1.Rows;
vector3[3].X = 0; vector3[3].Y = image1.Rows;
Point2d pointtest; pointtest.X = 0; pointtest.Y = 0;
List <Point2d> vector4 = new List <Point2d>()
{
pointtest, pointtest, pointtest, pointtest
};
//Отобразить углы целевого объекта, используя найденное преобразование, на сцену
Cv2.PerspectiveTransform(InputArray.Create(vector3), OutputArray.Create(vector4), H); //?
Point2d point1;
Point2d point2;
int k;
for (int i = 0; i < 4; i++)
{
if (i == 3)
{
k = 0;
}
else
{
k = i + 1;
}
point1.X = vector4[i].X + image1.Cols;
point1.Y = vector4[i].Y + 0;
point2.X = vector4[k].X + image1.Cols;
point2.Y = vector4[k].Y + 0;
Cv2.Line(image3, point1, point2, Scalar.RandomColor(), 4);
}
labelstrip.Text = "Объект локализован";
Bitmap image3b = BitmapConverter.ToBitmap(image3);
imageForm.getresultimage(image3b);
imageForm.ShowDialog();
}
catch (Exception ex)
{
MessageBox.Show(Convert.ToString(ex));
labelstrip.Text = "Произошла ошибка";
}
}
