void Start()
{
Mat mainMat = new Mat(baseTexture.height, baseTexture.width, CvType.CV_8UC3);
Mat grayMat = new Mat();
sourceRawImage.texture = baseTexture;
Utils.texture2DToMat(baseTexture, mainMat);
mainMat.copyTo(grayMat);
Imgproc.cvtColor(grayMat, grayMat, Imgproc.COLOR_BGR2GRAY);
Imgproc.GaussianBlur(grayMat, grayMat, new Size(5, 5), 0);
Imgproc.threshold(grayMat, grayMat, 110, 225, Imgproc.THRESH_BINARY);
Imgproc.Canny(grayMat, grayMat, 20, 190);
List <MatOfPoint> contours = new List <MatOfPoint>();
Imgproc.findContours(grayMat, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
int num = 0;
List <MatOfPoint> contours_list = new List <MatOfPoint>();
// new logic
// List<MatOfPoint> contours_list = new List<MatOfPoint>();
for (int i = 0; i < contours.Count; i++)
{
MatOfPoint cp = contours[i];
MatOfPoint2f cn = new MatOfPoint2f(cp.toArray());
double p = Imgproc.arcLength(cn, true);
MatOfPoint2f approx = new MatOfPoint2f();
Imgproc.approxPolyDP(cn, approx, 0.01 * p, true);
double area = Imgproc.contourArea(contours[i]);
if ((area > 30 && area < 100) && approx.toArray().Length > 8)
{
// Imgproc.drawContours(mainMat, contours, -1, new Scalar(0, 255, 0), 4);
contours_list.Add(contours[i]);
num = num + 1;
Debug.Log(area);
}
}
// previously working
// for(int i =0; i< contours.Count ; i++){
// MatOfPoint cp = contours[i];
// MatOfPoint2f cn = new MatOfPoint2f(cp.toArray());
// double p = Imgproc.arcLength(cn,true);
// // fron akshay file
// double area = Imgproc.contourArea(contours[i]);
// if(area > 50){
// Imgproc.drawContours(mainMat, contours, -1, new Scalar(0, 255, 0), 4);
// num = num + 1;
// Debug.Log(area);
// }
// }
// for(int i =0; i< contours.Count ; i++){
// double area = Imgproc.contourArea(contours[i]);
// if(area > 50){
// // Imgproc.drawContours(mainMat, contours, -1, new Scalar(0, 255, 0), 4);
// contours_list.Add(contours[i]);
// num = num + 1;
// }
// }
for (int i = 0; i < contours_list.Count; i++)
{
Imgproc.drawContours(mainMat, contours_list, -1, new Scalar(0, 255, 0), 4);
}
Debug.Log("Number : " + num);
info.text += (num - 1).ToString();
Texture2D finaltexture = new Texture2D(grayMat.cols(), grayMat.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(grayMat, finaltexture);
sourceRawImage.texture = finaltexture;
}
public Mat TouchProcessing(Mat img, Mat pre_frame)
{
Mat img_delta = new Mat();
List <MatOfPoint> contours = new List <MatOfPoint>();
Mat hierarchy = new Mat();
MatOfPoint2f contour_2f = new MatOfPoint2f();
OpenCVForUnity.CoreModule.Rect BoundingRect = new OpenCVForUnity.CoreModule.Rect();
float hullRatio = 0;
// remove background & threshold
Core.absdiff(img, pre_frame, img_delta);
Imgproc.threshold(img_delta, img_delta, ShadeThreshold, 255, Imgproc.THRESH_BINARY);
//reduce noise
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5, 5));
Imgproc.morphologyEx(img_delta, img_delta, Imgproc.MORPH_OPEN, kernel);
Imgproc.morphologyEx(img_delta, img_delta, Imgproc.MORPH_CLOSE, kernel);
//Imgproc.blur(img_delta,img_delta,new Size(5,5));
// RETR_EXTERNAL & CHAIN_APPROX_SIMPLE
Imgproc.findContours(img_delta, contours, hierarchy, 0, 2);
//List<MatOfPoint> hulls = new List<MatOfPoint>();
List <MatOfInt> hulls = new List <MatOfInt>();
List <MatOfPoint> hulls_point = new List <MatOfPoint>();
//for store bounding rect
List <OpenCVForUnity.CoreModule.Rect> TouchRect = new List <OpenCVForUnity.CoreModule.Rect>();
for (int i = 0; i < contours.Count; i++)
{
hulls.Add(new MatOfInt());
}
// Only for max touching
if (touchType == TouchType.OneFinger)
{
int maxArea_i = -1;
for (int i = 0; i < contours.Count; i++)
{
double area = Imgproc.contourArea(contours[i]);
if (area > MaxArea)
{
MaxArea = area;
maxArea_i = i;
}
}
MaxArea = 0;
if (maxArea_i > -1)
{
contours[maxArea_i].convertTo(contour_2f, CvType.CV_32F);
BoundingRect = Imgproc.boundingRect((Mat)contour_2f);
TouchRect.Add(BoundingRect);
Imgproc.rectangle(img_delta, BoundingRect, new Scalar(255, 255, 255));
}
}
// for detecting all of the touch
if (touchType == TouchType.MultiFinger)
{
for (int i = 0; i < contours.Count; i++)
{
double area = Imgproc.contourArea(contours[i]);
//filter by size of contour
if (area > MinArea)
{
Imgproc.convexHull(contours[i], hulls[i], false);
MatOfPoint hull_point = new MatOfPoint();
// data type is different
hull_point = convertIndexToPoint(hulls[i], contours[i]);
//calculate convex ratio
hullRatio = (float)Imgproc.contourArea(contours[i]) / (float)Imgproc.contourArea(hull_point);
if (hullRatio > MaxHull)
{
// data type is different(point & point2f)
contours[i].convertTo(contour_2f, CvType.CV_32F);
BoundingRect = Imgproc.boundingRect((Mat)contour_2f);
TouchRect.Add(BoundingRect);
Imgproc.rectangle(img_delta, BoundingRect, new Scalar(255, 255, 255));
}
}
}
}
for (int i = 0; i < TouchRect.Count; i++)
{
Point tl = TouchRect[i].tl();
Point br = TouchRect[i].br();
Point centerRect = new Point((tl.x + br.x) / 2, (tl.y + br.y) / 2);
//Debug.Log(centerRect.x);
//Debug.Log(centerRect.y);
Vector3 CurrentPos = ImgPositionToTransform((float)centerRect.x, (float)centerRect.y);
OldPosedata = CurrentPos;
TouchObject.transform.position = OldPosedata;
}
//Imgproc.circle(img_delta,new Point(0,0),5,new Scalar(255,255,255));
//Imgproc.circle(img_delta, new Point(200, 0), 5, new Scalar(255, 255, 255));
return(img_delta);
}
// Update is called once per frame
void Update()
{
Resources.UnloadUnusedAssets(); //Fixes the memory leak
//Get new picture from camera
imgTexture = new Texture2D(webcamTexture.width, webcamTexture.height);
imgTexture.SetPixels(webcamTexture.GetPixels());
imgTexture.Apply();
Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3);
Utils.texture2DToMat(imgTexture, imgMat);
Mat processedMat = new Mat();
Imgproc.dilate(imgMat, imgMat, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1, 1)));
//Grayscale the picture
Imgproc.cvtColor(imgMat, processedMat, Imgproc.COLOR_RGB2GRAY);
//Blur the picture
Imgproc.GaussianBlur(processedMat, processedMat, new Size(3, 3), 1);
Imgproc.equalizeHist(processedMat, processedMat);
//Find Edges
Mat edgesOfPicture = new Mat();
Imgproc.Canny(processedMat, edgesOfPicture, 75, 225);
List <MatOfPoint> contours = new List <MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(edgesOfPicture, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
MatOfPoint2f matOfPoint2f = new MatOfPoint2f();
MatOfPoint2f approxCurve = new MatOfPoint2f();
List <Rect> rects = new List <Rect>();
try
{
for (int idx = 0; idx >= 0; idx = (int)hierarchy.get(0, idx)[0])
{
MatOfPoint contour = contours[idx];
Rect rect = Imgproc.boundingRect(contour);
double contourArea = Imgproc.contourArea(contour);
matOfPoint2f.fromList(contour.toList());
Imgproc.approxPolyDP(matOfPoint2f, approxCurve, Imgproc.arcLength(matOfPoint2f, true) * 0.02, true);
long total = approxCurve.total();
if (total == 4)
{
ArrayList cos = new ArrayList();
Point[] points = approxCurve.toArray();
for (int j = 2; j < total + 1; j++)
{
cos.Add(angle(points[(int)(j % total)], points[j - 2], points[j - 1]));
}
cos.Sort();
Double minCos = (Double)cos[0];
Double maxCos = (Double)cos[cos.Count - 1];
bool isRect = total == 4 && minCos >= -0.1 && maxCos <= 0.3;
if (isRect)
{
if (rect.width > 20)
{
rects.Add(rect);
}
List <double[]> Colors = new List <double[]>();
for (int op = 0; op < 10; op++)
{
if (rects.Count == 9)
{
allCubiesScaned = true;
Color[] blockOfColour = imgTexture.GetPixels(rect.x + rect.width / 2, rect.y + rect.height, rect.width / 3, rect.height / 3, 0);
float r = 0, g = 0, b = 0;
foreach (Color pixelBlock in blockOfColour)
{
r += pixelBlock.r;
g += pixelBlock.g;
b += pixelBlock.b;
}
r = r / blockOfColour.Length;
g = g / blockOfColour.Length;
b = b / blockOfColour.Length;
Rgb rgb = new Rgb(r, g, b);
Colors.Add(new double[] { rgb.R * 255, rgb.G * 255, rgb.B * 255 });
print(Colors.Count);
if (Colors.Count == 9)
{
ColorMap.Colors = Colors;
ColorMap.Redraw();
}
}
}
Imgproc.drawContours(imgMat, contours, idx, new Scalar(255, 100, 155), 4);
}
}
}
}
catch (ArgumentOutOfRangeException e)
{
}
Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(imgMat, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
}
void FormatImageSquare()
{
Mat mainMat = new Mat(baseTexture.height, baseTexture.width, CvType.CV_8UC3);
Mat grayMat = new Mat();
//Convert Texture2d to Matrix
Utils.texture2DToMat(baseTexture, mainMat);
//copy main matrix to grayMat
mainMat.copyTo(grayMat);
//Convert color to gray
Imgproc.cvtColor(grayMat, grayMat, Imgproc.COLOR_BGR2GRAY);
//Blur
Imgproc.GaussianBlur(grayMat, grayMat, new Size(25, 25), 0);
//contrast
Imgproc.threshold(grayMat, grayMat, 0, 255, Imgproc.THRESH_OTSU);
//extract edge
Imgproc.Canny(grayMat, grayMat, 50, 50);
//prepare for the finding contours
List <MatOfPoint> contours = new List <MatOfPoint>();
//find the contour from canny edge image
Imgproc.findContours(grayMat, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
List <MatOfPoint> tempTargets = new List <MatOfPoint>();
for (int i = 0; i < contours.Count; i++)
{
MatOfPoint cp = contours[i];
MatOfPoint2f cn = new MatOfPoint2f(cp.toArray());
double p = Imgproc.arcLength(cn, true);
MatOfPoint2f approx = new MatOfPoint2f();
//convret to polygon
Imgproc.approxPolyDP(cn, approx, 0.03 * p, true);
//find a contour with four points and large area
int minContourArea = 10000;
if (approx.toArray().Length == 4 && Imgproc.contourArea(approx) > minContourArea)
{
MatOfPoint approxPt = new MatOfPoint();
approx.convertTo(approxPt, CvType.CV_32S);
float maxCosine = 0;
for (int j = 2; j < 5; j++)
{
Vector2 v1 = new Vector2((float)(approx.toArray()[j % 4].x - approx.toArray()[j - 1].x), (float)(approx.toArray()[j % 4].y - approx.toArray()[j - 1].y));
Vector2 v2 = new Vector2((float)(approx.toArray()[j - 2].x - approx.toArray()[j - 1].x), (float)(approx.toArray()[j - 2].y - approx.toArray()[j - 1].y));
float angle = Mathf.Abs(Vector2.Angle(v1, v2));
maxCosine = Mathf.Max(maxCosine, angle);
}
if (maxCosine < 135f)
{
tempTargets.Add(approxPt);
}
}
}
if (tempTargets.Count > 0)
{
//Get the first contour
MatOfPoint approxPt = tempTargets[0];
//Making Source Mat
Mat srcPointMat = Converters.vector_Point_to_Mat(approxPt.toList(), CvType.CV_32F);
//Making Destination Mat
/*change these values*/
List <Point> dstPoints = new List <Point>();
dstPoints.Add(new Point(512, 0));
dstPoints.Add(new Point(0, 0));
dstPoints.Add(new Point(0, 512));
dstPoints.Add(new Point(512, 512));
Mat dstPointMat = Converters.vector_Point_to_Mat(dstPoints, CvType.CV_32F);
//Make Perp transform
Mat M = Imgproc.getPerspectiveTransform(srcPointMat, dstPointMat);
Mat warpedMat = new Mat(mainMat.size(), mainMat.type());
//Crop and warp the image
Imgproc.warpPerspective(mainMat, warpedMat, M, new Size(512, 512), Imgproc.INTER_LINEAR);
warpedMat.convertTo(warpedMat, CvType.CV_8UC3);
//Convert color to gray
Imgproc.cvtColor(warpedMat, warpedMat, Imgproc.COLOR_BGR2GRAY);
////Blur
//Imgproc.GaussianBlur(warpedMat, warpedMat, new Size(25, 25), 0);
//contrast
Imgproc.threshold(warpedMat, warpedMat, 0, 255, Imgproc.THRESH_OTSU);
//resize
Imgproc.resize(warpedMat, warpedMat, new Size(28, 28));
//Create an empty final texture
finalTexture = new Texture2D(warpedMat.width(), warpedMat.height(), TextureFormat.RGB24, false);
//Convert material to texture2d
Utils.matToTexture2D(warpedMat, finalTexture);
targetRawImage.texture = finalTexture;
}
}
public void ProcessSkin(Mat rgbaImage)
{
Imgproc.pyrDown(rgbaImage, mPyrDownMat);
Imgproc.pyrDown(mPyrDownMat, mPyrDownMat);
Imgproc.cvtColor(mPyrDownMat, mHsvMat, Imgproc.COLOR_RGB2HSV_FULL);
Imgproc.cvtColor(mPyrDownMat, mRGBAMat, Imgproc.COLOR_RGB2RGBA);
Imgproc.cvtColor(mPyrDownMat, mYCrCbMat, Imgproc.COLOR_RGB2YCrCb);
Core.inRange(mHsvMat, mLowerBoundHSV, mUpperBoundHSV, mMaskHSV);
Core.inRange(mPyrDownMat, mLowerBoundRGB, mUpperBoundRGB, mMaskRGB);
Core.inRange(mYCrCbMat, mLowerBoundYCrCb, mUpperBoundYCrCb, mMaskYCrCb);
mMask = mMaskYCrCb & mMaskHSV & mMaskRGB;
Imgproc.dilate(mMask, mDilatedMask, new Mat());
List <MatOfPoint> contours = new List <MatOfPoint>();
Imgproc.findContours(mDilatedMask, contours, mHierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
if (contours.Count == 0)
{
return;
}
// Find max contour area
double maxArea = 0;
double secondMaxArea = 0;
MatOfPoint biggestContour = null;
MatOfPoint secondBiggestContour = null;
foreach (MatOfPoint each in contours)
{
MatOfPoint wrapper = each;
double area = Imgproc.contourArea(wrapper);
if (area > maxArea)
{
secondMaxArea = maxArea;
secondBiggestContour = biggestContour;
maxArea = area;
biggestContour = each;
}
else if (area > secondMaxArea)
{
secondMaxArea = area;
secondBiggestContour = each;
}
}
handContourSize = maxArea;
if ((biggestContour != null) && (secondBiggestContour != null) && (ComputeAVGXForContour(biggestContour) >= ComputeAVGXForContour(secondBiggestContour)) && (secondMaxArea >= 0.3 * maxArea))
{
biggestContour = secondBiggestContour;
handContourSize = secondMaxArea;
}
MatOfPoint2f contours_res2f = new MatOfPoint2f();
MatOfPoint2f biggestContour2f = new MatOfPoint2f(biggestContour.toArray());
Imgproc.approxPolyDP(biggestContour2f, contours_res2f, 3, true);
handContour = new MatOfPoint(contours_res2f.toArray());
contours_res2f.Dispose();
biggestContour2f.Dispose();
if (Imgproc.contourArea(handContour) > mMinContourArea * maxArea)
{
Core.multiply(handContour, new Scalar(4, 4), handContour);
}
// Filter contours by area and resize to fit the original image size
mContours.Clear();
foreach (MatOfPoint each in contours)
{
MatOfPoint contour = each;
if (Imgproc.contourArea(contour) > mMinContourArea * maxArea)
{
Core.multiply(contour, new Scalar(4, 4), contour);
mContours.Add(contour);
}
}
}
private void Run()
{
Mat src = Imgcodecs.imread(image_filepath);
#if !UNITY_WSA_10_0
if (src.empty())
{
Debug.LogError("pca_test1.jpg is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
#endif
Debug.Log("src.ToString() " + src.ToString());
// Convert image to grayscale
Mat gray = new Mat();
Imgproc.cvtColor(src, gray, Imgproc.COLOR_BGR2GRAY);
// Convert image to binary
Mat bw = new Mat();
Imgproc.threshold(gray, bw, 50, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
// Find all the contours in the thresholded image
Mat hierarchy = new Mat();
List <MatOfPoint> contours = new List <MatOfPoint> ();
Imgproc.findContours(bw, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_NONE);
for (int i = 0; i < contours.Count; ++i)
{
// Calculate the area of each contour
double area = Imgproc.contourArea(contours [i]);
// Ignore contours that are too small or too large
if (area < 1e2 || 1e5 < area)
{
continue;
}
// Draw each contour only for visualisation purposes
Imgproc.drawContours(src, contours, i, new Scalar(0, 0, 255), 2);
//Construct a buffer used by the pca analysis
List <Point> pts = contours [i].toList();
int sz = pts.Count;
Mat data_pts = new Mat(sz, 2, CvType.CV_64FC1);
for (int p = 0; p < data_pts.rows(); ++p)
{
data_pts.put(p, 0, pts [p].x);
data_pts.put(p, 1, pts [p].y);
}
Mat mean = new Mat();
Mat eigenvectors = new Mat();
Core.PCACompute(data_pts, mean, eigenvectors, 1);
Debug.Log("mean.dump() " + mean.dump());
Debug.Log("eigenvectors.dump() " + eigenvectors.dump());
Point cntr = new Point(mean.get(0, 0) [0], mean.get(0, 1) [0]);
Point vec = new Point(eigenvectors.get(0, 0) [0], eigenvectors.get(0, 1) [0]);
drawAxis(src, cntr, vec, new Scalar(255, 255, 0), 150);
data_pts.Dispose();
mean.Dispose();
eigenvectors.Dispose();
}
Imgproc.cvtColor(src, src, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(src.cols(), src.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(src, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}
