// Use this for initialization void Start() { Mat imgMat = new Mat(500, 500, CvType.CV_8UC3, new Scalar(0, 0, 0)); Debug.Log("imgMat.ToString() " + imgMat.ToString()); int rand_num = 50; MatOfPoint pointsMat = new MatOfPoint(); pointsMat.alloc(rand_num); Core.randu(pointsMat, 100, 400); Point[] points = pointsMat.toArray(); for (int i = 0; i < rand_num; ++i) { Imgproc.circle(imgMat, points [i], 2, new Scalar(255, 255, 255), -1); } MatOfInt hullInt = new MatOfInt(); Imgproc.convexHull(pointsMat, hullInt); List <Point> pointMatList = pointsMat.toList(); List <int> hullIntList = hullInt.toList(); List <Point> hullPointList = new List <Point> (); for (int j = 0; j < hullInt.toList().Count; j++) { hullPointList.Add(pointMatList [hullIntList [j]]); } MatOfPoint hullPointMat = new MatOfPoint(); hullPointMat.fromList(hullPointList); List <MatOfPoint> hullPoints = new List <MatOfPoint> (); hullPoints.Add(hullPointMat); Imgproc.drawContours(imgMat, hullPoints, -1, new Scalar(0, 255, 0), 2); Imgproc.cvtColor(imgMat, imgMat, Imgproc.COLOR_BGR2RGB); Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false); Utils.matToTexture2D(imgMat, texture); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; }
// Use this for initialization void Start () { Mat imgMat = new Mat (500, 500, CvType.CV_8UC3, new Scalar (0, 0, 0)); Debug.Log ("imgMat dst ToString " + imgMat.ToString ()); int rand_num = 50; MatOfPoint pointsMat = new MatOfPoint (); pointsMat.alloc (rand_num); Core.randu (pointsMat, 100, 400); Point[] points = pointsMat.toArray (); for (int i=0; i<rand_num; ++i) { Core.circle (imgMat, points [i], 2, new Scalar (255, 255, 255), -1); } MatOfInt hullInt = new MatOfInt (); Imgproc.convexHull (pointsMat, hullInt); List<Point> pointMatList = pointsMat.toList (); List<int> hullIntList = hullInt.toList (); List<Point> hullPointList = new List<Point> (); for (int j=0; j < hullInt.toList().Count; j++) { hullPointList.Add (pointMatList [hullIntList [j]]); } MatOfPoint hullPointMat = new MatOfPoint (); hullPointMat.fromList (hullPointList); List<MatOfPoint> hullPoints = new List<MatOfPoint> (); hullPoints.Add (hullPointMat); Imgproc.drawContours (imgMat, hullPoints, -1, new Scalar (0, 255, 0), 2); Imgproc.cvtColor (imgMat, imgMat, Imgproc.COLOR_BGR2RGB); Texture2D texture = new Texture2D (imgMat.cols (), imgMat.rows (), TextureFormat.RGBA32, false); Utils.matToTexture2D (imgMat, texture); gameObject.GetComponent<Renderer> ().material.mainTexture = texture; }
/// <summary> /// Perimeter the specified a. /// </summary> /// <param name="a">The alpha component.</param> float perimeter(MatOfPoint a) { List <Point> aList = a.toList(); float sum = 0, dx = 0, dy = 0; for (int i = 0; i < aList.Count; i++) { int i2 = (i + 1) % aList.Count; dx = (float)aList [i].x - (float)aList [i2].x; dy = (float)aList [i].y - (float)aList [i2].y; sum += Mathf.Sqrt(dx * dx + dy * dy); } return(sum); }
//public RawImage document; void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat mainMat = webCamTextureToMatHelper.GetMat(); if (!selectTarget) //find paper by contours { grayMat = new Mat(); // convert texture to matrix mainMat.copyTo(grayMat); mainMat = findPaper(mainMat); // display matrix on the screen Utils.fastMatToTexture2D(mainMat, texture); } else { // using optical flow // set the currentGrayMat mat currentGrayMat = new Mat(mainMat.rows(), mainMat.cols(), Imgproc.COLOR_RGB2GRAY); Imgproc.cvtColor(mainMat, currentGrayMat, Imgproc.COLOR_RGBA2GRAY); if (initOpticalFlow == true) // doing the init setting for optical flow { // create 40 points Point[] points = new Point[40]; // set those points near the corner // paperCornerMatOfPoint is the corner of the paper for (int i = 0; i < 4; i++) { points[i * 10] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 1] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 2] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y + 1); points[i * 10 + 3] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y + 1); points[i * 10 + 4] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 1); points[i * 10 + 5] = new Point(paperCornerMatOfPoint.toList()[i].x - 1, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 6] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 1); points[i * 10 + 7] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 2); points[i * 10 + 8] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 2); points[i * 10 + 9] = new Point(paperCornerMatOfPoint.toList()[i].x + 2, paperCornerMatOfPoint.toList()[i].y + 2); } // make the points closer to the corners (Harris Corner Detection ) //Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40, qualityLevel, minDistance, none, blockSize, false, 0.04); //Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40,0.05,20); corners.fromArray(points); prevFeatures.fromList(corners.toList()); currentFeatures.fromList(corners.toList()); prevGrayMat = currentGrayMat.clone(); // won't go back t again initOpticalFlow = false; // not that useful lol // create random color // not working now for (int i = 0; i < maxCorners; i++) { color.Add(new Scalar((int)(Random.value * 255), (int)(Random.value * 255), (int)(Random.value * 255), 255)); } } else { // Don't want ball move //currentFeatures.fromArray(prevFeatures.toArray()); // want ball move prevFeatures.fromArray(currentFeatures.toArray()); // optical flow it will changes the valu of currentFeatures Video.calcOpticalFlowPyrLK(prevGrayMat, currentGrayMat, prevFeatures, currentFeatures, mMOBStatus, err); //Debug.Log(st.rows()); // change to points list List <Point> prevList = prevFeatures.toList(), nextList = currentFeatures.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus[x] == 1) { Point pt = nextList[x]; Point pt2 = prevList[x]; Imgproc.circle(mainMat, pt, 10, new Scalar(0, 0, 255), -1); Imgproc.line(mainMat, pt, pt2, new Scalar(0, 0, 255)); } } // draw the data //for (int i = 0; i < prevList.Count; i++) //{ // //Imgproc.circle(frame, prevList[i], 5, color[10]); // Imgproc.circle(mainMat, nextList[i], 10, new Scalar(0, 0, 255), -1); // Imgproc.line(mainMat, prevList[i], nextList[i], color[20]); //} List <List <Point> > cornersFeatures = new List <List <Point> >(40); cornersFeatures.Add(new List <Point>(10)); // put the corners features data into the list int tmp = 0; bool last = true; for (int i = 0; i < nextList.Count - 1; i++) { if (Mathf.Abs((float)(nextList[i].x - nextList[i + 1].x)) < 10 && Mathf.Abs((float)(nextList[i].y - nextList[i + 1].y)) < 10) { if (last == true) { cornersFeatures[tmp].Add(nextList[i]); } else { cornersFeatures.Add(new List <Point>(10)); tmp = tmp + 1; cornersFeatures[tmp].Add(nextList[i]); } last = true; } else { last = false; } } // count corners int manyCornersFeatures = 0; for (int i = 0; i < cornersFeatures.Count; i++) { Debug.Log(cornersFeatures[i].Count); if (cornersFeatures[i].Count < 5) { cornersFeatures.RemoveAt(i); } else { manyCornersFeatures++; } } //Debug.Log("Length" + manyCornersFeatures); // if corners equal 4 then diplay virtual docunment into the frame // doing the perspective transform if (manyCornersFeatures == 4) { Mat documentMat = new Mat(document.height, document.width, CvType.CV_8UC3); Utils.texture2DToMat(document, documentMat); List <Point> srcPoints = new List <Point>(); srcPoints.Add(new Point(0, 0)); srcPoints.Add(new Point(documentMat.cols(), 0)); srcPoints.Add(new Point(documentMat.cols(), documentMat.rows())); srcPoints.Add(new Point(0, documentMat.rows())); Mat srcPointsMat = Converters.vector_Point_to_Mat(srcPoints, CvType.CV_32F); List <Point> dstPoints = new List <Point>() { cornersFeatures[0][0], cornersFeatures[1][0], cornersFeatures[2][0], cornersFeatures[3][0] }; Mat dstPointsMat = Converters.vector_Point_to_Mat(dstPoints, CvType.CV_32F); //Make perspective transform Mat m = Imgproc.getPerspectiveTransform(srcPointsMat, dstPointsMat); Mat warpedMat = new Mat(new Size(), documentMat.type()); Debug.Log((cornersFeatures[1][0].x - cornersFeatures[0][0].x) + " " + (cornersFeatures[2][0].y - cornersFeatures[1][0].y)); Imgproc.warpPerspective(documentMat, warpedMat, m, mainMat.size(), Imgproc.INTER_LINEAR); //warpedMat.convertTo(warpedMat, CvType.CV_32F); //warpedMat.convertTo(warpedMat, CvType.CV_8UC3); warpedMat.convertTo(warpedMat, CvType.CV_8UC3); // same size as frame Mat dst = new Mat(mainMat.size(), CvType.CV_8UC3); //Mat dst = new Mat(frame.size(), CvType.CV_8UC3); //Mat dst2 = new Mat(); Imgproc.cvtColor(mainMat, dst, Imgproc.COLOR_RGBA2RGB); //dst.setTo(new Scalar(0, 255, 0)); //currentGrayMat.copyTo(dst); //dst.convertTo(dst, CvType.CV_8UC3); //Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA); Mat img1 = new Mat(); Mat mask = new Mat(mainMat.size(), CvType.CV_8UC1, new Scalar(0)); Imgproc.cvtColor(warpedMat, img1, Imgproc.COLOR_RGB2GRAY); Imgproc.Canny(img1, img1, 100, 200); List <MatOfPoint> doc_contours = new List <MatOfPoint>();; Imgproc.findContours(img1, doc_contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_NONE); Imgproc.drawContours(mask, doc_contours, -1, new Scalar(255), Core.FILLED); warpedMat.copyTo(dst, mask); dst.convertTo(dst, CvType.CV_8UC3); Debug.Log("dst" + dst.size()); Imgproc.cvtColor(dst, mainMat, Imgproc.COLOR_RGB2RGBA); // display on the right Texture2D finalTextue = new Texture2D(dst.width(), dst.height(), TextureFormat.RGB24, false); Utils.matToTexture2D(dst, finalTextue); targetRawImage.texture = finalTextue; } // current frame to old frame prevGrayMat = currentGrayMat.clone(); //Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA); // display matrix on the screen Utils.fastMatToTexture2D(mainMat, texture); } } } }
private void HandPoseEstimationProcess(Mat rgbaMat) { //Imgproc.blur(mRgba, mRgba, new Size(5,5)); Imgproc.GaussianBlur(rgbaMat, rgbaMat, new OpenCVForUnity.Size(3, 3), 1, 1); //Imgproc.medianBlur(mRgba, mRgba, 3); if (!isColorSelected) { return; } List <MatOfPoint> contours = detector.GetContours(); detector.Process(rgbaMat); // Debug.Log ("Contours count: " + contours.Count); if (contours.Count <= 0) { return; } RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours [0].toArray())); double boundWidth = rect.size.width; double boundHeight = rect.size.height; int boundPos = 0; for (int i = 1; i < contours.Count; i++) { rect = Imgproc.minAreaRect(new MatOfPoint2f(contours [i].toArray())); if (rect.size.width * rect.size.height > boundWidth * boundHeight) { boundWidth = rect.size.width; boundHeight = rect.size.height; boundPos = i; } } MatOfPoint contour = contours [boundPos]; OpenCVForUnity.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray())); Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), CONTOUR_COLOR_WHITE, 2, 8, 0); // Debug.Log ( // " Row start [" + // (int)boundRect.tl ().y + "] row end [" + // (int)boundRect.br ().y + "] Col start [" + // (int)boundRect.tl ().x + "] Col end [" + // (int)boundRect.br ().x + "]"); double a = boundRect.br().y - boundRect.tl().y; a = a * 0.7; a = boundRect.tl().y + a; // Debug.Log ( // " A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]"); Imgproc.rectangle(rgbaMat, boundRect.tl(), new Point(boundRect.br().x, a), CONTOUR_COLOR, 2, 8, 0); MatOfPoint2f pointMat = new MatOfPoint2f(); Imgproc.approxPolyDP(new MatOfPoint2f(contour.toArray()), pointMat, 3, true); contour = new MatOfPoint(pointMat.toArray()); MatOfInt hull = new MatOfInt(); MatOfInt4 convexDefect = new MatOfInt4(); Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull); if (hull.toArray().Length < 3) { return; } Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect); List <MatOfPoint> hullPoints = new List <MatOfPoint> (); List <Point> listPo = new List <Point> (); for (int j = 0; j < hull.toList().Count; j++) { listPo.Add(contour.toList() [hull.toList() [j]]); } MatOfPoint e = new MatOfPoint(); e.fromList(listPo); hullPoints.Add(e); List <Point> listPoDefect = new List <Point> (); if (convexDefect.rows() > 0) { List <int> convexDefectList = convexDefect.toList(); List <Point> contourList = contour.toList(); for (int j = 0; j < convexDefectList.Count; j = j + 4) { Point farPoint = contourList [convexDefectList [j + 2]]; int depth = convexDefectList [j + 3]; if (depth > threasholdSlider.value && farPoint.y < a) { listPoDefect.Add(contourList [convexDefectList [j + 2]]); } // Debug.Log ("convexDefectList [" + j + "] " + convexDefectList [j + 3]); } } // Debug.Log ("hull: " + hull.toList ()); // if (convexDefect.rows () > 0) { // Debug.Log ("defects: " + convexDefect.toList ()); // } Imgproc.drawContours(rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3); // int defectsTotal = (int)convexDefect.total(); // Debug.Log ("Defect total " + defectsTotal); this.numberOfFingers = listPoDefect.Count; if (this.numberOfFingers > 5) { this.numberOfFingers = 5; } // Debug.Log ("numberOfFingers " + numberOfFingers); // Core.putText (mRgba, "" + numberOfFingers, new Point (mRgba.cols () / 2, mRgba.rows () / 2), Core.FONT_HERSHEY_PLAIN, 4.0, new Scalar (255, 255, 255, 255), 6, Core.LINE_AA, false); numberOfFingersText.text = numberOfFingers.ToString(); foreach (Point p in listPoDefect) { Imgproc.circle(rgbaMat, p, 6, new Scalar(255, 0, 255, 255), -1); } }
/// <summary> /// Perimeter the specified a. /// </summary> /// <param name="a">The alpha component.</param> float perimeter (MatOfPoint a) { List<Point> aList = a.toList (); float sum = 0, dx = 0, dy = 0; for (int i=0; i<aList.Count; i++) { int i2 = (i + 1) % aList.Count; dx = (float)aList [i].x - (float)aList [i2].x; dy = (float)aList [i].y - (float)aList [i2].y; sum += Mathf.Sqrt (dx * dx + dy * dy); } return sum; }
public static List <Vector2> MatOfPointToVector2List(MatOfPoint matOfPoint) { return(PointListToVector2List(matOfPoint.toList())); }
// Update is called once per frame void Update() { Resources.UnloadUnusedAssets(); //Fixes the memory leak //Get new picture from camera imgTexture = new Texture2D(webcamTexture.width, webcamTexture.height); imgTexture.SetPixels(webcamTexture.GetPixels()); imgTexture.Apply(); Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3); Utils.texture2DToMat(imgTexture, imgMat); Mat maskMat = new Mat(); Mat maskMatOP = new Mat(); Mat grayMat = new Mat(); Imgproc.dilate(imgMat, imgMat, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1, 1))); //Grayscale the picture Imgproc.cvtColor(imgMat, grayMat, Imgproc.COLOR_RGB2GRAY); //Blur the picture Imgproc.GaussianBlur(grayMat, grayMat, new Size(3, 3), 1); Imgproc.equalizeHist(grayMat, grayMat); //Find Edges Mat edgesOfPicture = new Mat(); Imgproc.Canny(grayMat, edgesOfPicture, 75, 225); List <MatOfPoint> contours = new List <MatOfPoint>(); Mat hierarchy = new Mat(); Imgproc.findContours(edgesOfPicture, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); MatOfPoint2f matOfPoint2f = new MatOfPoint2f(); MatOfPoint2f approxCurve = new MatOfPoint2f(); List <Rect> rectPre = new List <Rect>(); List <Rect> rectAfter = new List <Rect>(); try { List <MatOfPoint2f> kvadrater = new List <MatOfPoint2f>(); for (int idx = 0; idx >= 0; idx = (int)hierarchy.get(0, idx)[0]) { MatOfPoint contour = contours[idx]; Rect rect = Imgproc.boundingRect(contour); double contourArea = Imgproc.contourArea(contour); matOfPoint2f.fromList(contour.toList()); Imgproc.approxPolyDP(matOfPoint2f, approxCurve, Imgproc.arcLength(matOfPoint2f, true) * 0.02, true); long total = approxCurve.total(); if (total > 0) { kvadrater.Add(approxCurve); ArrayList cos = new ArrayList(); Point[] points = approxCurve.toArray(); for (int j = 2; j < total + 1; j++) { cos.Add(angle(points[(int)(j % total)], points[j - 2], points[j - 1])); } cos.Sort(); Double minCos = (Double)cos[0]; Double maxCos = (Double)cos[cos.Count - 1]; bool isRect = total == 4 && minCos >= -0.1 && maxCos <= 0.3; //List<double[]> Colors = new List<double[]>(); if (isRect) { if (rect.width > 20) { rectPre.Add(rect); } List <Color> Colors = new List <Color>(); List <double[]> colorDoubles = new List <double[]>(); for (int op = 0; op < 9; op++) { if (rectPre.Count == 9) { // print("Pre verify: " + rectPre.ToString()); //rectPre = CoordinateVerifier.Verify(rectPre); Använd inte LINQ !! ! ! ! ! // print("After verify: " + rectPre.ToString()); var punkt = imgTexture.GetPixel(rect.x + (rect.width / 2), rect.y + (rect.height / 2)); Imgproc.putText(imgMat, op.ToString(), new Point(rectPre[op].x + 20, rectPre[op].y + 30), Core.FONT_HERSHEY_DUPLEX, 3, new Scalar(200)); Rgb rgb = new Rgb(punkt.r, punkt.g, punkt.b); // print("rect[" + op + "] was found at" + rect.x.ToString() + "and y: " + rect.y.ToString()); var hsv = rgb.To <Hsv>(); String farg = "Ingen farg"; if (hsv.H >= 45 && hsv.H <= 70) { farg = "Gul"; } if (hsv.H >= 10 && hsv.H <= 45) { farg = "Orange"; } // print(farg); Colors.Clear(); for (int q = 0; q < rectPre.Count; q++) { Color[] blockOfColour = imgTexture.GetPixels(rectPre[q].x + (rectPre[q].width / 2), rectPre[q].y + (rectPre[q].height / 2), rectPre[q].width / 3, rectPre[q].height / 3, 0); float r = 0, g = 0, b = 0; foreach (Color pixelBlock in blockOfColour) { r += pixelBlock.r; g += pixelBlock.g; b += pixelBlock.b; } r = r / blockOfColour.Length; g = g / blockOfColour.Length; b = b / blockOfColour.Length; var eColor = _colorDetection.ColorEnumFromScalarColor(new double[] { r * 255, g * 255, b * 255 }); var color = ColorDetection.UnityColorFromEnum(eColor); Colors.Add(color); } if (Colors.Count == 9) { ColorTracker.Instance.addToTemp(Colors); foreach (Color c in Colors) { // print(c.ToString()); } } } } Imgproc.drawContours(imgMat, contours, idx, new Scalar(255, 100, 155), 4); } } } } catch (ArgumentOutOfRangeException e) { } Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false); Utils.matToTexture2D(imgMat, texture); gameObject.GetComponent <Renderer>().material.mainTexture = texture; }
void Process() { string imText = "DRAW PATTERN"; Core.putText(frame_pot, imText, new Point(110, 50), Core.FONT_HERSHEY_COMPLEX, 1.0, new Scalar(255, 0, 0), 2); Mat hierarchy = new Mat (); List<MatOfPoint> contours = new List<MatOfPoint> (); MatOfPoint maxitem = new MatOfPoint (); MatOfInt hullInt = new MatOfInt (); frameclone = frame_thresh_final.clone (); Imgproc.findContours (frameclone, contours, hierarchy, Imgproc.RETR_LIST , Imgproc.CHAIN_APPROX_NONE); maxitem = contours [0]; n = 0; for(int i=0; i<contours.Count; i++){ if(contours[i].total() > maxitem.total()){ maxitem = contours[i]; n=i; } } OpenCVForUnity.Rect bRect = Imgproc.boundingRect (maxitem); int bRect_height = bRect.height; int bRect_width = bRect.width; if (bRect_height < 200 || bRect_width < 200) return; // Drawing Contours on the Frame //Imgproc.drawContours (frame_pot, contours, n, new Scalar(0, 255, 0), 2); Imgproc.convexHull (maxitem, hullInt); List<Point> maxitemPointList = maxitem.toList (); List<int> hullIntList = hullInt.toList (); List<Point> hullPointList = new List<Point> (); for (int j=0; j < hullInt.toList().Count; j++) { hullPointList.Add (maxitemPointList [hullIntList [j]]); } MatOfPoint hullPointMat = new MatOfPoint (); hullPointMat.fromList (hullPointList); List<MatOfPoint> hullPoints = new List<MatOfPoint> (); hullPoints.Add (hullPointMat); // Drawing Convex Hull on the Frame //Imgproc.drawContours (frame_pot, hullPoints, -1, new Scalar (0, 0, 255), 2); MatOfInt4 convexityDef = new MatOfInt4 (); Imgproc.convexityDefects (maxitem, hullInt, convexityDef); List<int> conDefIntList = convexityDef.toList (); List<Point> startpts = new List<Point> (); List<Point> farpts = new List<Point> (); List<Point> endpts = new List<Point> (); int tolerance = (int)(bRect_height/6); //Debug.Log ("Tolerance: " + tolerance); int[] defarray = new int[100]; int coordX = 10000, coordY = 10000; int x1 = (int) sphere1.transform.position.x; int y1 = (int) sphere1.transform.position.y; int x2 = (int) sphere2.transform.position.x; int y2 = (int) sphere2.transform.position.y; int x3 = (int) sphere3.transform.position.x; int y3 = (int) sphere3.transform.position.y; int x4 = (int) sphere4.transform.position.x; int y4 = (int) sphere4.transform.position.y; Point pointer = new Point(); for(int i=0; i < conDefIntList.Count/4 ; i++) { startpts.Add(maxitemPointList[conDefIntList[4*i]]); endpts.Add(maxitemPointList[conDefIntList[4*i+1]]); farpts.Add(maxitemPointList[conDefIntList[4*i+2]]); Point s = startpts[i]; Point e = endpts[i]; Point f = farpts[i]; if (GetDistance(s,f) > tolerance) { //Core.circle(frame_pot, s, 15, new Scalar(255, 225, 0), -1); if (s.y < coordY) { pointer = s; coordY = (int) s.y; coordX = (int) s.x; } } } Core.circle(frame_pot, pointer, 15, new Scalar(255, 225, 0), -1); coordX = coordX - 240; coordY = -coordY + 320; if (coordX > x1-50 && coordX < x1+50 && coordY > y1-50 && coordY < y1+50) { if (previous.Equals('1')) return; input += "1"; AddLine(previous, '1'); previous = '1'; Material mat1 = sphere1.GetComponent<Renderer>().material; mat1.color = Color.yellow; StartCoroutine(WaitAndChangeColor("1")); } else if (coordX > x2-50 && coordX < x2+50 && coordY > y2-50 && coordY < y2+50) { if (previous.Equals('2')) return; input += "2"; AddLine(previous, '2'); previous = '2'; Material mat2 = sphere2.GetComponent<Renderer>().material; mat2.color = Color.yellow; StartCoroutine(WaitAndChangeColor("2")); } else if (coordX > x3-50 && coordX < x3+50 && coordY > y3-50 && coordY < y3+50) { if (previous.Equals('3')) return; input += "3"; AddLine(previous, '3'); previous = '3'; Material mat3 = sphere3.GetComponent<Renderer>().material; mat3.color = Color.yellow; StartCoroutine(WaitAndChangeColor("3")); } else if (coordX > x4-50 && coordX < x4+50 && coordY > y4-50 && coordY < y4+50) { if (previous.Equals('4')) return; input += "4"; AddLine(previous, '4'); previous = '4'; Material mat4 = sphere4.GetComponent<Renderer>().material; mat4.color = Color.yellow; StartCoroutine(WaitAndChangeColor("4")); } if (input.Length == password.Length) { auth = true; if (input.Equals(password)) { correct = true; } else { correct = false; } } }
private void HandPoseEstimationProcess(Mat rgbaMat) { //Imgproc.blur(mRgba, mRgba, new Size(5,5)); Imgproc.GaussianBlur(rgbaMat, rgbaMat, new Size(3, 3), 1, 1); //Imgproc.medianBlur(mRgba, mRgba, 3); if (!isColorSelected) { return; } List <MatOfPoint> contours = detector.GetContours(); detector.Process(rgbaMat); //Debug.Log ("Contours count: " + contours.Count); if (contours.Count <= 0) { return; } RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[0].toArray())); double boundWidth = rect.size.width; double boundHeight = rect.size.height; int boundPos = 0; for (int i = 1; i < contours.Count; i++) { rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[i].toArray())); if (rect.size.width * rect.size.height > boundWidth * boundHeight) { boundWidth = rect.size.width; boundHeight = rect.size.height; boundPos = i; } } MatOfPoint contour = contours[boundPos]; OpenCVForUnity.CoreModule.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray())); Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), CONTOUR_COLOR_WHITE, 2, 8, 0); // Debug.Log ( // " Row start [" + //(int)boundRect.tl ().y + "] row end [" + // (int)boundRect.br ().y + "] Col start [" + // (int)boundRect.tl ().x + "] Col end [" + // (int)boundRect.br ().x + "]"); Point bottomLeft = new Point(boundRect.x, boundRect.y + boundRect.height); Point topLeft = new Point(boundRect.x, boundRect.y); Point bottomRight = new Point(boundRect.x + boundRect.width, boundRect.y + boundRect.height); Point topRight = new Point(boundRect.x + boundRect.width, boundRect.y); rectPoints = new MatOfPoint2f(new Point(boundRect.x, boundRect.y), //topleft new Point(boundRect.x + boundRect.width, boundRect.y), //Top Right new Point(boundRect.x + boundRect.width, boundRect.y + boundRect.height), //Bottom Right new Point(boundRect.x, boundRect.y + boundRect.height) //Bottom Left ); //double a = boundRect.br ().y - boundRect.tl ().y; //a = a * 0.7; //a = boundRect.tl ().y + a; //Debug.Log (" A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]"); //Imgproc.rectangle (rgbaMat, boundRect.tl (), new Point (boundRect.br ().x, a), CONTOUR_COLOR, 2, 8, 0); List <Point3> m_markerCorners3dList = new List <Point3>(); m_markerCorners3dList.Add(new Point3(-0.5f, -0.5f, 0)); //Top, Left (A) m_markerCorners3dList.Add(new Point3(+0.5f, -0.5f, 0)); //Top, Right (B) m_markerCorners3dList.Add(new Point3(+0.5f, +0.5f, 0)); //Bottom, Right (C) m_markerCorners3dList.Add(new Point3(-0.5f, +0.5f, 0)); //Bottom, Left (D) m_markerCorners3d.fromList(m_markerCorners3dList); //estimate pose Mat Rvec = new Mat(); Mat Tvec = new Mat(); Mat raux = new Mat(); Mat taux = new Mat(); Calib3d.solvePnP(m_markerCorners3d, rectPoints, camMatrix, distCoeff, raux, taux); raux.convertTo(Rvec, CvType.CV_32F); taux.convertTo(Tvec, CvType.CV_32F); rotMat = new Mat(3, 3, CvType.CV_64FC1); Calib3d.Rodrigues(Rvec, rotMat); transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0)[0], (float)rotMat.get(0, 1)[0], (float)rotMat.get(0, 2)[0], (float)Tvec.get(0, 0)[0])); transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0)[0], (float)rotMat.get(1, 1)[0], (float)rotMat.get(1, 2)[0], (float)Tvec.get(1, 0)[0])); transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0)[0], (float)rotMat.get(2, 1)[0], (float)rotMat.get(2, 2)[0], (float)Tvec.get(2, 0)[0])); transformationM.SetRow(3, new Vector4(0, 0, 0, 1)); //Debug.Log ("transformationM " + transformationM.ToString ()); Rvec.Dispose(); Tvec.Dispose(); raux.Dispose(); taux.Dispose(); rotMat.Dispose(); ARM = ARCamera.transform.localToWorldMatrix * invertYM * transformationM * invertZM; //Debug.Log("arM " + ARM.ToString()); if (ARGameObject != null) { ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM); if (deactivateCoroutine == null) { deactivateCoroutine = StartCoroutine(Wait(10.0f)); } ARGameObject.SetActive(true); } //end pose estimation MatOfPoint2f pointMat = new MatOfPoint2f(); Imgproc.approxPolyDP(new MatOfPoint2f(contour.toArray()), pointMat, 3, true); contour = new MatOfPoint(pointMat.toArray()); MatOfInt hull = new MatOfInt(); MatOfInt4 convexDefect = new MatOfInt4(); Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull); if (hull.toArray().Length < 3) { return; } Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect); List <MatOfPoint> hullPoints = new List <MatOfPoint>(); List <Point> listPo = new List <Point>(); for (int j = 0; j < hull.toList().Count; j++) { listPo.Add(contour.toList()[hull.toList()[j]]); } MatOfPoint e = new MatOfPoint(); e.fromList(listPo); hullPoints.Add(e); List <Point> listPoDefect = new List <Point>(); if (convexDefect.rows() > 0) { List <int> convexDefectList = convexDefect.toList(); List <Point> contourList = contour.toList(); for (int j = 0; j < convexDefectList.Count; j = j + 4) { Point farPoint = contourList[convexDefectList[j + 2]]; int depth = convexDefectList[j + 3]; //if (depth > threasholdSlider.value && farPoint.y < a) //{ // listPoDefect.Add(contourList[convexDefectList[j + 2]]); //} //Debug.Log ("convexDefectList [" + j + "] " + convexDefectList [j + 3]); } } Debug.Log("hull: " + hull.toList()); if (convexDefect.rows() > 0) { Debug.Log("defects: " + convexDefect.toList()); } //use these contours to do heart detection Imgproc.drawContours(rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3); int defectsTotal = (int)convexDefect.total(); Debug.Log("Defect total " + defectsTotal); this.numberOfFingers = listPoDefect.Count; if (this.numberOfFingers > 5) { this.numberOfFingers = 5; } Debug.Log("numberOfFingers " + numberOfFingers); Imgproc.putText(rgbaMat, "" + numberOfFingers, new Point(rgbaMat.cols() / 2, rgbaMat.rows() / 2), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 255, 255, 255), 6, Imgproc.LINE_AA, false); numberOfFingersText.text = numberOfFingers.ToString(); foreach (Point p in listPoDefect) { Imgproc.circle(rgbaMat, p, 6, new Scalar(255, 0, 255, 255), -1); } }
// Update is called once per frame void Update() { Resources.UnloadUnusedAssets(); //Fixes the memory leak //Get new picture from camera imgTexture = new Texture2D(webcamTexture.width, webcamTexture.height); imgTexture.SetPixels(webcamTexture.GetPixels()); imgTexture.Apply(); Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC3); Utils.texture2DToMat(imgTexture, imgMat); Mat processedMat = new Mat(); Imgproc.dilate(imgMat, imgMat, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1, 1))); //Grayscale the picture Imgproc.cvtColor(imgMat, processedMat, Imgproc.COLOR_RGB2GRAY); //Blur the picture Imgproc.GaussianBlur(processedMat, processedMat, new Size(3, 3), 1); Imgproc.equalizeHist(processedMat, processedMat); //Find Edges Mat edgesOfPicture = new Mat(); Imgproc.Canny(processedMat, edgesOfPicture, 75, 225); List <MatOfPoint> contours = new List <MatOfPoint>(); Mat hierarchy = new Mat(); Imgproc.findContours(edgesOfPicture, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); MatOfPoint2f matOfPoint2f = new MatOfPoint2f(); MatOfPoint2f approxCurve = new MatOfPoint2f(); List <Rect> rects = new List <Rect>(); try { for (int idx = 0; idx >= 0; idx = (int)hierarchy.get(0, idx)[0]) { MatOfPoint contour = contours[idx]; Rect rect = Imgproc.boundingRect(contour); double contourArea = Imgproc.contourArea(contour); matOfPoint2f.fromList(contour.toList()); Imgproc.approxPolyDP(matOfPoint2f, approxCurve, Imgproc.arcLength(matOfPoint2f, true) * 0.02, true); long total = approxCurve.total(); if (total == 4) { ArrayList cos = new ArrayList(); Point[] points = approxCurve.toArray(); for (int j = 2; j < total + 1; j++) { cos.Add(angle(points[(int)(j % total)], points[j - 2], points[j - 1])); } cos.Sort(); Double minCos = (Double)cos[0]; Double maxCos = (Double)cos[cos.Count - 1]; bool isRect = total == 4 && minCos >= -0.1 && maxCos <= 0.3; if (isRect) { if (rect.width > 20) { rects.Add(rect); } List <double[]> Colors = new List <double[]>(); for (int op = 0; op < 10; op++) { if (rects.Count == 9) { allCubiesScaned = true; Color[] blockOfColour = imgTexture.GetPixels(rect.x + rect.width / 2, rect.y + rect.height, rect.width / 3, rect.height / 3, 0); float r = 0, g = 0, b = 0; foreach (Color pixelBlock in blockOfColour) { r += pixelBlock.r; g += pixelBlock.g; b += pixelBlock.b; } r = r / blockOfColour.Length; g = g / blockOfColour.Length; b = b / blockOfColour.Length; Rgb rgb = new Rgb(r, g, b); Colors.Add(new double[] { rgb.R * 255, rgb.G * 255, rgb.B * 255 }); print(Colors.Count); if (Colors.Count == 9) { ColorMap.Colors = Colors; ColorMap.Redraw(); } } } Imgproc.drawContours(imgMat, contours, idx, new Scalar(255, 100, 155), 4); } } } } catch (ArgumentOutOfRangeException e) { } Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false); Utils.matToTexture2D(imgMat, texture); gameObject.GetComponent <Renderer>().material.mainTexture = texture; }
void FormatImageSquare() { Mat mainMat = new Mat(baseTexture.height, baseTexture.width, CvType.CV_8UC3); Mat grayMat = new Mat(); //Convert Texture2d to Matrix Utils.texture2DToMat(baseTexture, mainMat); //copy main matrix to grayMat mainMat.copyTo(grayMat); //Convert color to gray Imgproc.cvtColor(grayMat, grayMat, Imgproc.COLOR_BGR2GRAY); //Blur Imgproc.GaussianBlur(grayMat, grayMat, new Size(25, 25), 0); //contrast Imgproc.threshold(grayMat, grayMat, 0, 255, Imgproc.THRESH_OTSU); //extract edge Imgproc.Canny(grayMat, grayMat, 50, 50); //prepare for the finding contours List <MatOfPoint> contours = new List <MatOfPoint>(); //find the contour from canny edge image Imgproc.findContours(grayMat, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); List <MatOfPoint> tempTargets = new List <MatOfPoint>(); for (int i = 0; i < contours.Count; i++) { MatOfPoint cp = contours[i]; MatOfPoint2f cn = new MatOfPoint2f(cp.toArray()); double p = Imgproc.arcLength(cn, true); MatOfPoint2f approx = new MatOfPoint2f(); //convret to polygon Imgproc.approxPolyDP(cn, approx, 0.03 * p, true); //find a contour with four points and large area int minContourArea = 10000; if (approx.toArray().Length == 4 && Imgproc.contourArea(approx) > minContourArea) { MatOfPoint approxPt = new MatOfPoint(); approx.convertTo(approxPt, CvType.CV_32S); float maxCosine = 0; for (int j = 2; j < 5; j++) { Vector2 v1 = new Vector2((float)(approx.toArray()[j % 4].x - approx.toArray()[j - 1].x), (float)(approx.toArray()[j % 4].y - approx.toArray()[j - 1].y)); Vector2 v2 = new Vector2((float)(approx.toArray()[j - 2].x - approx.toArray()[j - 1].x), (float)(approx.toArray()[j - 2].y - approx.toArray()[j - 1].y)); float angle = Mathf.Abs(Vector2.Angle(v1, v2)); maxCosine = Mathf.Max(maxCosine, angle); } if (maxCosine < 135f) { tempTargets.Add(approxPt); } } } if (tempTargets.Count > 0) { //Get the first contour MatOfPoint approxPt = tempTargets[0]; //Making Source Mat Mat srcPointMat = Converters.vector_Point_to_Mat(approxPt.toList(), CvType.CV_32F); //Making Destination Mat /*change these values*/ List <Point> dstPoints = new List <Point>(); dstPoints.Add(new Point(512, 0)); dstPoints.Add(new Point(0, 0)); dstPoints.Add(new Point(0, 512)); dstPoints.Add(new Point(512, 512)); Mat dstPointMat = Converters.vector_Point_to_Mat(dstPoints, CvType.CV_32F); //Make Perp transform Mat M = Imgproc.getPerspectiveTransform(srcPointMat, dstPointMat); Mat warpedMat = new Mat(mainMat.size(), mainMat.type()); //Crop and warp the image Imgproc.warpPerspective(mainMat, warpedMat, M, new Size(512, 512), Imgproc.INTER_LINEAR); warpedMat.convertTo(warpedMat, CvType.CV_8UC3); //Convert color to gray Imgproc.cvtColor(warpedMat, warpedMat, Imgproc.COLOR_BGR2GRAY); ////Blur //Imgproc.GaussianBlur(warpedMat, warpedMat, new Size(25, 25), 0); //contrast Imgproc.threshold(warpedMat, warpedMat, 0, 255, Imgproc.THRESH_OTSU); //resize Imgproc.resize(warpedMat, warpedMat, new Size(28, 28)); //Create an empty final texture finalTexture = new Texture2D(warpedMat.width(), warpedMat.height(), TextureFormat.RGB24, false); //Convert material to texture2d Utils.matToTexture2D(warpedMat, finalTexture); targetRawImage.texture = finalTexture; } }
void Process(){ Mat hierarchy = new Mat (); List<MatOfPoint> contours = new List<MatOfPoint> (); MatOfPoint maxitem = new MatOfPoint (); MatOfInt hullInt = new MatOfInt (); frameclone = frame_thresh_final.clone (); Imgproc.findContours (frameclone, contours, hierarchy, Imgproc.RETR_LIST , Imgproc.CHAIN_APPROX_NONE); maxitem = contours[0]; n = 0; for(int i=0; i<contours.Count; i++){ if(contours[i].total() > maxitem.total()){ maxitem = contours[i]; n=i; } } OpenCVForUnity.Rect bRect = Imgproc.boundingRect (maxitem); int bRect_height = bRect.height; int bRect_width = bRect.width; Imgproc.drawContours(frame_thresh_final, contours, n, new Scalar(255, 255, 255), -1); Imgproc.convexHull( maxitem, hullInt); List<Point> maxitemPointList = maxitem.toList (); List<int> hullIntList = hullInt.toList (); List<Point> hullPointList = new List<Point> (); for (int j=0; j < hullInt.toList().Count; j++) { hullPointList.Add (maxitemPointList [hullIntList[j]]); } MatOfPoint hullPointMat = new MatOfPoint (); hullPointMat.fromList (hullPointList); List<MatOfPoint> hullPoints = new List<MatOfPoint> (); hullPoints.Add (hullPointMat); //Imgproc.drawContours (frame, hullPoints, -1, new Scalar (0, 255, 0), 2); MatOfInt4 convexityDef = new MatOfInt4 (); Imgproc.convexityDefects (maxitem, hullInt, convexityDef); List<int> conDefIntList = convexityDef.toList (); List<Point> startpts = new List<Point> (); List<Point> farpts = new List<Point> (); List<Point> endpts = new List<Point> (); double defx1 = 1000, defx2 = 1000; int countx1 = 0, countx2 = 0; int tolerance = (int)(bRect_height/5.5); int count = 0, index = 0; //Debug.Log ("Tolerance: " + tolerance); double angleTol = 95.0; int[] defarray = new int[100]; //CvFont font = new CvFont (FontFace.Vector0, 1.0, 1.0); for(int i=0; i < conDefIntList.Count/4 ; i++){ startpts.Add(maxitemPointList[conDefIntList[4*i]]); endpts.Add(maxitemPointList[conDefIntList[4*i+1]]); farpts.Add(maxitemPointList[conDefIntList[4*i+2]]); Point s = startpts[i]; Point e = endpts[i]; Point f = farpts[i]; if( GetAngle(s, f, e) < angleTol && GetDistance(s,f) > tolerance && GetDistance(e,f) > tolerance ){ //string text = Convert.ToString(count); //Debug.Log("Depth1: "+GetDistance(s,f)); //Debug.Log("Depth2: "+GetDistance(e,f)); //Core.circle( frame_pot, f, 10, new Scalar(0, 0, 255), -1); //Core.circle( frame_pot, s, 10, new Scalar(0, 255, 0), -1); //Core.circle( frame_pot, e, 10, new Scalar(255, 0, 0), -1); //Core.putText(frame_pot, text, f, Core.FONT_HERSHEY_COMPLEX , 1.0, new Scalar(255, 255, 255)); //frame_pot.PutText(text, f, font, CvColor.White); if(f.x < defx1){ defx2 = defx1; countx2 = countx1; defx1 = f.x; countx1 = count; } else if(f.x < defx2) { defx2 = f.x; countx2 = count; } defarray[count] = index; count++; } index++; } //Debug.Log ("Count: " + count); //Debug.Log ("Total: " + farpts.Count); Point point1 = farpts [defarray [countx1]]; Point point2 = farpts [defarray [countx2]]; //Core.circle (frame_pot, point1, 15, new Scalar (255, 0, 0), 2); //Core.circle (frame_pot, point2, 15, new Scalar (255, 0, 0), 2); point1.y -= 5; double posX = (point1.x + point2.x)/2.0; double posY = (point1.y + point2.y)/2.0; posX_new = (float)(posX - 240); posY_new = (float)(-posY + 320); double dist = Math.Sqrt(Math.Pow(point1.x - point2.x, 2) + Math.Pow(point1.y - point2.y, 2)); scale1 = dist * 500000 / 640.0; scale2 = dist * 700 / 640.0; scale3 = dist * 600 / 640.0; scale4 = dist * 15 / 640.0; scale5 = dist * 70 / 640.0; ringObj[0].transform.position = new Vector3(posX_new, posY_new, 0.0f); ringObj[1].transform.position = new Vector3(posX_new, posY_new, 0.0f); ringObj[2].transform.position = new Vector3(posX_new, posY_new, 0.0f); ringObj[3].transform.position = new Vector3(posX_new, posY_new, 0.0f); ringObj[4].transform.position = new Vector3(posX_new, posY_new, 0.0f); ringObj[0].transform.localScale = new Vector3((float)scale1, (float)scale1, (float)(scale1*1.5)); ringObj[1].transform.localScale = new Vector3((float)scale2, (float)scale2, (float)(scale2)); ringObj[2].transform.localScale = new Vector3((float)scale3, (float)scale3, (float)(scale3)); ringObj[3].transform.localScale = new Vector3((float)scale4, (float)scale4, (float)(scale4)); ringObj[4].transform.localScale = new Vector3((float)scale5, (float)scale5, (float)(scale5)); Point point3 = new Point(point1.x, point2.y); angle_rot = GetAngle( point1, point2, point3); ringObj[0].transform.RotateAround( new Vector3(posX_new, posY_new, 0.0f), Vector3.forward, (float)angle_rot); ringObj[1].transform.RotateAround( new Vector3(posX_new, posY_new, 0.0f), Vector3.forward, (float)angle_rot); ringObj[2].transform.RotateAround( new Vector3(posX_new, posY_new, 0.0f), Vector3.forward, (float)angle_rot); ringObj[3].transform.RotateAround( new Vector3(posX_new, posY_new, 0.0f), Vector3.forward, (float)angle_rot); ringObj[4].transform.RotateAround( new Vector3(posX_new, posY_new, 0.0f), Vector3.forward, (float)angle_rot); }
private void HandPoseEstimationProcess(Mat rgbaMat) { // rgbaMat.copyTo(mRgba); float DOWNSCALE_RATIO = 1.0f; if (enableDownScale) { mRgba = imageOptimizationHelper.GetDownScaleMat(rgbaMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { // mRgba = rgbaMat; rgbaMat.copyTo(mRgba); DOWNSCALE_RATIO = 1.0f; } // Imgproc.blur(mRgba, mRgba, new Size(5,5)); Imgproc.GaussianBlur(mRgba, mRgba, new Size(3, 3), 1, 1); // Imgproc.medianBlur(mRgba, mRgba, 3); if (!isColorSelected) { return; } List <MatOfPoint> contours = detector.GetContours(); detector.Process(mRgba); // Debug.Log ("Contours count: " + contours.Count); if (contours.Count <= 0) { return; } RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[0].toArray())); double boundWidth = rect.size.width; double boundHeight = rect.size.height; int boundPos = 0; for (int i = 1; i < contours.Count; i++) { rect = Imgproc.minAreaRect(new MatOfPoint2f(contours[i].toArray())); if (rect.size.width * rect.size.height > boundWidth * boundHeight) { boundWidth = rect.size.width; boundHeight = rect.size.height; boundPos = i; } } MatOfPoint contour = contours[boundPos]; OpenCVForUnity.CoreModule.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray())); Imgproc.rectangle(mRgba, boundRect.tl(), boundRect.br(), CONTOUR_COLOR_WHITE, 2, 8, 0); // Debug.Log ( // " Row start [" + // (int)boundRect.tl ().y + "] row end [" + // (int)boundRect.br ().y + "] Col start [" + // (int)boundRect.tl ().x + "] Col end [" + // (int)boundRect.br ().x + "]"); double a = boundRect.br().y - boundRect.tl().y; a = a * 0.7; a = boundRect.tl().y + a; // Debug.Log (" A [" + a + "] br y - tl y = [" + (boundRect.br ().y - boundRect.tl ().y) + "]"); // Imgproc.rectangle(mRgba, boundRect.tl(), new Point(boundRect.br().x, a), CONTOUR_COLOR, 2, 8, 0); MatOfPoint2f pointMat = new MatOfPoint2f(); Imgproc.approxPolyDP(new MatOfPoint2f(contour.toArray()), pointMat, 3, true); contour = new MatOfPoint(pointMat.toArray()); MatOfInt hull = new MatOfInt(); MatOfInt4 convexDefect = new MatOfInt4(); Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull); if (hull.toArray().Length < 3) { return; } Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect); List <MatOfPoint> hullPoints = new List <MatOfPoint>(); List <Point> listPo = new List <Point>(); for (int j = 0; j < hull.toList().Count; j++) { listPo.Add(contour.toList()[hull.toList()[j]] * DOWNSCALE_RATIO); } /* * MatOfPoint e = new MatOfPoint(); * e.fromList(listPo); * hullPoints.Add(e); * * List<Point> listPoDefect = new List<Point>(); * * if (convexDefect.rows() > 0) * { * List<int> convexDefectList = convexDefect.toList(); * List<Point> contourList = contour.toList(); * for (int j = 0; j < convexDefectList.Count; j = j + 4) * { * Point farPoint = contourList[convexDefectList[j + 2]]; * int depth = convexDefectList[j + 3]; * if (depth > threshholdDetect && farPoint.y < a) * { * listPoDefect.Add(contourList[convexDefectList[j + 2]]); * Imgproc.line(rgbaMat, farPoint, listPo[convexDefectList[j + 2]], new Scalar(255, 0, 0, 255),1,1); * } * // Debug.Log ("convexDefectList [" + j + "] " + convexDefectList [j + 3]); * } * }*/ // Debug.Log ("hull: " + hull.toList ()); // if (convexDefect.rows () > 0) { // Debug.Log ("defects: " + convexDefect.toList ()); // } //Imgproc.drawContours (rgbaMat, hullPoints, -1, CONTOUR_COLOR, 3); for (int p = 0; p < listPo.Count; p++) { if (p % 2 == 0) { Imgproc.circle(rgbaMat, listPo[p], 6, new Scalar(255, 0, 0, 255), -1); // Imgproc.putText(rgbaMat,p.ToString(),listPo[p],1,1,new Scalar(255,0,0,255)); // check if close List <Point> fLMscaled = OpenCVForUnityUtils.ConvertVector2ListToPointList(facePoints); for (int q = 0; q < fLMscaled.Count; q++) { if (ifLessThanDPoint(listPo[p], fLMscaled[q], 8)) { //Point1 = listPo[p]; //Point2 = fLMscaled[q]; handPoint = p; facePoint = q; print(Point1 + " " + Point2); } } if (p == handPoint && facePoint != 0) { Point1 = listPo[p]; Point2 = fLMscaled[facePoint]; Imgproc.line(rgbaMat, Point1, Point2, new Scalar(255, 255, 255, 255)); } } } // int defectsTotal = (int)convexDefect.total(); // Debug.Log ("Defect total " + defectsTotal); /* numberOfFingers = listPoDefect.Count; * if (numberOfFingers > 5) * numberOfFingers = 5;/ * * // Debug.Log ("numberOfFingers " + numberOfFingers); * * // Imgproc.putText (rgbaMat, "" + numberOfFingers, new Point (rgbaMat.cols () / 2, rgbaMat.rows () / 2), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar (255, 255, 255, 255), 6, Imgproc.LINE_AA, false); * * * /* foreach (Point p in listPoDefect) { * * Point tempp = GetNearestL(p, listPo); * tempp = ConvertDownscale(tempp, DOWNSCALE_RATIO); * Point p2 = ConvertDownscale(p, DOWNSCALE_RATIO); * * Imgproc.circle (rgbaMat, tempp, 6, new Scalar (0, 0, 255, 255), -1); * Imgproc.circle(rgbaMat, p2, 6, new Scalar(255, 0, 255, 255), -1); * }*/ }
private Mat findPaper(Mat mainMat) { Imgproc.cvtColor(grayMat, grayMat, Imgproc.COLOR_BGR2GRAY); // blur image Imgproc.GaussianBlur(grayMat, grayMat, new Size(5, 5), 0); grayMat.get(0, 0, grayPixels); for (int i = 0; i < grayPixels.Length; i++) { maskPixels[i] = 0; if (grayPixels[i] < 70) { grayPixels[i] = 0; //maskPixels [i] = 1; } else if (70 <= grayPixels[i] && grayPixels[i] < 120) { grayPixels[i] = 100; } else { grayPixels[i] = 255; //maskPixels [i] = 1; } } grayMat.put(0, 0, grayPixels); //thresholding make mage blake and white Imgproc.threshold(grayMat, grayMat, 0, 255, Imgproc.THRESH_OTSU); //extract the edge image Imgproc.Canny(grayMat, grayMat, 50, 50); //prepare for finding contours List <MatOfPoint> contours = new List <MatOfPoint>(); Imgproc.findContours(grayMat, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); List <MatOfPoint> tmpTargets = new List <MatOfPoint>(); for (int i = 0; i < contours.Count; i++) { MatOfPoint cp = contours[i]; MatOfPoint2f cn = new MatOfPoint2f(cp.toArray()); double p = Imgproc.arcLength(cn, true); MatOfPoint2f approx = new MatOfPoint2f(); // lager skew greater 0.03? //convert contours to readable polyagon Imgproc.approxPolyDP(cn, approx, 0.03 * p, true); //find contours with 4 points if (approx.toArray().Length == 4) { MatOfPoint approxPt = new MatOfPoint(); approx.convertTo(approxPt, CvType.CV_32S); float maxCosine = 0; float rate = 0; float min_length = 100000000000000; for (int j = 2; j < 5; j++) { Vector2 v1 = new Vector2((float)(approx.toArray()[j % 4].x - approx.toArray()[j - 1].x), (float)(approx.toArray()[j % 4].y - approx.toArray()[j - 1].y)); Vector2 v2 = new Vector2((float)(approx.toArray()[j - 2].x - approx.toArray()[j - 1].x), (float)(approx.toArray()[j - 2].y - approx.toArray()[j - 1].y)); float v1_length = Mathf.Sqrt(v1.x * v1.x + v1.y * v1.y); float v2_length = Mathf.Sqrt(v2.x * v2.x + v2.y * v2.y); min_length = Mathf.Min(Mathf.Min((float)(v1_length), (float)v2_length), min_length); if (v1_length > v2_length) { rate = v2_length / v1_length; } else { rate = v1_length / v2_length; } float angle = Mathf.Abs(Vector2.Angle(v1, v2)); maxCosine = Mathf.Max(maxCosine, angle); } if (min_length > 100 && maxCosine < 135f)// && rate >= 0.6 maxCosine < 135f && { tmpTargets.Add(approxPt); //Debug.Log("Length -----------" + min_length); //Debug.Log("------------rate" + rate + "---------------"); } } } if (tmpTargets.Count > 0) { // -----------------------DRAW RECTANGLE--------------------------- //MatOfPoint2f approxCurve = new MatOfPoint2f(); //for (int i = 0; i < tmpTargets.Count; i++) //{ // //Convert contours(i) from MatOfPoint to MatOfPoint2f // MatOfPoint2f contour2f = new MatOfPoint2f(tmpTargets[i].toArray()); // //Processing on mMOP2f1 which is in type MatOfPoint2f // double approxDistance = Imgproc.arcLength(contour2f, true) * 0.02; // Imgproc.approxPolyDP(contour2f, approxCurve, approxDistance, true); // //Convert back to MatOfPoint // MatOfPoint points = new MatOfPoint(approxCurve.toArray()); // // Get bounding rect of contour // OpenCVForUnity.Rect rect = Imgproc.boundingRect(points); // // draw enclosing rectangle (all same color, but you could use variable i to make them unique) // Imgproc.rectangle(mainMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(0, 130, 255), 3); // Imgproc.rectangle(mainMat, new Point(rect.x, rect.y), new Point(rect.x + 5, rect.y + 5), new Scalar(0, 0, 255), 5); // Imgproc.rectangle(mainMat, new Point(rect.x + rect.width, rect.y), new Point(rect.x + +rect.width + 5, rect.y + 5), new Scalar(0, 0, 255), 5); // Imgproc.rectangle(mainMat, new Point(rect.x + rect.width, rect.y + rect.height), new Point(rect.x + +rect.width + 5, rect.y + rect.height + 5), new Scalar(0, 0, 255), 5); // Imgproc.rectangle(mainMat, new Point(rect.x, rect.y + rect.height), new Point(rect.x + 5, rect.y + rect.height + 5), new Scalar(0, 0, 255), 5); //} // -----------------------DRAW RECTANGLE--------------------------- // get the first contours int largestPaper = findLargestContour(tmpTargets); //Debug.Log(largestPaper); // using the largest one paperCornerMatOfPoint = tmpTargets[largestPaper]; // draw boundary Imgproc.line(mainMat, paperCornerMatOfPoint.toList()[0], paperCornerMatOfPoint.toList()[1], new Scalar(0, 255, 0), 3); Imgproc.line(mainMat, paperCornerMatOfPoint.toList()[0], paperCornerMatOfPoint.toList()[3], new Scalar(0, 255, 0), 3); Imgproc.line(mainMat, paperCornerMatOfPoint.toList()[2], paperCornerMatOfPoint.toList()[3], new Scalar(0, 255, 0), 3); Imgproc.line(mainMat, paperCornerMatOfPoint.toList()[1], paperCornerMatOfPoint.toList()[2], new Scalar(0, 255, 0), 3); // extract target from the frame and adjust some angle.... Mat srcPointsMat = Converters.vector_Point_to_Mat(paperCornerMatOfPoint.toList(), CvType.CV_32F); List <Point> dstPoints = new List <Point>(); dstPoints.Add(new Point(0, 0)); dstPoints.Add(new Point(0, 300)); dstPoints.Add(new Point(200, 300)); dstPoints.Add(new Point(200, 0)); Mat dstPointsMat = Converters.vector_Point_to_Mat(dstPoints, CvType.CV_32F); //Make perspective transform Mat m = Imgproc.getPerspectiveTransform(srcPointsMat, dstPointsMat); Mat warpedMat = new Mat(mainMat.size(), mainMat.type()); Imgproc.warpPerspective(mainMat, warpedMat, m, new Size(200, 300), Imgproc.INTER_LINEAR); warpedMat.convertTo(warpedMat, CvType.CV_8UC3); Texture2D finalTargetTextue = new Texture2D(warpedMat.width(), warpedMat.height(), TextureFormat.RGB24, false); Utils.matToTexture2D(warpedMat, finalTargetTextue); targetRawImage.texture = finalTargetTextue; //Debug.Log(paperCornerMatOfPoint.toList()[0].ToString() + " " + paperCornerMatOfPoint.toList()[1].ToString()+ " " + paperCornerMatOfPoint.toList()[2].ToString()+ " " + paperCornerMatOfPoint.toList()[3].ToString()); } //-------------------------------------------------------- return(mainMat); }
void Run() { rgbaMat = Imgcodecs.imread(Application.dataPath + "/Resources/changer.jpg", 1); Imgproc.cvtColor(rgbaMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); //1. 人脸dlib检测 List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect>(); OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (UnityEngine.Rect unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); OpenCVForUnity.Rect openCVRect = detectResult[0]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); //通过检测器从rect中提取point landmarkPoints.Add(points); //2. 计算凸包 pointList = new List <Point>(); for (int i = 0; i < points.Count; i++) { //绘制点 //Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 2, new Scalar(255, 255, 255), -1); Point pt = new Point(landmarkPoints[0][i].x, landmarkPoints[0][i].y); Imgproc.circle(rgbaMat, pt, 0, new Scalar(0, 255, 0, 255), 2, Imgproc.LINE_8, 0); //绘制68点 pointList.Add(pt); } //Debug.Log(pointList.Count); //68 //3. 三角剖份 TriangleDivide(); //4. (遍历三角形)仿射变换 Affine(); //5. 显示 MatOfPoint pointsMat = new MatOfPoint(); pointsMat.fromList(pointList); MatOfInt hullInt = new MatOfInt(); Imgproc.convexHull(pointsMat, hullInt); List <Point> pointMatList = pointsMat.toList(); List <int> hullIntList = hullInt.toList(); List <Point> hullPointList = new List <Point>(); for (int j = 0; j < hullInt.toList().Count; j++) { hullPointList.Add(pointMatList[hullIntList[j]]); } MatOfPoint hullPointMat = new MatOfPoint(); hullPointMat.fromList(hullPointList); //23*1 List <MatOfPoint> hullPoints = new List <MatOfPoint>(); hullPoints.Add(hullPointMat); //1 Imgproc.drawContours(rgbaMat, hullPoints, -1, new Scalar(255, 255, 0, 255), 2); //------------------------------------------------// //Try---------------------------------------------// dstPointList = new List <Point>(); for (int i = 0; i < pointList.Count; i++) { Point pt = pointList[i] + new Point(100, 0); dstPointList.Add(pt); } //MatOfPoint2f srcTri = new MatOfPoint2f(); //不能超过3 //MatOfPoint2f dstTri = new MatOfPoint2f(); //不能超过3 //srcTri.fromList(pointList); //srcTri.fromList(dstPointList); for (int j = 0; j < 3; j++) { //srcTri.push_back(hullPointMat); //dstTri.push_back(hull2[corpd.index[j]]); } Mat mask = Mat.zeros(rgbaMat.size(), CvType.CV_8UC1); Point p0 = new Point(0, 0); Point p1 = new Point(0, 256); Point p2 = new Point(256, 0); Point p3 = new Point(256, 0); Point p4 = new Point(512, 0); Point p5 = new Point(512, 256); Point p6 = new Point(256, 64); MatOfPoint pts1 = new MatOfPoint(new Point[3] { p0, p1, p2 }); MatOfPoint pts2 = new MatOfPoint(new Point[3] { p3, p4, p5 }); MatOfPoint2f srcTri = new MatOfPoint2f(new Point[3] { p0, p1, p2 }); MatOfPoint2f dstTri = new MatOfPoint2f(new Point[3] { p0, p1, p6 }); List <MatOfPoint> contour = new List <MatOfPoint>() { pts1 }; for (int i = 0; i < contour.Count; i++) { //轮廓提取 Imgproc.drawContours(mask, contour, i, new Scalar(255), -1); //全部放到mask上 } rgbaMat.copyTo(mask, mask); Mat warpMat = Imgproc.getAffineTransform(srcTri, dstTri); Mat warpImage = Mat.zeros(mask.size(), mask.type()); Imgproc.warpAffine(mask, warpImage, warpMat, warpImage.size()); //------------------------------------------------// Texture2D t2d = new Texture2D(rgbaMat.width(), rgbaMat.height(), TextureFormat.RGBA32, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, t2d); Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero); srcImage.sprite = sp; srcImage.preserveAspect = true; //warpImage Texture2D dst_t2d = new Texture2D(warpImage.width(), warpImage.height(), TextureFormat.RGBA32, false); OpenCVForUnity.Utils.matToTexture2D(warpImage, dst_t2d); Sprite dst_sp = Sprite.Create(dst_t2d, new UnityEngine.Rect(0, 0, dst_t2d.width, dst_t2d.height), Vector2.zero); dstImage.sprite = dst_sp; dstImage.preserveAspect = true; }
/*=============================================* * 輪郭ごとの頂点から手を判別するまで *=============================================*/ /// <summary> /// Contours to hand gesture. /// </summary> /// <param name="rgbaMat">Rgba mat.</param> /// <param name="contour">Contour.</param> private static void _contourToHandGesture(Mat rgbaMat, MatOfPoint contour) { try { //頂点を調査する準備をする _pointOfVertices(rgbaMat, contour); //基準輪郭のサイズの取得と描画(長方形) OpenCVForUnity.Rect boundRect = Imgproc.boundingRect(new MatOfPoint(contour.toArray())); Imgproc.rectangle(rgbaMat, boundRect.tl(), boundRect.br(), HGColorSpuiter.ColorToScalar(ContourRangeColor), 2, 8, 0); /*=============================================* * 腕まで含んだ手の大きさを取得する **=============================================*/ //腕まで含んだ手の大きさを識別する MatOfInt hull = new MatOfInt(); Imgproc.convexHull(new MatOfPoint(contour.toArray()), hull); //腕まで含んだ手の範囲を取得 List <Point> armPointList = new List <Point>(); for (int j = 0; j < hull.toList().Count; j++) { Point armPoint = contour.toList()[hull.toList()[j]]; bool addFlag = true; foreach (Point point in armPointList.ToArray()) { //輪郭の1/10より近い頂点は誤差としてまとめる double distance = Mathf.Sqrt((float)((armPoint.x - point.x) * (armPoint.x - point.x) + (armPoint.y - point.y) * (armPoint.y - point.y))); if (distance <= Mathf.Min((float)boundRect.width, (float)boundRect.height) / 10) { addFlag = false; break; } } if (addFlag) { armPointList.Add(armPoint); } } MatOfPoint armMatOfPoint = new MatOfPoint(); armMatOfPoint.fromList(armPointList); List <MatOfPoint> armPoints = new List <MatOfPoint>(); armPoints.Add(armMatOfPoint); //腕まで含んだ手の範囲を描画 Imgproc.drawContours(rgbaMat, armPoints, -1, HGColorSpuiter.ColorToScalar(ArmRangeColor), 3); //腕まで含んだ手が三角形の場合はそれ以上の識別が難しい if (hull.toArray().Length < 3) { return; } /*=============================================* * 掌の大きさを取得する **=============================================*/ //凸面の頂点から凹面の点のみを取得し、掌の範囲を取得する MatOfInt4 convexDefect = new MatOfInt4(); Imgproc.convexityDefects(new MatOfPoint(contour.toArray()), hull, convexDefect); //凹面の点をフィルタリングして取得 List <Point> palmPointList = new List <Point>(); for (int j = 0; j < convexDefect.toList().Count; j = j + 4) { Point farPoint = contour.toList()[convexDefect.toList()[j + 2]]; int depth = convexDefect.toList()[j + 3]; if (depth > depthThreashold && farPoint.y < boundRect.br().y - boundRect.tl().y) { palmPointList.Add(contour.toList()[convexDefect.toList()[j + 2]]); } } MatOfPoint palmMatOfPoint = new MatOfPoint(); palmMatOfPoint.fromList(palmPointList); List <MatOfPoint> palmPoints = new List <MatOfPoint>(); palmPoints.Add(palmMatOfPoint); //掌の範囲を描画 Imgproc.drawContours(rgbaMat, palmPoints, -1, HGColorSpuiter.ColorToScalar(PalmRangeColor), 3); /*=============================================* * 掌+指先の大きさを取得する **=============================================*/ //掌の位置を元に手首を除いた範囲を取得する List <Point> handPointList = new List <Point>(); handPointList.AddRange(armPointList.ToArray()); handPointList.Reverse(); handPointList.RemoveAt(0); handPointList.Insert(0, palmPointList.ToArray()[0]); handPointList.RemoveAt(handPointList.Count - 1); handPointList.Insert(handPointList.Count, palmPointList.ToArray()[palmPointList.Count - 1]); MatOfPoint handMatOfPoint = new MatOfPoint(); handMatOfPoint.fromList(handPointList); List <MatOfPoint> handPoints = new List <MatOfPoint>(); handPoints.Add(handMatOfPoint); Imgproc.drawContours(rgbaMat, handPoints, -1, HGColorSpuiter.ColorToScalar(HandRangeColor), 3); /*=============================================* * 指先の位置を取得する **=============================================*/ //掌の各頂点の中心を求める List <Point> palmCenterPoints = new List <Point>(); for (int i = 0; i < palmPointList.Count; i++) { Point palmPoint = palmPointList.ToArray()[i]; Point palmPointNext = new Point(); if (i + 1 < palmPointList.Count) { palmPointNext = palmPointList.ToArray()[i + 1]; } else { palmPointNext = palmPointList.ToArray()[0]; } Point palmCenterPoint = new Point((palmPoint.x + palmPointNext.x) / 2, (palmPoint.y + palmPointNext.y) / 2); palmCenterPoints.Add(palmCenterPoint); } //掌の頂点から最も近い手の頂点を求める for (int i = 0; i < palmCenterPoints.Count && i + 1 < handPointList.Count && i < 5; i++) { Point palmPoint = palmCenterPoints.ToArray()[i]; List <Point> fingerList = new List <Point>(); fingerList.Add(palmPoint); fingerList.Add(handPointList.ToArray()[i + 1]); MatOfPoint fingerPoint = new MatOfPoint(); fingerPoint.fromList(fingerList); List <MatOfPoint> fingerPoints = new List <MatOfPoint>(); fingerPoints.Add(fingerPoint); Imgproc.drawContours(rgbaMat, fingerPoints, -1, HGColorSpuiter.ColorToScalar(FingerRangeColor), 3); } // Imgproc.putText(rgbaMat, "", new Point(2, rgbaMat.rows()-30), Core.FONT_HERSHEY_SIMPLEX, 1.0, HGColorSpuiter.ColorToScalar(Color.black), 2, Imgproc.LINE_AA, false); } catch (System.Exception e) { Debug.Log(e.Message); } }