protected static MatOfPoint convertIndexToPoint(MatOfInt index, MatOfPoint contour) { Point[] arrPoint = contour.toArray(); int[] arrIndex = index.toArray(); Point[] arrResult = new Point[arrIndex.Length]; for (int i = 0; i < arrIndex.Length; i++) { arrResult[i] = arrPoint[arrIndex[i]]; } MatOfPoint hull = new MatOfPoint(); hull.fromArray(arrResult); return(hull); }
//public RawImage document; void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat mainMat = webCamTextureToMatHelper.GetMat(); if (!selectTarget) //find paper by contours { grayMat = new Mat(); // convert texture to matrix mainMat.copyTo(grayMat); mainMat = findPaper(mainMat); // display matrix on the screen Utils.fastMatToTexture2D(mainMat, texture); } else { // using optical flow // set the currentGrayMat mat currentGrayMat = new Mat(mainMat.rows(), mainMat.cols(), Imgproc.COLOR_RGB2GRAY); Imgproc.cvtColor(mainMat, currentGrayMat, Imgproc.COLOR_RGBA2GRAY); if (initOpticalFlow == true) // doing the init setting for optical flow { // create 40 points Point[] points = new Point[40]; // set those points near the corner // paperCornerMatOfPoint is the corner of the paper for (int i = 0; i < 4; i++) { points[i * 10] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 1] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 2] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y + 1); points[i * 10 + 3] = new Point(paperCornerMatOfPoint.toList()[i].x + 1, paperCornerMatOfPoint.toList()[i].y + 1); points[i * 10 + 4] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 1); points[i * 10 + 5] = new Point(paperCornerMatOfPoint.toList()[i].x - 1, paperCornerMatOfPoint.toList()[i].y); points[i * 10 + 6] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 1); points[i * 10 + 7] = new Point(paperCornerMatOfPoint.toList()[i].x, paperCornerMatOfPoint.toList()[i].y - 2); points[i * 10 + 8] = new Point(paperCornerMatOfPoint.toList()[i].x - 2, paperCornerMatOfPoint.toList()[i].y - 2); points[i * 10 + 9] = new Point(paperCornerMatOfPoint.toList()[i].x + 2, paperCornerMatOfPoint.toList()[i].y + 2); } // make the points closer to the corners (Harris Corner Detection ) //Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40, qualityLevel, minDistance, none, blockSize, false, 0.04); //Imgproc.goodFeaturesToTrack(currentGrayMat, corners, 40,0.05,20); corners.fromArray(points); prevFeatures.fromList(corners.toList()); currentFeatures.fromList(corners.toList()); prevGrayMat = currentGrayMat.clone(); // won't go back t again initOpticalFlow = false; // not that useful lol // create random color // not working now for (int i = 0; i < maxCorners; i++) { color.Add(new Scalar((int)(Random.value * 255), (int)(Random.value * 255), (int)(Random.value * 255), 255)); } } else { // Don't want ball move //currentFeatures.fromArray(prevFeatures.toArray()); // want ball move prevFeatures.fromArray(currentFeatures.toArray()); // optical flow it will changes the valu of currentFeatures Video.calcOpticalFlowPyrLK(prevGrayMat, currentGrayMat, prevFeatures, currentFeatures, mMOBStatus, err); //Debug.Log(st.rows()); // change to points list List <Point> prevList = prevFeatures.toList(), nextList = currentFeatures.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus[x] == 1) { Point pt = nextList[x]; Point pt2 = prevList[x]; Imgproc.circle(mainMat, pt, 10, new Scalar(0, 0, 255), -1); Imgproc.line(mainMat, pt, pt2, new Scalar(0, 0, 255)); } } // draw the data //for (int i = 0; i < prevList.Count; i++) //{ // //Imgproc.circle(frame, prevList[i], 5, color[10]); // Imgproc.circle(mainMat, nextList[i], 10, new Scalar(0, 0, 255), -1); // Imgproc.line(mainMat, prevList[i], nextList[i], color[20]); //} List <List <Point> > cornersFeatures = new List <List <Point> >(40); cornersFeatures.Add(new List <Point>(10)); // put the corners features data into the list int tmp = 0; bool last = true; for (int i = 0; i < nextList.Count - 1; i++) { if (Mathf.Abs((float)(nextList[i].x - nextList[i + 1].x)) < 10 && Mathf.Abs((float)(nextList[i].y - nextList[i + 1].y)) < 10) { if (last == true) { cornersFeatures[tmp].Add(nextList[i]); } else { cornersFeatures.Add(new List <Point>(10)); tmp = tmp + 1; cornersFeatures[tmp].Add(nextList[i]); } last = true; } else { last = false; } } // count corners int manyCornersFeatures = 0; for (int i = 0; i < cornersFeatures.Count; i++) { Debug.Log(cornersFeatures[i].Count); if (cornersFeatures[i].Count < 5) { cornersFeatures.RemoveAt(i); } else { manyCornersFeatures++; } } //Debug.Log("Length" + manyCornersFeatures); // if corners equal 4 then diplay virtual docunment into the frame // doing the perspective transform if (manyCornersFeatures == 4) { Mat documentMat = new Mat(document.height, document.width, CvType.CV_8UC3); Utils.texture2DToMat(document, documentMat); List <Point> srcPoints = new List <Point>(); srcPoints.Add(new Point(0, 0)); srcPoints.Add(new Point(documentMat.cols(), 0)); srcPoints.Add(new Point(documentMat.cols(), documentMat.rows())); srcPoints.Add(new Point(0, documentMat.rows())); Mat srcPointsMat = Converters.vector_Point_to_Mat(srcPoints, CvType.CV_32F); List <Point> dstPoints = new List <Point>() { cornersFeatures[0][0], cornersFeatures[1][0], cornersFeatures[2][0], cornersFeatures[3][0] }; Mat dstPointsMat = Converters.vector_Point_to_Mat(dstPoints, CvType.CV_32F); //Make perspective transform Mat m = Imgproc.getPerspectiveTransform(srcPointsMat, dstPointsMat); Mat warpedMat = new Mat(new Size(), documentMat.type()); Debug.Log((cornersFeatures[1][0].x - cornersFeatures[0][0].x) + " " + (cornersFeatures[2][0].y - cornersFeatures[1][0].y)); Imgproc.warpPerspective(documentMat, warpedMat, m, mainMat.size(), Imgproc.INTER_LINEAR); //warpedMat.convertTo(warpedMat, CvType.CV_32F); //warpedMat.convertTo(warpedMat, CvType.CV_8UC3); warpedMat.convertTo(warpedMat, CvType.CV_8UC3); // same size as frame Mat dst = new Mat(mainMat.size(), CvType.CV_8UC3); //Mat dst = new Mat(frame.size(), CvType.CV_8UC3); //Mat dst2 = new Mat(); Imgproc.cvtColor(mainMat, dst, Imgproc.COLOR_RGBA2RGB); //dst.setTo(new Scalar(0, 255, 0)); //currentGrayMat.copyTo(dst); //dst.convertTo(dst, CvType.CV_8UC3); //Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA); Mat img1 = new Mat(); Mat mask = new Mat(mainMat.size(), CvType.CV_8UC1, new Scalar(0)); Imgproc.cvtColor(warpedMat, img1, Imgproc.COLOR_RGB2GRAY); Imgproc.Canny(img1, img1, 100, 200); List <MatOfPoint> doc_contours = new List <MatOfPoint>();; Imgproc.findContours(img1, doc_contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_NONE); Imgproc.drawContours(mask, doc_contours, -1, new Scalar(255), Core.FILLED); warpedMat.copyTo(dst, mask); dst.convertTo(dst, CvType.CV_8UC3); Debug.Log("dst" + dst.size()); Imgproc.cvtColor(dst, mainMat, Imgproc.COLOR_RGB2RGBA); // display on the right Texture2D finalTextue = new Texture2D(dst.width(), dst.height(), TextureFormat.RGB24, false); Utils.matToTexture2D(dst, finalTextue); targetRawImage.texture = finalTextue; } // current frame to old frame prevGrayMat = currentGrayMat.clone(); //Imgproc.cvtColor(currentGrayMat, frame, Imgproc.COLOR_GRAY2RGBA); // display matrix on the screen Utils.fastMatToTexture2D(mainMat, texture); } } } }
public int getAnswerNumber(Mat align, Rect r) { Mat roi = new Mat(align, r); Mat roi_gray = new Mat(), roi_edges = new Mat(); Imgproc.cvtColor(roi, roi_gray, Imgproc.COLOR_RGB2GRAY); Imgproc.Canny(roi_gray, roi_edges, 200, 200); // Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new Size(2 + 1, 2 + 1), new Point(1, 1)); // Imgproc.dilate(roi_edges, roi_edges, element); //Shape detection List <MatOfPoint> contours = new List <MatOfPoint>(); Mat hierarchy = new Mat(); Imgproc.findContours(roi_edges, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0)); List <MatOfPoint> hulls = new List <MatOfPoint>(); for (int i = 0; i < contours.Count; i++) { MatOfInt hull_temp = new MatOfInt(); Imgproc.convexHull(contours[i], hull_temp); int[] arrIndex = hull_temp.toArray(); Point[] arrContour = contours[i].toArray(); Point[] arrPoints = new Point[arrIndex.Length]; for (int k = 0; k < arrIndex.Length; k++) { arrPoints[k] = arrContour[arrIndex[k]]; } MatOfPoint temp = new MatOfPoint(); temp.fromArray(arrPoints); //Filter outliers if (Imgproc.contourArea(temp) > 40 && Imgproc.contourArea(temp) < 200) { hulls.Add(temp); } } List <MatOfPoint2f> hull2f = new List <MatOfPoint2f>(); for (int i = 0; i < hulls.Count; i++) { MatOfPoint2f newPoint = new MatOfPoint2f(hulls[i].toArray()); hull2f.Add(newPoint); } for (int i = 0; i < hulls.Count; i++) { //Approximate polygon MatOfPoint2f approx = new MatOfPoint2f(); Imgproc.approxPolyDP(hull2f[i], approx, 0.01 * Imgproc.arcLength(hull2f[i], true), true); List <Point> approx_polygon = approx.toList(); approx_polygon = Scannerproc.filterPolygon(approx_polygon); double area = Imgproc.contourArea(approx); //Center of mass int cx = 0, cy = 0; for (int k = 0; k < approx_polygon.Count; k++) { cx += (int)approx_polygon[k].x; cy += (int)approx_polygon[k].y; } cx /= approx_polygon.Count; cy /= approx_polygon.Count; // Imgproc.circle(roi, new Point(cx, cy), 5, new Scalar(255), -1); // Texture2D tex = new Texture2D(roi.width(), roi.height(), TextureFormat.RGB24, false); // Utils.matToTexture2D(roi, tex); // byte[] bytes1 = tex.EncodeToJPG(); // File.WriteAllBytes("D:/2019/OMR/" + "test.png", bytes1); Point pos1 = new Point((roi.width() * 1) / 10, cy); Point pos2 = new Point((roi.width() * 3) / 10, cy); Point pos3 = new Point((roi.width() * 5) / 10, cy); Point pos4 = new Point((roi.width() * 7) / 10, cy); Point pos5 = new Point((roi.width() * 9) / 10, cy); Point nowPos = new Point(cx, cy); double[] dist = new double[5]; dist[0] = Scannerproc.distanceTwoPoints(pos1, nowPos); dist[1] = Scannerproc.distanceTwoPoints(pos2, nowPos); dist[2] = Scannerproc.distanceTwoPoints(pos3, nowPos); dist[3] = Scannerproc.distanceTwoPoints(pos4, nowPos); dist[4] = Scannerproc.distanceTwoPoints(pos5, nowPos); int id = -1; double min_dist = 999999; for (int t = 0; t < 5; t++) { if (dist[t] < min_dist) { min_dist = dist[t]; id = t; } } return(id + 1); //return plusPoints(tl, new Point(cx, cy)); } return(0); }
public void getAnswerNumber(Mat align) { Mat align_gray = new Mat(), align_edges = new Mat(); Imgproc.cvtColor(align, align_gray, Imgproc.COLOR_RGB2GRAY); Imgproc.Canny(align_gray, align_edges, 50, 50); Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new Size(2 + 1, 2 + 1), new Point(1, 1)); Imgproc.dilate(align_edges, align_edges, element); //Shape detection List <MatOfPoint> contours = new List <MatOfPoint>(); Mat hierarchy = new Mat(); Imgproc.findContours(align_edges, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0)); List <MatOfPoint> hulls = new List <MatOfPoint>(); //Texture2D tex = new Texture2D(align_edges.width(), align_edges.height(), TextureFormat.RGB24, false); //Utils.matToTexture2D(align_edges, tex); //byte[] bytes1 = tex.EncodeToJPG(); //File.WriteAllBytes("D:/2019/OMR/" + "test.png", bytes1); for (int i = 0; i < contours.Count; i++) { MatOfInt hull_temp = new MatOfInt(); Imgproc.convexHull(contours[i], hull_temp); int[] arrIndex = hull_temp.toArray(); Point[] arrContour = contours[i].toArray(); Point[] arrPoints = new Point[arrIndex.Length]; for (int k = 0; k < arrIndex.Length; k++) { arrPoints[k] = arrContour[arrIndex[k]]; } MatOfPoint temp = new MatOfPoint(); temp.fromArray(arrPoints); //Filter outliers if (Imgproc.contourArea(temp) > 90000 && Imgproc.contourArea(temp) < 110000) { hulls.Add(temp); } } List <MatOfPoint2f> hull2f = new List <MatOfPoint2f>(); for (int i = 0; i < hulls.Count; i++) { MatOfPoint2f newPoint = new MatOfPoint2f(hulls[i].toArray()); hull2f.Add(newPoint); } List <Rect> rects = new List <Rect>(); for (int i = 0; i < hulls.Count; i++) { //Approximate polygon MatOfPoint2f approx = new MatOfPoint2f(); Imgproc.approxPolyDP(hull2f[i], approx, 0.01 * Imgproc.arcLength(hull2f[i], true), true); List <Point> approx_polygon = approx.toList(); approx_polygon = Scannerproc.filterPolygon(approx_polygon); double area = Imgproc.contourArea(approx); if (Scannerproc.isSquare(approx_polygon)) { Rect r = Imgproc.boundingRect(new MatOfPoint(approx_polygon.ToArray())); bool isContain = false; for (int k = 0; k < rects.Count; k++) { if (Scannerproc.distanceTwoPoints(rects[k].tl(), r.tl()) < 100) { //if (rects[k].contains(r) || r.contains(rects[k])) isContain = true; } } if (!isContain) { rects.Add(r); // Imgproc.rectangle(align, r.tl(), r.br(), new Scalar(255, 0, 0, 255), 3); for (int j = 1; j < 21; j++) { Rect roi = new Rect((int)r.tl().x + (int)((r.width * 1.3) / 6), (int)r.tl().y + (r.height / 21) * j, (int)((r.width * 4.7) / 6), r.height / 21); int num = getAnswerNumber(align, roi); if (num != 0) { Imgproc.putText(align, " " + num, new Point(roi.x - 40, roi.y + 25), 1, 2, new Scalar(255, 0, 0, 255), 3, Core.LINE_AA, false); Imgproc.rectangle(align, roi.tl(), roi.br(), new Scalar(0, 255, 0, 255), 2); } } } } //Center of mass int cx = 0, cy = 0; for (int k = 0; k < approx_polygon.Count; k++) { cx += (int)approx_polygon[k].x; cy += (int)approx_polygon[k].y; } cx /= approx_polygon.Count; cy /= approx_polygon.Count; // Imgproc.circle(roi, new Point(cx, cy), 5, new Scalar(255), -1); } if (rects.Count == 4) { nowDetected = false; } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { frame = webCamTextureToMatHelper.GetMat(); frame.copyTo(img_orig); drawing = img_orig.clone(); int lowThreshold = 50;// (int)200;// slider.value; const int ratio = 1; const int kernel_size = 3; Imgproc.cvtColor(img_orig, img_lab, Imgproc.COLOR_BGR2Lab); double omrSize = img_orig.cols() * img_orig.rows(); Imgproc.cvtColor(img_orig, img_gray, Imgproc.COLOR_RGBA2GRAY); Imgproc.GaussianBlur(img_gray, img_gray, new Size(15, 15), 1.5, 1.5); //Gaussian blur Imgproc.erode(img_gray, img_gray, new Mat(), new Point(-1, -1), 1); //Erosion // Imgproc.dilate(img_gray, img_gray, new Mat(), new Point(-1, -1), 10, 1, new Scalar(10)); //Dilation Imgproc.Canny(img_gray, img_edges, lowThreshold, lowThreshold * ratio, kernel_size, false); //Shape detection List <MatOfPoint> contours = new List <MatOfPoint>(); Mat hierarchy = new Mat(); Imgproc.findContours(img_edges, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0)); //Texture2D tex = new Texture2D(img_edges.width(), img_edges.height(), TextureFormat.RGB24, false); //Utils.matToTexture2D(img_edges, tex); //byte[] bytes1 = tex.EncodeToJPG(); //File.WriteAllBytes("D:/2019/OMR/" + "test213123.png", bytes1); List <MatOfPoint> hulls = new List <MatOfPoint>(); for (int i = 0; i < contours.Count; i++) { MatOfInt hull_temp = new MatOfInt(); Imgproc.convexHull(contours[i], hull_temp); int[] arrIndex = hull_temp.toArray(); Point[] arrContour = contours[i].toArray(); Point[] arrPoints = new Point[arrIndex.Length]; for (int k = 0; k < arrIndex.Length; k++) { arrPoints[k] = arrContour[arrIndex[k]]; } MatOfPoint temp = new MatOfPoint(); temp.fromArray(arrPoints); //Filter outliers if (Imgproc.contourArea(temp) > omrSize / 3 && Imgproc.contourArea(temp) < (omrSize * 4) / 5) { hulls.Add(temp); } } List <MatOfPoint2f> hull2f = new List <MatOfPoint2f>(); for (int i = 0; i < hulls.Count; i++) { MatOfPoint2f newPoint = new MatOfPoint2f(hulls[i].toArray()); hull2f.Add(newPoint); } for (int i = 0; i < hulls.Count; i++) { //Approximate polygon MatOfPoint2f approx = new MatOfPoint2f(); Imgproc.approxPolyDP(hull2f[i], approx, 0.01 * Imgproc.arcLength(hull2f[i], true), true); List <Point> approx_polygon = approx.toList(); // approx_polygon = Scannerproc.filterPolygon(approx_polygon); // Debug.Log(approx_polygon.Count); if (!Scannerproc.isSquare(approx_polygon)) { continue; } else { nowRectPoints.Clear(); nowRectPoints.AddRange(approx_polygon); perspectiveAlign(); } //Center of mass int cx = 0, cy = 0; for (int k = 0; k < approx_polygon.Count; k++) { cx += (int)approx_polygon[k].x; cy += (int)approx_polygon[k].y; } cx /= approx_polygon.Count; cy /= approx_polygon.Count; Scannerproc.drawShape(drawing, approx_polygon, new Scalar(0, 255, 0)); } if (showTextureOnScreen) { showCurrentTextureOnScreen(); } } }