// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (net.empty() || classes == null) { Imgproc.putText(rgbaMat, "model file or class names list file is not loaded.", new Point(5, rgbaMat.rows() - 50), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(rgbaMat, "The model and class names list can be downloaded here:", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(rgbaMat, "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } else { blob = Dnn.blobFromImage(rgbaMat, 1, new Size(224, 224), new Scalar(104, 117, 123), false); net.setInput(blob); Mat prob = net.forward(); Core.MinMaxLocResult minmax = Core.minMaxLoc(prob.reshape(1, 1)); // Debug.Log ("Best match " + (int)minmax.maxLoc.x); // Debug.Log ("Best match class " + classes [(int)minmax.maxLoc.x]); // Debug.Log ("Probability: " + minmax.maxVal * 100 + "%"); prob.Dispose(); Imgproc.putText(rgbaMat, "Best match class " + classes [(int)minmax.maxLoc.x], new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (cascade != null) { cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size()); } OpenCVForUnity.Rect[] rects = faces.toArray(); for (int i = 0; i < rects.Length; i++) { // Debug.Log ("detect faces " + rects [i]); Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2); } // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); //first find blue objects Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV); Core.inRange(hsvMat, blue.getHSVmin(), blue.getHSVmax(), thresholdMat); morphOps(thresholdMat); trackFilteredObject(blue, thresholdMat, hsvMat, rgbMat); //then yellows Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV); Core.inRange(hsvMat, yellow.getHSVmin(), yellow.getHSVmax(), thresholdMat); morphOps(thresholdMat); trackFilteredObject(yellow, thresholdMat, hsvMat, rgbMat); //then reds Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV); Core.inRange(hsvMat, red.getHSVmin(), red.getHSVmax(), thresholdMat); morphOps(thresholdMat); trackFilteredObject(red, thresholdMat, hsvMat, rgbMat); //then greens Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV); Core.inRange(hsvMat, green.getHSVmin(), green.getHSVmax(), thresholdMat); morphOps(thresholdMat); trackFilteredObject(green, thresholdMat, hsvMat, rgbMat); Imgproc.putText(rgbMat, "W:" + rgbMat.width() + " H:" + rgbMat.height() + " SO:" + Screen.orientation, new Point(5, rgbMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat circles = new Mat()) { Imgproc.HoughCircles(grayMat, circles, Imgproc.CV_HOUGH_GRADIENT, 2, 10, 160, 50, 10, 40); Point pt = new Point(); for (int i = 0; i < circles.cols(); i++) { double[] data = circles.get(0, i); pt.x = data [0]; pt.y = data [1]; double rho = data [2]; Imgproc.circle(rgbaMat, pt, (int)rho, new Scalar(255, 0, 0, 255), 5); } } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (net == null) { Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } else { Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR); blob = Dnn.blobFromImage(bgrMat, inScaleFactor, new Size(inWidth, inHeight), new Scalar(meanVal, meanVal, meanVal), false, false); net.setInput(blob); Mat prob = net.forward(); prob = prob.reshape(1, (int)prob.total() / 7); float[] data = new float[7]; float confidenceThreshold = 0.2f; for (int i = 0; i < prob.rows(); i++) { prob.get(i, 0, data); float confidence = data [2]; if (confidence > confidenceThreshold) { int class_id = (int)(data [1]); float left = data [3] * rgbaMat.cols(); float top = data [4] * rgbaMat.rows(); float right = data [5] * rgbaMat.cols(); float bottom = data [6] * rgbaMat.rows(); Imgproc.rectangle(rgbaMat, new Point(left, top), new Point(right, bottom), new Scalar(0, 255, 0, 255), 2); string label = classNames [class_id] + ": " + confidence; int[] baseLine = new int[1]; Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine); Imgproc.rectangle(rgbaMat, new Point(left, top), new Point(left + labelSize.width, top + labelSize.height + baseLine [0]), new Scalar(255, 255, 255, 255), Core.FILLED); Imgproc.putText(rgbaMat, label, new Point(left, top + labelSize.height), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255)); } } prob.Dispose(); } Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0) { rgbaMat.copyTo(bgMat); setBgTexture(bgMat); } //set fgMaskMat findFgMaskMat(rgbaMat, bgMat, thresh); //set bgMaskMat Core.bitwise_not(fgMaskMat, bgMaskMat); //copy greenMat using bgMaskMat greenMat.copyTo(rgbaMat, bgMaskMat); Imgproc.putText(rgbaMat, "SPACE KEY or TOUCH SCREEN: Reset backgroud img", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (net.empty() || classes == null) { Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } else { Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR); blob = Dnn.blobFromImage(bgrMat, 1, new Size(224, 224), new Scalar(104, 117, 123), false, true); net.setInput(blob); Mat prob = net.forward(); Core.MinMaxLocResult minmax = Core.minMaxLoc(prob.reshape(1, 1)); // Debug.Log ("Best match " + (int)minmax.maxLoc.x); // Debug.Log ("Best match class " + classes [(int)minmax.maxLoc.x]); // Debug.Log ("Probability: " + minmax.maxVal * 100 + "%"); prob.Dispose(); Imgproc.putText(rgbaMat, "Best match class " + classes [(int)minmax.maxLoc.x], new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); // Utils.webCamTextureToMat (webCamTexture, grayMat, colors); bgMat.copyTo(dstMat); Imgproc.GaussianBlur(grayMat, lineMat, new Size(3, 3), 0); grayMat.get(0, 0, grayPixels); for (int i = 0; i < grayPixels.Length; i++) { maskPixels [i] = 0; if (grayPixels [i] < 70) { grayPixels [i] = 0; maskPixels [i] = 1; } else if (70 <= grayPixels [i] && grayPixels [i] < 120) { grayPixels [i] = 100; } else { grayPixels [i] = 255; maskPixels [i] = 1; } } grayMat.put(0, 0, grayPixels); maskMat.put(0, 0, maskPixels); grayMat.copyTo(dstMat, maskMat); Imgproc.Canny(lineMat, lineMat, 20, 120); lineMat.copyTo(maskMat); Core.bitwise_not(lineMat, lineMat); lineMat.copyTo(dstMat, maskMat); // Imgproc.putText (dstMat, "W:" + dstMat.width () + " H:" + dstMat.height () + " SO:" + Screen.orientation, new Point (5, dstMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (0), 2, Imgproc.LINE_AA, false); // Imgproc.cvtColor(dstMat,rgbaMat,Imgproc.COLOR_GRAY2RGBA); // Utils.matToTexture2D (rgbaMat, texture); Utils.matToTexture2D(dstMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); copyMat = rgbaMat.clone(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (cascade != null) { cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size()); } OpenCVForUnity.Rect[] rects = faces.toArray(); if (rects.Length == 0) { noFaceFrameCount++; if (rectsLast != null && noFaceFrameCount <= maxNegativeFrames && rectsLast.Length > 0) { blurBackground(rectsLast, rgbaMat); } else { //clear last rect fields rowRangeTopLast = null; rowRangeButtomLast = null; colRangeleftLast = null; colRangeRightLast = null; } } if (rects != null && rects.Length > 0) { blurBackground(rects, rgbaMat); noFaceFrameCount = 0; rectsLast = null; rectsLast = new OpenCVForUnity.Rect[rects.Length]; for (int i = 0; i < rects.Length; i++) { rectsLast[i] = rects[i]; } } // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false); Utils.matToTexture2D(webCamTextureMat, texture, webCamTextureToMatHelper.GetBufferColors()); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; gameObject.transform.localScale = new Vector3(webCamTextureMat.cols(), webCamTextureMat.rows(), 1); Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); if (fpsMonitor != null) { fpsMonitor.Add("width", webCamTextureMat.width().ToString()); fpsMonitor.Add("height", webCamTextureMat.height().ToString()); fpsMonitor.Add("orientation", Screen.orientation.ToString()); } float width = webCamTextureMat.width(); float height = webCamTextureMat.height(); float widthScale = (float)Screen.width / width; float heightScale = (float)Screen.height / height; if (widthScale < heightScale) { Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = height / 2; } grayMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC1); }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); backgroundSubstractorMOG2.apply(rgbMat, fgmaskMat); Core.bitwise_not(fgmaskMat, fgmaskMat); rgbaMat.setTo(new Scalar(0, 0, 0, 0), fgmaskMat); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGB2HSV); Scalar lower_red = new Scalar(145, 42, 154); Scalar lower_blue = new Scalar(90, 50, 50); Scalar upper_red = new Scalar(255, 255, 255); Scalar upper_blue = new Scalar(130, 255, 255); Core.inRange(grayMat, lower_red, upper_red, redframe_threshold); Core.inRange(grayMat, lower_blue, upper_blue, blueframe_threshold); Core.bitwise_or(redframe_threshold, blueframe_threshold, frame_threshold); Size size = new Size(5, 5); Imgproc.erode(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size)); Imgproc.dilate(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size)); Imgproc.erode(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size)); Imgproc.dilate(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size)); Imgproc.erode(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size)); Imgproc.dilate(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size)); using (Mat circles = new Mat()) { Imgproc.HoughCircles(frame_threshold, circles, Imgproc.CV_HOUGH_GRADIENT, 1, frame_threshold.rows() / 2, 20, 15, 15, 100); Point pt = new Point(); for (int i = 0; i < circles.cols(); i++) { double[] data = circles.get(0, i); pt.x = data [0]; pt.y = data [1]; double rho = data [2]; // Imgproc.circle (rgbaMat, pt, 3, new Scalar (255, 0, 255), 5); Imgproc.circle(rgbaMat, pt, (int)rho, new Scalar(255, 0, 0, 255), 5); } } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { bool isTouched = false; #if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR) if (Input.touchCount == 1) { Touch t = Input.GetTouch(0); if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId)) { isTouched = true; } } #else if (Input.GetKeyUp(KeyCode.Space)) { isTouched = true; } #endif if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (isTouched) { rgbaMat.copyTo(bgMat); setBgTexture(bgMat); } //set fgMaskMat findFgMaskMat(rgbaMat, bgMat, thresh); //set bgMaskMat Core.bitwise_not(fgMaskMat, bgMaskMat); //copy greenMat using bgMaskMat greenMat.copyTo(rgbaMat, bgMaskMat); //Imgproc.putText (rgbaMat, "SPACE KEY or TOUCH SCREEN: Reset backgroud image.", new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { #if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR) //Touch int touchCount = Input.touchCount; if (touchCount == 1) { Touch t = Input.GetTouch(0); if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId)) { storedTouchPoint = new Point(t.position.x, t.position.y); //Debug.Log ("touch X " + t.position.x); //Debug.Log ("touch Y " + t.position.y); } } #else //Mouse if (Input.GetMouseButtonUp(0) && !EventSystem.current.IsPointerOverGameObject()) { storedTouchPoint = new Point(Input.mousePosition.x, Input.mousePosition.y); //Debug.Log ("mouse X " + Input.mousePosition.x); //Debug.Log ("mouse Y " + Input.mousePosition.y); } #endif if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (storedTouchPoint != null) { onTouch(rgbaMat, convertScreenPoint(storedTouchPoint, gameObject, Camera.main)); storedTouchPoint = null; } handPoseEstimationProcess(rgbaMat); Imgproc.putText(rgbaMat, "Please touch the area of the open hand.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); //Here starts the OpenCV script /* * // ENTER YOUR OPENCV CODE HERES */ // Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); // detect markers and estimate pose Aruco.detectMarkers(rgbMat, dictionary, corners, ids, detectorParams, rejected); // Aruco.detectMarkers (imgMat, dictionary, corners, ids); if (estimatePose && ids.total() > 0) { Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs); } // draw results if (ids.total() > 0) { Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(255, 0, 0)); if (estimatePose) { for (int i = 0; i < ids.total(); i++) { Aruco.drawAxis(rgbMat, camMatrix, distCoeffs, rvecs, tvecs, markerLength * 0.5f); //This example can display ARObject on only first detected marker. if (i == 0) { // position double[] tvec = tvecs.get(i, 0); Vector4 pos = new Vector4((float)tvec [0], (float)tvec [1], (float)tvec [2], 0); // from OpenCV // right-handed coordinates system (OpenCV) to left-handed one (Unity) ARGameObject.transform.localPosition = new Vector3(pos.x, -pos.y, pos.z); // rotation double[] rv = rvecs.get(i, 0); Mat rvec = new Mat(3, 1, CvType.CV_64FC1); rvec.put(0, 0, rv[0]); rvec.put(1, 0, rv[1]); rvec.put(2, 0, rv[2]); Calib3d.Rodrigues(rvec, rotMat); Vector3 forward = new Vector3((float)rotMat.get(0, 2) [0], (float)rotMat.get(1, 2) [0], (float)rotMat.get(2, 2) [0]); // from OpenCV Vector3 up = new Vector3((float)rotMat.get(0, 1) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(2, 1) [0]); // from OpenCV // right-handed coordinates system (OpenCV) to left-handed one (Unity) Quaternion rot = Quaternion.LookRotation(new Vector3(forward.x, -forward.y, forward.z), new Vector3(up.x, -up.y, up.z)); ARGameObject.transform.localRotation = rot; // Apply Z axis inverted matrix. invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1)); transformationM = ARGameObject.transform.localToWorldMatrix * invertZM; // Apply camera transform matrix. transformationM = ARCamera.transform.localToWorldMatrix * transformationM; ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref transformationM); } } } } if (showRejected && rejected.Count > 0) { Aruco.drawDetectedMarkers(rgbMat, rejected, new Mat(), new Scalar(0, 0, 255)); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (net == null) { Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } else { Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR); //! [Resizing without keeping aspect ratio] Imgproc.resize(bgrMat, resized, new Size(network_width, network_height)); //! [Resizing without keeping aspect ratio] //! [Prepare blob] inputBlob = Dnn.blobFromImage(resized, 1 / 255.0, new Size(), new Scalar(0), true, true); //Convert Mat to batch of images //! [Prepare blob] //! [Set input blob] net.setInput(inputBlob, "data"); //set the network input //! [Set input blob] // TickMeter tm = new TickMeter (); // tm.start (); //! [Make forward pass] Mat detectionMat = net.forward("detection_out"); //compute output //! [Make forward pass] // tm.stop (); // Debug.Log ("Inference time, ms: " + tm.getTimeMilli ()); // Debug.Log ("detectionMat.ToString(): " + detectionMat.ToString ()); float[] position = new float[5]; float[] confidences = new float[80]; float confidenceThreshold = 0.24f; for (int i = 0; i < detectionMat.rows(); i++) { detectionMat.get(i, 0, position); detectionMat.get(i, 5, confidences); int maxIdx = confidences.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I; float confidence = confidences [maxIdx]; if (confidence > confidenceThreshold) { float x = position [0]; float y = position [1]; float width = position [2]; float height = position [3]; int xLeftBottom = (int)((x - width / 2) * rgbaMat.cols()); int yLeftBottom = (int)((y - height / 2) * rgbaMat.rows()); int xRightTop = (int)((x + width / 2) * rgbaMat.cols()); int yRightTop = (int)((y + height / 2) * rgbaMat.rows()); // Debug.Log ("confidence: " + confidence); // // Debug.Log (" " + xLeftBottom // + " " + yLeftBottom // + " " + xRightTop // + " " + yRightTop); Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop), new Scalar(0, 255, 0, 255), 2); if (maxIdx < classNames.Count) { string label = classNames [maxIdx] + ": " + confidence; int[] baseLine = new int[1]; Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine); Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom), new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]), new Scalar(255, 255, 255, 255), Core.FILLED); Imgproc.putText(rgbaMat, label, new Point(xLeftBottom, yLeftBottom + labelSize.height), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255)); } } } detectionMat.Dispose(); } Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (net == null) { Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } else { Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR); net.setInput(Dnn.blobFromImage(bgrMat, 1.0, new Size(inWidth, inHeight), new Scalar(104, 177, 123), false, false), "data"); Mat detections = net.forward("detection_out"); detections = detections.reshape(1, (int)detections.total() / 7); float[] data = new float[7]; for (int i = 0; i < detections.rows(); i++) { detections.get(i, 0, data); float confidence = data [2]; if (confidence > confThreshold) { float xLeftBottom = data [3] * bgrMat.cols(); float yLeftBottom = data [4] * bgrMat.rows(); float xRightTop = data [5] * bgrMat.cols(); float yRightTop = data [6] * bgrMat.rows(); Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop), new Scalar(0, 255, 0, 255), 2); string label = "face: " + confidence; int[] baseLine = new int[1]; Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine); Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom), new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]), new Scalar(255, 255, 255, 255), Core.FILLED); Imgproc.putText(rgbaMat, label, new Point(xLeftBottom, yLeftBottom + labelSize.height), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255)); } } detections.Dispose(); } Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); // detect markers. Aruco.detectMarkers(rgbMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); // refine marker detection. if (refineMarkerDetection) { switch (markerType) { case (int)MarkerType.GridBoard: Aruco.refineDetectedMarkers(rgbMat, gridBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); break; case (int)MarkerType.ChArUcoBoard: Aruco.refineDetectedMarkers(rgbMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); break; } } // if at least one marker detected if (ids.total() > 0) { if (markerType != (int)MarkerType.ChArUcoDiamondMarker) { if (markerType == (int)MarkerType.ChArUcoBoard) { Aruco.interpolateCornersCharuco(corners, ids, rgbMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers); // draw markers. Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(0, 255, 0)); if (charucoIds.total() > 0) { Aruco.drawDetectedCornersCharuco(rgbMat, charucoCorners, charucoIds, new Scalar(0, 0, 255)); } } else { // draw markers. Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(0, 255, 0)); } // estimate pose. if (applyEstimationPose) { switch (markerType) { case (int)MarkerType.CanonicalMarker: EstimatePoseCanonicalMarker(rgbMat); break; case (int)MarkerType.GridBoard: EstimatePoseGridBoard(rgbMat); break; case (int)MarkerType.ChArUcoBoard: EstimatePoseChArUcoBoard(rgbMat); break; } } } else { // detect diamond markers. Aruco.detectCharucoDiamond(rgbMat, corners, ids, diamondSquareLength / diamondMarkerLength, diamondCorners, diamondIds, camMatrix, distCoeffs); // draw markers. Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(0, 255, 0)); // draw diamond markers. Aruco.drawDetectedDiamonds(rgbMat, diamondCorners, diamondIds, new Scalar(0, 0, 255)); // estimate pose. if (applyEstimationPose) { EstimatePoseChArUcoDiamondMarker(rgbMat); } } } if (showRejectedCorners && rejectedCorners.Count > 0) { Aruco.drawDetectedMarkers(rgbMat, rejectedCorners, new Mat(), new Scalar(255, 0, 0)); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { /// Mouse interaction, not developed yet #if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR) //Touch int touchCount = Input.touchCount; if (touchCount == 1) { Touch t = Input.GetTouch(0); if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId)) { storedTouchPoint = new Point(t.position.x, t.position.y); //Debug.Log ("touch X " + t.position.x); //Debug.Log ("touch Y " + t.position.y); } } #else //Mouse if (Input.GetMouseButtonUp(0) && !EventSystem.current.IsPointerOverGameObject()) { storedTouchPoint = new Point(Input.mousePosition.x, Input.mousePosition.y); //Debug.Log ("mouse X " + Input.mousePosition.x); //Debug.Log ("mouse Y " + Input.mousePosition.y); } #endif if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); //Here starts the OpenCV script //onTouch para el 1er clic if (selectedPointList.Count == 1) { if (storedTouchPoint != null) { ConvertScreenPointToTexturePoint(storedTouchPoint, storedTouchPoint, gameObject, rgbaMat.cols(), rgbaMat.rows()); OnTouch(rgbaMat, storedTouchPoint); //rgab o gray¡ Debug.Log("primer clic por " + storedTouchPoint.x + ";" + storedTouchPoint.y); storedTouchPoint = null; } } //error PlayerLoop called recursively! on iOS.reccomend WebCamTexture. if (selectedPointList.Count != 1) { //onTouch para el 2do clic if (storedTouchPoint != null) { ConvertScreenPointToTexturePoint(storedTouchPoint, storedTouchPoint, gameObject, rgbaMat.cols(), rgbaMat.rows()); OnTouch(rgbaMat, storedTouchPoint); storedTouchPoint = null; } //si ya es 2, creeme otra trackinArea if (selectedPointList.Count < 2) { foreach (var point in selectedPointList) { Imgproc.circle(rgbaMat, point, 6, new Scalar(0, 0, 255), 2); } } else { //key line! DM using (MatOfPoint selectedPointMat = new MatOfPoint(selectedPointList.ToArray())) { //TODO 13n cambiar essa region por varias regiones, que le pasara OpenCVForUnity.Rect region = Imgproc.boundingRect(selectedPointMat); //si se necesita al fin y al cabo SelectTracker(tracker_type); //16n monotracker = TrackerKCF.create(); //8n trackerInitialized = monotracker.init(grayMat, new Rect2d(region.x, region.y, region.width, region.height)); } selectedPointList.Clear(); } // aca ya no lo inicializa, sino que lo actualiza if (trackerInitialized) { bool updated = monotracker.update(grayMat, bbox);//8n //pero nunca le pasa la bbox ni las coordenadas, unicamente al momento de tracker.init if (bbox.width != 0 && bbox.height != 0) { Debug.Log("tracking por aca:" + bbox.x + ";" + bbox.y); Imgproc.rectangle(rgbaMat, bbox.tl(), bbox.br(), new Scalar(255, 255, 255, 255), 2, 1, 0); previousBox = new Rect2d(bbox.x, bbox.y, bbox.width, bbox.height); } else { Debug.Log("Se perdio en:" + previousBox.x + ";" + previousBox.y); Imgproc.rectangle(rgbaMat, previousBox.tl(), previousBox.br(), new Scalar(255, 0, 0, 255), 2, 1, 0); //8n } } // bool updated = trackers.update (rgbaMat, objects); // Debug.Log ("updated " + updated); // if (!updated && bbox.width > 0) // OnResetTrackerButtonClick (); // } if (selectedPointList.Count != 1) { //Imgproc.putText (rgbaMat, "Please touch the screen, and select tracking regions.", new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); if (fpsMonitor != null) { fpsMonitor.consoleText = "Touch the screen to select a new tracking region."; } } else { //Imgproc.putText (rgbaMat, "Please select the end point of the new tracking region.", new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); if (fpsMonitor != null) { fpsMonitor.consoleText = "Please select the end point of the new tracking region."; } } Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); //Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } }
// Update is called once per frame void Update() { #if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR) //Touch int touchCount = Input.touchCount; if (touchCount == 1) { Touch t = Input.GetTouch(0); if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId)) { storedTouchPoint = new Point(t.position.x, t.position.y); //Debug.Log ("touch X " + t.position.x); //Debug.Log ("touch Y " + t.position.y); } } #else //Mouse if (Input.GetMouseButtonUp(0) && !EventSystem.current.IsPointerOverGameObject()) { storedTouchPoint = new Point(Input.mousePosition.x, Input.mousePosition.y); //Debug.Log ("mouse X " + Input.mousePosition.x); //Debug.Log ("mouse Y " + Input.mousePosition.y); } #endif if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, hsvMat, Imgproc.COLOR_RGBA2RGB); Imgproc.cvtColor(hsvMat, hsvMat, Imgproc.COLOR_RGB2HSV); if (storedTouchPoint != null) { ConvertScreenPointToTexturePoint(storedTouchPoint, storedTouchPoint, gameObject, rgbaMat.cols(), rgbaMat.rows()); OnTouch(rgbaMat, storedTouchPoint); storedTouchPoint = null; } Point[] points = roiPointList.ToArray(); if (shouldStartCamShift) { shouldStartCamShift = false; using (MatOfPoint roiPointMat = new MatOfPoint(points)) { roiRect = Imgproc.boundingRect(roiPointMat); } if (roiHistMat != null) { roiHistMat.Dispose(); roiHistMat = null; } roiHistMat = new Mat(); using (Mat roiHSVMat = new Mat(hsvMat, roiRect)) using (Mat maskMat = new Mat()) { Imgproc.calcHist(new List <Mat> (new Mat[] { roiHSVMat }), new MatOfInt(0), maskMat, roiHistMat, new MatOfInt(16), new MatOfFloat(0, 180)); Core.normalize(roiHistMat, roiHistMat, 0, 255, Core.NORM_MINMAX); //Debug.Log ("roiHist " + roiHistMat.ToString ()); } } else if (points.Length == 4) { using (Mat backProj = new Mat()) { Imgproc.calcBackProject(new List <Mat> (new Mat[] { hsvMat }), new MatOfInt(0), roiHistMat, backProj, new MatOfFloat(0, 180), 1.0); RotatedRect r = Video.CamShift(backProj, roiRect, termination); r.points(points); } } if (points.Length < 4) { for (int i = 0; i < points.Length; i++) { Imgproc.circle(rgbaMat, points [i], 6, new Scalar(0, 0, 255, 255), 2); } } else { for (int i = 0; i < 4; i++) { Imgproc.line(rgbaMat, points [i], points [(i + 1) % 4], new Scalar(255, 0, 0, 255), 2); } Imgproc.rectangle(rgbaMat, roiRect.tl(), roiRect.br(), new Scalar(0, 255, 0, 255), 2); } Imgproc.putText(rgbaMat, "Please touch the 4 points surrounding the tracking object.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (mMOP2fptsPrev.rows() == 0) { // first time through the loop so we need prev and this mats // plus prev points // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // copy that to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get prev corners Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsPrev.fromArray(MOPcorners.toArray()); // get safe copy of this corners mMOP2fptsPrev.copyTo(mMOP2fptsSafe); } else { // we've been through before so // this mat is valid. Copy it to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // get the corners for this mat Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsThis.fromArray(MOPcorners.toArray()); // retrieve the corners from the prev mat // (saves calculating them again) mMOP2fptsSafe.copyTo(mMOP2fptsPrev); // and save this corners for next time through mMOP2fptsThis.copyTo(mMOP2fptsSafe); } /* * Parameters: * prevImg first 8-bit input image * nextImg second input image * prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers. * nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input. * status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0. * err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases). */ Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr); if (mMOBStatus.rows() > 0) { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus [x] == 1) { Point pt = cornersThis [x]; Point pt2 = cornersPrev [x]; Imgproc.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1); Imgproc.line(rgbaMat, pt, pt2, colorRed, iLineThickness); } } } // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); // detect markers. Aruco.detectMarkers(rgbMat, dictionary, corners, ids, detectorParams, rejected); // estimate pose. if (applyEstimationPose && ids.total() > 0) { Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs); } if (ids.total() > 0) { Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(255, 0, 0)); if (applyEstimationPose) { for (int i = 0; i < ids.total(); i++) { Aruco.drawAxis(rgbMat, camMatrix, distCoeffs, rvecs, tvecs, markerLength * 0.5f); // This example can display ARObject on only first detected marker. if (i == 0) { // position double[] tvec = tvecs.get(i, 0); // rotation double[] rv = rvecs.get(i, 0); Mat rvec = new Mat(3, 1, CvType.CV_64FC1); rvec.put(0, 0, rv [0]); rvec.put(1, 0, rv [1]); rvec.put(2, 0, rv [2]); Calib3d.Rodrigues(rvec, rotMat); transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0) [0], (float)rotMat.get(0, 1) [0], (float)rotMat.get(0, 2) [0], (float)tvec [0])); transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(1, 2) [0], (float)tvec [1])); transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0) [0], (float)rotMat.get(2, 1) [0], (float)rotMat.get(2, 2) [0], (float)tvec [2])); transformationM.SetRow(3, new Vector4(0, 0, 0, 1)); // right-handed coordinates system (OpenCV) to left-handed one (Unity) ARM = invertYM * transformationM; // Apply Z axis inverted matrix. ARM = ARM * invertZM; if (shouldMoveARCamera) { ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse; ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM); } else { ARM = ARCamera.transform.localToWorldMatrix * ARM; ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM); } } } } } if (showRejected && rejected.Count > 0) { Aruco.drawDetectedMarkers(rgbMat, rejected, new Mat(), new Scalar(0, 0, 255)); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame() && !imageOptimizationHelper.IsCurrentFrameSkipped()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); //get downScaleMat; Mat downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat((rgbaMat)); //grayscale Imgproc.cvtColor(downScaleRgbaMat, gray1Mat, Imgproc.COLOR_RGBA2GRAY); //blur Imgproc.blur(gray1Mat, gray2Mat, new Size(5, 5)); //edge filter Imgproc.filter2D(gray2Mat, gray1Mat, gray1Mat.depth(), kernel); //blur Imgproc.blur(gray1Mat, gray2Mat, new Size(3, 3)); //detect edge Imgproc.threshold(gray2Mat, gray2Mat, EDGE_DETECT_VALUE, 255, Imgproc.THRESH_BINARY); //copy Mat to byteArray Utils.copyFromMat <byte> (gray2Mat, byteArray); //set edge pointList List <Point> pointList = new List <Point> (); int w = gray1Mat.width(); int h = gray1Mat.height(); for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) { if (byteArray [x + w * y] == 255) { pointList.Add(new Point(x, y)); } } } int limit = Mathf.RoundToInt((float)(pointList.Count * POINT_RATE)); if (limit > POINT_MAX_NUM) { limit = POINT_MAX_NUM; } while (pointList.Count > limit) { pointList.RemoveAt(Random.Range(0, pointList.Count)); } // Debug.Log ("pointList.Count " + pointList.Count); //init subdiv subdiv.initDelaunay(new OpenCVForUnity.Rect(0, 0, downScaleRgbaMat.width(), downScaleRgbaMat.height())); for (int i = 0; i < pointList.Count; i++) { subdiv.insert(pointList [i]); } subdiv.insert(new Point(0, 0)); subdiv.insert(new Point(gray1Mat.width() / 2 - 1, 0)); subdiv.insert(new Point(gray1Mat.width() - 1, 0)); subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() / 2 - 1)); subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() - 1)); subdiv.insert(new Point(gray1Mat.width() / 2 - 1, gray1Mat.height() - 1)); subdiv.insert(new Point(0, gray1Mat.height() - 1)); subdiv.insert(new Point(0, gray1Mat.height() / 2 - 1)); using (MatOfFloat6 triangleList = new MatOfFloat6()) { subdiv.getTriangleList(triangleList); float[] pointArray = triangleList.toArray(); float downScaleRatio = imageOptimizationHelper.downscaleRatio; if (downScaleRatio < 1) { downScaleRatio = 1; } byte[] color = new byte[4]; for (int i = 0; i < pointArray.Length / 6; i++) { Point p0 = new Point(pointArray [i * 6 + 0] * downScaleRatio, pointArray [i * 6 + 1] * downScaleRatio); Point p1 = new Point(pointArray [i * 6 + 2] * downScaleRatio, pointArray [i * 6 + 3] * downScaleRatio); Point p2 = new Point(pointArray [i * 6 + 4] * downScaleRatio, pointArray [i * 6 + 5] * downScaleRatio); if (p0.x < 0 || p0.x > rgbaMat.width()) { continue; } if (p0.y < 0 || p0.y > rgbaMat.height()) { continue; } if (p1.x < 0 || p1.x > rgbaMat.width()) { continue; } if (p1.y < 0 || p1.y > rgbaMat.height()) { continue; } if (p2.x < 0 || p2.x > rgbaMat.width()) { continue; } if (p2.y < 0 || p2.y > rgbaMat.height()) { continue; } //get center of gravity int cx = (int)((p0.x + p1.x + p2.x) * 0.33333); int cy = (int)((p0.y + p1.y + p2.y) * 0.33333); // Debug.Log ("cx " + cx + " cy " + cy ); //get center of gravity color rgbaMat.get(cy, cx, color); // Debug.Log ("r " + color[0] + " g " + color[1] + " b " + color[2] + " a " + color[3]); //fill Polygon Imgproc.fillConvexPoly(rgbaMat, new MatOfPoint(p0, p1, p2), new Scalar(color [0], color [1], color [2], color [3]), Imgproc.LINE_AA, 0); // Imgproc.line (rgbaMat, p0, p1, new Scalar (64, 255, 128, 255)); // Imgproc.line (rgbaMat, p1, p2, new Scalar (64, 255, 128, 255)); // Imgproc.line (rgbaMat, p2, p0, new Scalar (64, 255, 128, 255)); } } //Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " DOWNSCALE W:" + downScaleRgbaMat.width () + " H:" + downScaleRgbaMat.height (), new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); /*Text Detection*/ Imgproc.cvtColor(rgbMat, binaryMat, Imgproc.COLOR_RGB2GRAY); Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU); Core.absdiff(binaryMat, new Scalar(255), maskMat); List <MatOfPoint> regions = new List <MatOfPoint> (); OpenCVForUnity.Text.detectRegions(binaryMat, er_filter1, er_filter2, regions); // Debug.Log ("regions.Count " + regions.Count); MatOfRect groups_rects = new MatOfRect(); List <OpenCVForUnity.Rect> rects = new List <OpenCVForUnity.Rect> (); if (regions.Count > 0) { OpenCVForUnity.Text.erGrouping(rgbMat, binaryMat, regions, groups_rects); } for (int i = 0; i < regions.Count; i++) { regions [i].Dispose(); } regions.Clear(); rects.AddRange(groups_rects.toList()); groups_rects.Dispose(); /*Text Recognition (OCR)*/ List <Mat> detections = new List <Mat> (); for (int i = 0; i < (int)rects.Count; i++) { Mat group_img = new Mat(); maskMat.submat(rects [i]).copyTo(group_img); Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0)); detections.Add(group_img); } // Debug.Log ("detections.Count " + detections.Count); // Debug.Log ("rects.Count " + rects.Count); //#Visualization for (int i = 0; i < rects.Count; i++) { Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2); Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 255, 255, 255), 1); Imgproc.putText(rgbaMat, "" + i, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false); } for (int i = 0; i < detections.Count; i++) { string output = decoder.run(detections [i], 0); Debug.Log("output " + output); if (string.IsNullOrEmpty(output)) { Debug.LogError("IsNullOrEmpty output " + output); } else { Imgproc.putText(rgbaMat, " " + output, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false); } } for (int i = 0; i < detections.Count; i++) { detections [i].Dispose(); } detections.Clear(); Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (cascade != null) { cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size()); } OpenCVForUnity.Rect[] rects = faces.toArray(); for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2); //make the coordinates for the rectangle accessible elPosX = rects [i].x + rects [i].width; elPosY = rects [i].y + rects [i].height; } if (rects.Length > 0) { //yay we see a face Debug.Log("Update: Face detected!!!"); //start the party (once) //partyObject.SetActive (true); //set the hat graphic true //TODO: need to figure out a way for this to not flicker so much //m_partyHat.SetActive (true); //update the position of the hat // TODO: need to provide an offset based on the size of the rectangle (i think!) m_partyHat.transform.position = new Vector3(elPosX, elPosY, 0); } else { //set the hat graphic false Debug.Log("Update: No face yet!"); //m_partyHat.SetActive (false); } //float elPosX = rects [0].x; //float elPosX = 400f; //float elPosY = rects [0].y; //float elPosY = 100f; //Debug.Log (elPosX + ", " + elPosY); //m_partyHat.transform.position = new Vector3 (elPosX, elPosY, 0); // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }