protected virtual void DisposeOpticalFlow() { if (prevTrackPts != null) { prevTrackPts.Clear(); } if (nextTrackPts != null) { nextTrackPts.Clear(); } if (prevgray != null) { prevgray.Dispose(); } if (gray != null) { gray.Dispose(); } if (mOP2fPrevTrackPts != null) { mOP2fPrevTrackPts.Dispose(); } if (mOP2fNextTrackPts != null) { mOP2fNextTrackPts.Dispose(); } if (status != null) { status.Dispose(); } if (err != null) { err.Dispose(); } }
/// <summary> /// Raises the web cam texture to mat helper disposed event. /// </summary> public void OnWebCamTextureToMatHelperDisposed() { Debug.Log("OnWebCamTextureToMatHelperDisposed"); matOpFlowThis.Dispose(); matOpFlowPrev.Dispose(); MOPcorners.Dispose(); mMOP2fptsThis.Dispose(); mMOP2fptsPrev.Dispose(); mMOP2fptsSafe.Dispose(); mMOBStatus.Dispose(); mMOFerr.Dispose(); }
/// <summary> /// Raises the web cam texture to mat helper disposed event. /// </summary> public void OnWebCamTextureToMatHelperDisposed() { Debug.Log("OnWebCamTextureToMatHelperDisposed"); if (texture != null) { Texture2D.Destroy(texture); texture = null; } if (matOpFlowThis != null) { matOpFlowThis.Dispose(); } if (matOpFlowPrev != null) { matOpFlowPrev.Dispose(); } if (MOPcorners != null) { MOPcorners.Dispose(); } if (mMOP2fptsThis != null) { mMOP2fptsThis.Dispose(); } if (mMOP2fptsPrev != null) { mMOP2fptsPrev.Dispose(); } if (mMOP2fptsSafe != null) { mMOP2fptsSafe.Dispose(); } if (mMOBStatus != null) { mMOBStatus.Dispose(); } if (mMOFerr != null) { mMOFerr.Dispose(); } }
void OnDisable() { Debug.Log("OnDisable"); if (matOpFlowThis != null) { matOpFlowThis.Dispose(); } if (matOpFlowPrev != null) { matOpFlowPrev.Dispose(); } if (MOPcorners != null) { MOPcorners.Dispose(); } if (mMOP2fptsThis != null) { mMOP2fptsThis.Dispose(); } if (mMOP2fptsPrev != null) { mMOP2fptsPrev.Dispose(); } if (mMOP2fptsSafe != null) { mMOP2fptsSafe.Dispose(); } if (mMOBStatus != null) { mMOBStatus.Dispose(); } if (mMOFerr != null) { mMOFerr.Dispose(); } //webCamTextureToMatHelper.Dispose(); }
/// <summary> /// Postprocess the specified frame, outs and net. /// </summary> /// <param name="frame">Frame.</param> /// <param name="outs">Outs.</param> /// <param name="net">Net.</param> private void postprocess(Mat frame, List <Mat> outs, Net net) { string outLayerType = outBlobTypes[0]; List <int> classIdsList = new List <int>(); List <float> confidencesList = new List <float>(); List <OpenCVForUnity.CoreModule.Rect> boxesList = new List <OpenCVForUnity.CoreModule.Rect>(); if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1) { // Faster-RCNN or R-FCN // Network produces output blob with a shape 1x1xNx7 where N is a number of // detections and an every detection is a vector of values // [batchId, classId, confidence, left, top, right, bottom] if (outs.Count == 1) { outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7); //Debug.Log ("outs[i].ToString() " + outs [0].ToString ()); float[] data = new float[7]; for (int i = 0; i < outs[0].rows(); i++) { outs[0].get(i, 0, data); float confidence = data[2]; if (confidence > confThreshold) { int class_id = (int)(data[1]); int left = (int)(data[3] * frame.cols()); int top = (int)(data[4] * frame.rows()); int right = (int)(data[5] * frame.cols()); int bottom = (int)(data[6] * frame.rows()); int width = right - left + 1; int height = bottom - top + 1; classIdsList.Add((int)(class_id) - 0); confidencesList.Add((float)confidence); boxesList.Add(new OpenCVForUnity.CoreModule.Rect(left, top, width, height)); } } } } else if (outLayerType == "DetectionOutput") { // Network produces output blob with a shape 1x1xNx7 where N is a number of // detections and an every detection is a vector of values // [batchId, classId, confidence, left, top, right, bottom] if (outs.Count == 1) { outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7); //Debug.Log ("outs[i].ToString() " + outs [0].ToString ()); float[] data = new float[7]; for (int i = 0; i < outs[0].rows(); i++) { outs[0].get(i, 0, data); float confidence = data[2]; if (confidence > confThreshold) { int class_id = (int)(data[1]); int left = (int)(data[3] * frame.cols()); int top = (int)(data[4] * frame.rows()); int right = (int)(data[5] * frame.cols()); int bottom = (int)(data[6] * frame.rows()); int width = right - left + 1; int height = bottom - top + 1; classIdsList.Add((int)(class_id) - 0); confidencesList.Add((float)confidence); boxesList.Add(new OpenCVForUnity.CoreModule.Rect(left, top, width, height)); } } } } else if (outLayerType == "Region") { for (int i = 0; i < outs.Count; ++i) { // Network produces output blob with a shape NxC where N is a number of // detected objects and C is a number of classes + 4 where the first 4 // numbers are [center_x, center_y, width, height] //Debug.Log ("outs[i].ToString() "+outs[i].ToString()); float[] positionData = new float[5]; float[] confidenceData = new float[outs[i].cols() - 5]; for (int p = 0; p < outs[i].rows(); p++) { outs[i].get(p, 0, positionData); outs[i].get(p, 5, confidenceData); int maxIdx = confidenceData.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I; float confidence = confidenceData[maxIdx]; if (confidence > confThreshold) { int centerX = (int)(positionData[0] * frame.cols()); int centerY = (int)(positionData[1] * frame.rows()); int width = (int)(positionData[2] * frame.cols()); int height = (int)(positionData[3] * frame.rows()); int left = centerX - width / 2; int top = centerY - height / 2; classIdsList.Add(maxIdx); confidencesList.Add((float)confidence); boxesList.Add(new OpenCVForUnity.CoreModule.Rect(left, top, width, height)); } } } } else { Debug.Log("Unknown output layer type: " + outLayerType); } MatOfRect boxes = new MatOfRect(); boxes.fromList(boxesList); MatOfFloat confidences = new MatOfFloat(); confidences.fromList(confidencesList); MatOfInt indices = new MatOfInt(); Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); //Debug.Log ("indices.dump () "+indices.dump ()); //Debug.Log ("indices.ToString () "+indices.ToString()); for (int i = 0; i < indices.total(); ++i) { int idx = (int)indices.get(i, 0)[0]; OpenCVForUnity.CoreModule.Rect box = boxesList[idx]; drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y, box.x + box.width, box.y + box.height, frame); } indices.Dispose(); boxes.Dispose(); confidences.Dispose(); }
private IEnumerator init() { if (webCamTexture != null) { webCamTexture.Stop(); initDone = false; rgbaMat.Dispose(); matOpFlowThis.Dispose(); matOpFlowPrev.Dispose(); MOPcorners.Dispose(); mMOP2fptsThis.Dispose(); mMOP2fptsPrev.Dispose(); mMOP2fptsSafe.Dispose(); mMOBStatus.Dispose(); mMOFerr.Dispose(); } // Checks how many and which cameras are available on the device for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) { if (WebCamTexture.devices [cameraIndex].isFrontFacing == isFrontFacing) { Debug.Log(cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing); webCamDevice = WebCamTexture.devices [cameraIndex]; webCamTexture = new WebCamTexture(webCamDevice.name, width, height); break; } } if (webCamTexture == null) { webCamDevice = WebCamTexture.devices [0]; webCamTexture = new WebCamTexture(webCamDevice.name, width, height); } Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS); // Starts the camera webCamTexture.Play(); while (true) { //If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/) #if UNITY_IPHONE && !UNITY_EDITOR if (webCamTexture.width > 16 && webCamTexture.height > 16) { #else if (webCamTexture.didUpdateThisFrame) { #endif Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS); Debug.Log("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing); colors = new Color32[webCamTexture.width * webCamTexture.height]; rgbaMat = new Mat(webCamTexture.height, webCamTexture.width, CvType.CV_8UC4); matOpFlowThis = new Mat(); matOpFlowPrev = new Mat(); MOPcorners = new MatOfPoint(); mMOP2fptsThis = new MatOfPoint2f(); mMOP2fptsPrev = new MatOfPoint2f(); mMOP2fptsSafe = new MatOfPoint2f(); mMOBStatus = new MatOfByte(); mMOFerr = new MatOfFloat(); texture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false); gameObject.transform.eulerAngles = new Vector3(0, 0, 0); #if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR gameObject.transform.eulerAngles = new Vector3(0, 0, -90); #endif // gameObject.transform.rotation = gameObject.transform.rotation * Quaternion.AngleAxis (webCamTexture.videoRotationAngle, Vector3.back); gameObject.transform.localScale = new Vector3(webCamTexture.width, webCamTexture.height, 1); // bool videoVerticallyMirrored = webCamTexture.videoVerticallyMirrored; // float scaleX = 1; // float scaleY = videoVerticallyMirrored ? -1.0f : 1.0f; // if (webCamTexture.videoRotationAngle == 270) // scaleY = -1.0f; // gameObject.transform.localScale = new Vector3 (scaleX * gameObject.transform.localScale.x, scaleY * gameObject.transform.localScale.y, 1); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; #if (UNITY_ANDROID || UNITY_IPHONE) && !UNITY_EDITOR Camera.main.orthographicSize = webCamTexture.width / 2; #else Camera.main.orthographicSize = webCamTexture.height / 2; #endif initDone = true; break; } else { yield return(0); } } } // Update is called once per frame void Update() { if (!initDone) { return; } #if UNITY_IPHONE && !UNITY_EDITOR if (webCamTexture.width > 16 && webCamTexture.height > 16) { #else if (webCamTexture.didUpdateThisFrame) { #endif Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors); if (webCamTexture.videoVerticallyMirrored) { if (webCamDevice.isFrontFacing) { if (webCamTexture.videoRotationAngle == 0) { Core.flip(rgbaMat, rgbaMat, 1); } else if (webCamTexture.videoRotationAngle == 90) { Core.flip(rgbaMat, rgbaMat, 0); } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, 1); } } else { if (webCamTexture.videoRotationAngle == 90) { } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, -1); } } } else { if (webCamDevice.isFrontFacing) { if (webCamTexture.videoRotationAngle == 0) { Core.flip(rgbaMat, rgbaMat, 1); } else if (webCamTexture.videoRotationAngle == 90) { Core.flip(rgbaMat, rgbaMat, 0); } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, 1); } } else { if (webCamTexture.videoRotationAngle == 90) { } else if (webCamTexture.videoRotationAngle == 270) { Core.flip(rgbaMat, rgbaMat, -1); } } } if (mMOP2fptsPrev.rows() == 0) { // first time through the loop so we need prev and this mats // plus prev points // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // copy that to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get prev corners Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsPrev.fromArray(MOPcorners.toArray()); // get safe copy of this corners mMOP2fptsPrev.copyTo(mMOP2fptsSafe); } else { // we've been through before so // this mat is valid. Copy it to prev mat matOpFlowThis.copyTo(matOpFlowPrev); // get this mat Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY); // get the corners for this mat Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20); mMOP2fptsThis.fromArray(MOPcorners.toArray()); // retrieve the corners from the prev mat // (saves calculating them again) mMOP2fptsSafe.copyTo(mMOP2fptsPrev); // and save this corners for next time through mMOP2fptsThis.copyTo(mMOP2fptsSafe); } /* * Parameters: * prevImg first 8-bit input image * nextImg second input image * prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers. * nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input. * status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0. * err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases). */ Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr); if (!mMOBStatus.empty()) { List <Point> cornersPrev = mMOP2fptsPrev.toList(); List <Point> cornersThis = mMOP2fptsThis.toList(); List <byte> byteStatus = mMOBStatus.toList(); int x = 0; int y = byteStatus.Count - 1; for (x = 0; x < y; x++) { if (byteStatus [x] == 1) { Point pt = cornersThis [x]; Point pt2 = cornersPrev [x]; Core.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1); Core.line(rgbaMat, pt, pt2, colorRed, iLineThickness); } } } Utils.matToTexture2D(rgbaMat, texture, colors); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; } } void OnDisable() { webCamTexture.Stop(); } void OnGUI() { float screenScale = Screen.width / 240.0f; Matrix4x4 scaledMatrix = Matrix4x4.Scale(new Vector3(screenScale, screenScale, screenScale)); GUI.matrix = scaledMatrix; GUILayout.BeginVertical(); if (GUILayout.Button("back")) { Application.LoadLevel("OpenCVForUnitySample"); } if (GUILayout.Button("change camera")) { isFrontFacing = !isFrontFacing; StartCoroutine(init()); } GUILayout.EndVertical(); } } }
private IEnumerator init() { if (webCamTexture != null) { webCamTexture.Stop(); initDone = false; rgbaMat.Dispose(); matOpFlowThis.Dispose(); matOpFlowPrev.Dispose(); MOPcorners.Dispose(); mMOP2fptsThis.Dispose(); mMOP2fptsPrev.Dispose(); mMOP2fptsSafe.Dispose(); mMOBStatus.Dispose(); mMOFerr.Dispose(); } // Checks how many and which cameras are available on the device for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) { if (WebCamTexture.devices [cameraIndex].isFrontFacing == shouldUseFrontFacing) { Debug.Log(cameraIndex + " name " + WebCamTexture.devices [cameraIndex].name + " isFrontFacing " + WebCamTexture.devices [cameraIndex].isFrontFacing); webCamDevice = WebCamTexture.devices [cameraIndex]; webCamTexture = new WebCamTexture(webCamDevice.name, width, height); break; } } if (webCamTexture == null) { webCamDevice = WebCamTexture.devices [0]; webCamTexture = new WebCamTexture(webCamDevice.name, width, height); } Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS); // Starts the camera webCamTexture.Play(); while (true) { //If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/) #if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1) if (webCamTexture.width > 16 && webCamTexture.height > 16) { #else if (webCamTexture.didUpdateThisFrame) { #if UNITY_IOS && !UNITY_EDITOR && UNITY_5_2 while (webCamTexture.width <= 16) { webCamTexture.GetPixels32(); yield return(new WaitForEndOfFrame()); } #endif #endif Debug.Log("width " + webCamTexture.width + " height " + webCamTexture.height + " fps " + webCamTexture.requestedFPS); Debug.Log("videoRotationAngle " + webCamTexture.videoRotationAngle + " videoVerticallyMirrored " + webCamTexture.videoVerticallyMirrored + " isFrongFacing " + webCamDevice.isFrontFacing); colors = new Color32[webCamTexture.width * webCamTexture.height]; rgbaMat = new Mat(webCamTexture.height, webCamTexture.width, CvType.CV_8UC4); matOpFlowThis = new Mat(); matOpFlowPrev = new Mat(); MOPcorners = new MatOfPoint(); mMOP2fptsThis = new MatOfPoint2f(); mMOP2fptsPrev = new MatOfPoint2f(); mMOP2fptsSafe = new MatOfPoint2f(); mMOBStatus = new MatOfByte(); mMOFerr = new MatOfFloat(); texture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; updateLayout(); screenOrientation = Screen.orientation; initDone = true; break; } else { yield return(0); } } }
protected override void postprocess(Mat frame, List <Mat> outs, Net net, int backend = Dnn.DNN_BACKEND_OPENCV) { List <int> classIdsList = new List <int>(); List <float> confidencesList = new List <float>(); List <Rect2d> boxesList = new List <Rect2d>(); List <Point[]> pointsList = new List <Point[]>(); if (outs.Count == 2) { // reshape mat : outs[0]:[1, x, 4] to [x, 4], outs[1]:[1, x, 2] to [x, 2] Mat boxes_m = outs[0].reshape(1, new int[] { outs[0].size(1), outs[0].size(2) }); Mat scores_m = outs[1].reshape(1, new int[] { outs[1].size(1), outs[1].size(2) }); //Debug.Log("boxes_m: " + boxes_m); //Debug.Log("scores_m: " + scores_m); //Debug.Log("priors: " + priors); convertLocationsToBoxes(boxes_m, priors, 0.1f, 0.2f); centerFormToCornerForm(boxes_m); Mat boxes_0_4 = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(0, 4)); float[] boxes_arr = new float[boxes_0_4.rows() * boxes_0_4.cols()]; MatUtils.copyFromMat(boxes_0_4, boxes_arr); Mat scores_1_2 = new Mat(scores_m, new Range(0, scores_m.rows()), new Range(1, 2)); float[] confidences_arr = new float[scores_1_2.rows()]; MatUtils.copyFromMat(scores_1_2, confidences_arr); for (int i = 0; i < boxes_m.rows(); i++) { float confidence = confidences_arr[i]; if (confidence > confThreshold) { int boxes_index = i * 4; float left = boxes_arr[boxes_index] * frame.cols(); float top = boxes_arr[boxes_index + 1] * frame.rows(); float right = boxes_arr[boxes_index + 2] * frame.cols(); float bottom = boxes_arr[boxes_index + 3] * frame.rows(); float width = right - left + 1f; float height = bottom - top + 1f; classIdsList.Add(0); confidencesList.Add(confidence); boxesList.Add(new Rect2d(left, top, width, height)); } } if (boxes_m.cols() > 4 && boxes_m.cols() % 2 == 0) { Mat points = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(4, boxes_m.cols())); float[] points_arr = new float[points.rows() * points.cols()]; MatUtils.copyFromMat(points, points_arr); for (int i = 0; i < boxes_m.rows(); i++) { float confidence = confidences_arr[i]; if (confidence > confThreshold) { int points_index = i * points.cols(); Point[] p_arr = new Point[points.cols() / 2]; for (int index = 0; index < points.cols() / 2; index++) { float x = points_arr[points_index + index * 2] * frame.cols(); float y = points_arr[points_index + index * 2 + 1] * frame.rows(); p_arr[index] = new Point(x, y); } pointsList.Add(p_arr); } } } } MatOfRect2d boxes = new MatOfRect2d(); boxes.fromList(boxesList); MatOfFloat confidences = new MatOfFloat(); confidences.fromList(confidencesList); MatOfInt indices = new MatOfInt(); Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); //Debug.Log("indices.dump () " + indices.dump()); //Debug.Log ("indices.ToString () "+indices.ToString()); for (int i = 0; i < indices.total(); ++i) { int idx = (int)indices.get(i, 0)[0]; Rect2d box = boxesList[idx]; drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y, box.x + box.width, box.y + box.height, frame); if (pointsList.Count > 0) { drawPredPoints(pointsList[idx], frame); } } indices.Dispose(); boxes.Dispose(); confidences.Dispose(); }
/// <summary> /// Postprocess the specified frame, outs and net. /// </summary> /// <param name="frame">Frame.</param> /// <param name="outs">Outs.</param> /// <param name="net">Net.</param> private void postprocess(Mat frame, List <Mat> outs, Net net) { string outLayerType = outBlobTypes[0]; List <int> classIdsList = new List <int>(); List <float> confidencesList = new List <float>(); List <OpenCVForUnity.CoreModule.Rect> boxesList = new List <OpenCVForUnity.CoreModule.Rect>(); if (outLayerType == "Region") { for (int i = 0; i < outs.Count; ++i) { // Network produces output blob with a shape NxC where N is a number of // detected objects and C is a number of classes + 4 where the first 4 // numbers are [center_x, center_y, width, height] //Debug.Log("outs[i].ToString() " + outs[i].ToString()); float[] positionData = new float[5]; float[] confidenceData = new float[outs[i].cols() - 5]; for (int p = 0; p < outs[i].rows(); p++) { outs[i].get(p, 0, positionData); outs[i].get(p, 5, confidenceData); int maxIdx = confidenceData.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I; float confidence = confidenceData[maxIdx]; if (confidence > confThreshold) { int centerX = (int)(positionData[0] * frame.cols()); int centerY = (int)(positionData[1] * frame.rows()); int width = (int)(positionData[2] * frame.cols()); int height = (int)(positionData[3] * frame.rows()); int left = centerX - width / 2; int top = centerY - height / 2; classIdsList.Add(maxIdx); confidencesList.Add((float)confidence); boxesList.Add(new OpenCVForUnity.CoreModule.Rect(left, top, width, height)); } } } } else { Debug.Log("Unknown output layer type: " + outLayerType); } MatOfRect boxes = new MatOfRect(); boxes.fromList(boxesList); MatOfFloat confidences = new MatOfFloat(); confidences.fromList(confidencesList); MatOfInt indices = new MatOfInt(); Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); //Check the Language selected switch (menuVariables.GetLanguage()) { case "EN": vocOffset = 0; break; case "ES": vocOffset = 80; break; case "FR": vocOffset = 160; break; case "DE": vocOffset = 240; break; case "IT": vocOffset = 320; break; default: vocOffset = 0; break; } //Draw the bouding box only if its in the center of the image (On Cursor) for (int i = 0; i < indices.total(); ++i) { int idx = (int)indices.get(i, 0)[0]; OpenCVForUnity.CoreModule.Rect box = boxesList[idx]; if (isOnCursor(box, cursorObject.GetComponent <Cursor>())) { drawPred(vocOffset + classIdsList[idx], confidencesList[idx], box.x, box.y, box.x + box.width, box.y + box.height, frame); wordDisplay.text = classNames[classIdsList[idx]]; //Update the text summarizing the object encountered if (!vocIDList.Contains(classIdsList[idx])) { //Update the vocabulary learned vocIDList.Add(classIdsList[idx]); EnglishText.text += "\n" + classNames[classIdsList[idx]] + "\n"; SpanishText.text += "\n" + classNames[80 + classIdsList[idx]] + "\n"; FrenchText.text += "\n" + classNames[160 + classIdsList[idx]] + "\n"; GermanText.text += "\n" + classNames[240 + classIdsList[idx]] + "\n"; ItalianText.text += "\n" + classNames[320 + classIdsList[idx]] + "\n"; } } } indices.Dispose(); boxes.Dispose(); confidences.Dispose(); }
/// <summary> /// Process /// </summary> /// <returns></returns> private async void Process() { float DOWNSCALE_RATIO = 1.0f; while (true) { // Check TaskCancel if (tokenSource.Token.IsCancellationRequested) { break; } rgbaMat = webCamTextureToMatHelper.GetMat(); // Debug.Log ("rgbaMat.ToString() " + rgbaMat.ToString ()); Mat downScaleRgbaMat = null; DOWNSCALE_RATIO = 1.0f; if (enableDownScale) { downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(rgbaMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleRgbaMat = rgbaMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.cvtColor(downScaleRgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR); await Task.Run(() => { // detect faces on the downscale image if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped()) { if (net == null) { Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } else { // Create a 4D blob from a frame. Size inpSize = new Size(inpWidth > 0 ? inpWidth : bgrMat.cols(), inpHeight > 0 ? inpHeight : bgrMat.rows()); Mat blob = Dnn.blobFromImage(bgrMat, scale, inpSize, mean, swapRB, false); // Run a model. net.setInput(blob); if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1) { // Faster-RCNN or R-FCN Imgproc.resize(bgrMat, bgrMat, inpSize); Mat imInfo = new Mat(1, 3, CvType.CV_32FC1); imInfo.put(0, 0, new float[] { (float)inpSize.height, (float)inpSize.width, 1.6f }); net.setInput(imInfo, "im_info"); } TickMeter tm = new TickMeter(); tm.start(); List <Mat> outs = new List <Mat>(); net.forward(outs, outBlobNames); tm.stop(); // Debug.Log ("Inference time, ms: " + tm.getTimeMilli ()); postprocess(bgrMat, outs, net); for (int i = 0; i < outs.Count; i++) { outs[i].Dispose(); } blob.Dispose(); if (enableDownScale) { for (int i = 0; i < _boxesList.Count; ++i) { var rect = _boxesList[i]; _boxesList[i] = new OpenCVForUnity.CoreModule.Rect( (int)(rect.x * DOWNSCALE_RATIO), (int)(rect.y * DOWNSCALE_RATIO), (int)(rect.width * DOWNSCALE_RATIO), (int)(rect.height * DOWNSCALE_RATIO)); } } } //Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1); MatOfRect boxes = new MatOfRect(); boxes.fromList(_boxesList); MatOfFloat confidences = new MatOfFloat(); confidences.fromList(_confidencesList); MatOfInt indices = new MatOfInt(); Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); // Debug.Log ("indices.dump () "+indices.dump ()); // Debug.Log ("indices.ToString () "+indices.ToString()); for (int i = 0; i < indices.total(); ++i) { int idx = (int)indices.get(i, 0)[0]; OpenCVForUnity.CoreModule.Rect box = _boxesList[idx]; drawPred(_classIdsList[idx], _confidencesList[idx], box.x, box.y, box.x + box.width, box.y + box.height, rgbaMat); } indices.Dispose(); boxes.Dispose(); confidences.Dispose(); } }); Utils.fastMatToTexture2D(rgbaMat, texture); Thread.Sleep(10); } }
/// <summary> /// Scanning the specified frame, outs and net. /// </summary> /// <param name="frame">Frame.</param> /// <param name="outs">Outs.</param> /// <param name="net">Net.</param> private void postscan(Mat frame, List <Mat> outs, Net net) { string outLayerType = outBlobTypes[0]; List <int> classIdsList = new List <int>(); List <float> confidencesList = new List <float>(); List <OpenCVForUnity.CoreModule.Rect> boxesList = new List <OpenCVForUnity.CoreModule.Rect>(); if (outLayerType == "Region") { for (int i = 0; i < outs.Count; ++i) { // Network produces output blob with a shape NxC where N is a number of // detected objects and C is a number of classes + 4 where the first 4 // numbers are [center_x, center_y, width, height] //Debug.Log("outs[i].ToString() " + outs[i].ToString()); float[] positionData = new float[5]; float[] confidenceData = new float[outs[i].cols() - 5]; for (int p = 0; p < outs[i].rows(); p++) { outs[i].get(p, 0, positionData); outs[i].get(p, 5, confidenceData); int maxIdx = confidenceData.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I; float confidence = confidenceData[maxIdx]; if (confidence > confThreshold) { int centerX = (int)(positionData[0] * frame.cols()); int centerY = (int)(positionData[1] * frame.rows()); int width = (int)(positionData[2] * frame.cols()); int height = (int)(positionData[3] * frame.rows()); int left = centerX - width / 2; int top = centerY - height / 2; classIdsList.Add(maxIdx); confidencesList.Add((float)confidence); boxesList.Add(new OpenCVForUnity.CoreModule.Rect(left, top, width, height)); } } } } else { Debug.Log("Unknown output layer type: " + outLayerType); } MatOfRect boxes = new MatOfRect(); boxes.fromList(boxesList); MatOfFloat confidences = new MatOfFloat(); confidences.fromList(confidencesList); MatOfInt indices = new MatOfInt(); Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); //for-loop for the mini game - if a new class appears, add it to the for (int i = 0; i < indices.total(); ++i) { int idx = (int)indices.get(i, 0)[0]; if (!minigameList.Contains(classIdsList[idx])) { Debug.Log(classNames[classIdsList[idx]]); minigameList.Add(classIdsList[idx]); if (minigameList.Count() > 1) { wordDisplay.text = minigameList.Count().ToString() + " words"; } else { wordDisplay.text = minigameList.Count().ToString() + " word"; } } } indices.Dispose(); boxes.Dispose(); confidences.Dispose(); }