void clamp(float c) { float[] p_float = new float[p.total()]; MatUtils.copyFromMat <float>(p, p_float); int p_cols = p.cols(); float[] e_float = new float[e.total()]; MatUtils.copyFromMat <float>(e, e_float); int e_cols = e.cols(); double scale = p_float[0]; int rows = e.rows(); for (int i = 0; i < rows; i++) { if (e_float[i * e_cols] < 0) { continue; } float v = c * (float)Math.Sqrt(e_float[i * e_cols]); if (Math.Abs(p_float[i * p_cols] / scale) > v) { if (p_float[i * p_cols] > 0) { p_float[i * p_cols] = (float)(v * scale); } else { p_float[i * p_cols] = (float)(-v * scale); } } } MatUtils.copyToMat(p_float, p); }
public List <Point[]> convertMatOfRectToPoints(MatOfRect rects) { List <OpenCVForUnity.CoreModule.Rect> R = rects.toList(); List <Point[]> points = new List <Point[]>(R.Count); int n = reference.rows() / 2; float[] reference_float = new float[reference.total()]; MatUtils.copyFromMat <float>(reference, reference_float); foreach (var r in R) { Vector3 scale = detector_offset * r.width; Point[] p = new Point[n]; for (int i = 0; i < n; i++) { p[i] = new Point(); p[i].x = scale.z * reference_float[2 * i] + r.x + 0.5 * r.width + scale.x; p[i].y = scale.z * reference_float[(2 * i) + 1] + r.y + 0.5 * r.height + scale.y; } points.Add(p); } return(points); }
public void draw(Mat im, Scalar pts_color, Scalar con_color) { int[] smodel_C_int = new int[smodel.C.total()]; MatUtils.copyFromMat <int> (smodel.C, smodel_C_int); foreach (var point in points) { int n = point.Length; if (n == 0) { return; } int rows = smodel.C.rows(); int cols = smodel.C.cols(); for (int i = 0; i < rows; i++) { int j = smodel_C_int [i * cols], k = smodel_C_int [(i * cols) + 1]; #if OPENCV_2 Core.line(im, point[j], point[k], con_color, 1); #else Imgproc.line(im, point [j], point [k], con_color, 1); #endif } for (int i = 0; i < n; i++) { #if OPENCV_2 Core.circle(im, point [i], 1, pts_color, 2, Core.LINE_AA, 0); #else Imgproc.circle(im, point [i], 1, pts_color, 2, Imgproc.LINE_AA, 0); #endif } } }
// Pastes faces on original frame. private void AlphaBlend_pixel(Mat fg, Mat bg, Mat alpha, Mat dst) { byte[] fg_byte = new byte[fg.total() * fg.channels()]; MatUtils.copyFromMat <byte>(fg, fg_byte); byte[] bg_byte = new byte[bg.total() * bg.channels()]; MatUtils.copyFromMat <byte>(bg, bg_byte); byte[] alpha_byte = new byte[alpha.total() * alpha.channels()]; MatUtils.copyFromMat <byte>(alpha, alpha_byte); int pixel_i = 0; int channels = (int)bg.channels(); int total = (int)bg.total(); for (int i = 0; i < total; i++) { if (alpha_byte[i] == 0) { } else if (alpha_byte[i] == 255) { bg_byte[pixel_i] = fg_byte[pixel_i]; bg_byte[pixel_i + 1] = fg_byte[pixel_i + 1]; bg_byte[pixel_i + 2] = fg_byte[pixel_i + 2]; } else { bg_byte[pixel_i] = (byte)(((255 - alpha_byte[i]) * bg_byte[pixel_i] + alpha_byte[i] * fg_byte[pixel_i]) >> 8); bg_byte[pixel_i + 1] = (byte)(((255 - alpha_byte[i]) * bg_byte[pixel_i + 1] + alpha_byte[i] * fg_byte[pixel_i + 1]) >> 8); bg_byte[pixel_i + 2] = (byte)(((255 - alpha_byte[i]) * bg_byte[pixel_i + 2] + alpha_byte[i] * fg_byte[pixel_i + 2]) >> 8); } pixel_i += channels; } MatUtils.copyToMat(bg_byte, dst); }
public void CaptureFrame(byte[] pixelBuffer) { if (uprightMatrix == null) { return; } MatUtils.copyFromMat(uprightMatrix, pixelBuffer); }
Mat calc_simil(Mat pts) { float[] pts_float = new float[pts.total()]; MatUtils.copyFromMat <float>(pts, pts_float); int pts_cols = pts.cols(); //compute translation int n = pts.rows() / 2; float mx = 0, my = 0; for (int i = 0; i < n; i++) { mx += pts_float[(2 * pts_cols) * i]; my += pts_float[((2 * pts_cols) * i) + 1]; } using (Mat p = new Mat(2 * n, 1, CvType.CV_32F)) { float[] p_float = new float[p.total()]; MatUtils.copyFromMat <float>(p, p_float); int p_cols = p.cols(); mx /= n; my /= n; for (int i = 0; i < n; i++) { p_float[(2 * p_cols) * i] = pts_float[(2 * pts_cols) * i] - mx; p_float[((2 * p_cols) * i) + 1] = pts_float[((2 * pts_cols) * i) + 1] - my; } MatUtils.copyToMat(p_float, p); //compute rotation and scale float[] reference_float = new float[reference.total()]; MatUtils.copyFromMat <float>(reference, reference_float); int reference_cols = reference.cols(); float a = 0, b = 0, c = 0; for (int i = 0; i < n; i++) { a += reference_float[(2 * reference_cols) * i] * reference_float[(2 * reference_cols) * i] + reference_float[((2 * reference_cols) * i) + 1] * reference_float[((2 * reference_cols) * i) + 1]; b += reference_float[(2 * reference_cols) * i] * p_float[(2 * p_cols) * i] + reference_float[((2 * reference_cols) * i) + 1] * p_float[((2 * p_cols) * i) + 1]; c += reference_float[(2 * reference_cols) * i] * p_float[((2 * p_cols) * i) + 1] - reference_float[((2 * reference_cols) * i) + 1] * p_float[(2 * p_cols) * i]; } b /= a; c /= a; float scale = (float)Math.Sqrt(b * b + c * c), theta = (float)Math.Atan2(c, b); float sc = scale * (float)Math.Cos(theta), ss = scale * (float)Math.Sin(theta); Mat returnMat = new Mat(2, 3, CvType.CV_32F); returnMat.put(0, 0, sc, -ss, mx, ss, sc, my); return(returnMat); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); // detect faces cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size()); if (faces.total() > 0) { // fit landmarks for each found face List <MatOfPoint2f> landmarks = new List <MatOfPoint2f>(); facemark.fit(grayMat, faces, landmarks); Rect[] rects = faces.toArray(); for (int i = 0; i < rects.Length; i++) { //Debug.Log ("detect faces " + rects [i]); Imgproc.rectangle(rgbaMat, new Point(rects[i].x, rects[i].y), new Point(rects[i].x + rects[i].width, rects[i].y + rects[i].height), new Scalar(255, 0, 0, 255), 2); } // draw them for (int i = 0; i < landmarks.Count; i++) { MatOfPoint2f lm = landmarks[i]; float[] lm_float = new float[lm.total() * lm.channels()]; MatUtils.copyFromMat <float>(lm, lm_float); DrawFaceLandmark(rgbaMat, ConvertArrayToPointList(lm_float), new Scalar(0, 255, 0, 255), 2); //for (int j = 0; j < lm_float.Length; j = j + 2) //{ // Point p = new Point(lm_float[j], lm_float[j + 1]); // Imgproc.circle(rgbaMat, p, 2, new Scalar(255, 0, 0, 255), 1); //} } } // rgbaMat.convertTo(rgbaMat, CvType.CV_8UC3); // Debug.Log(rgbaMat.type()); // OpenCVForUnity.XphotoModule.Xphoto.applyChannelGains(rgbaMat, effectsMat, 1.0f,220.0f, 1.0f); // OpenCVForUnity.XphotoModule.Xphoto.oilPainting(effects, effects, 10, 10); Utils.fastMatToTexture2D(rgbaMat, texture); } }
public Point[] calc_peaks(Mat im, Point[] points, Size ssize) { int n = points.Length; using (Mat pt = (new MatOfPoint2f(points)).reshape(1, 2 * n)) using (Mat S = calc_simil(pt)) using (Mat Si = inv_simil(S)) { float[] pt_float = new float[pt.total()]; MatUtils.copyFromMat <float>(pt, pt_float); int pt_cols = pt.cols(); float[] S_float = new float[S.total()]; MatUtils.copyFromMat <float>(S, S_float); int S_cols = S.cols(); float[] A_float = new float[2 * 3]; Point[] pts = apply_simil(Si, points); for (int i = 0; i < n; i++) { Size wsize = new Size(ssize.width + patches[i].patch_size().width, ssize.height + patches[i].patch_size().height); using (Mat A = new Mat(2, 3, CvType.CV_32F)) { MatUtils.copyFromMat <float>(A, A_float); int A_cols = A.cols(); A_float[0] = S_float[0]; A_float[1] = S_float[1]; A_float[1 * A_cols] = S_float[1 * S_cols]; A_float[(1 * A_cols) + 1] = S_float[(1 * S_cols) + 1]; A_float[2] = (float)(pt_float[(2 * pt_cols) * i] - (A_float[0] * (wsize.width - 1) / 2 + A_float[1] * (wsize.height - 1) / 2)); A_float[(1 * A_cols) + 2] = (float)(pt_float[((2 * pt_cols) * i) + 1] - (A_float[1 * A_cols] * (wsize.width - 1) / 2 + A_float[(1 * A_cols) + 1] * (wsize.height - 1) / 2)); MatUtils.copyToMat(A_float, A); using (Mat I = new Mat()) { Imgproc.warpAffine(im, I, A, wsize, Imgproc.INTER_LINEAR + Imgproc.WARP_INVERSE_MAP); using (Mat R = patches[i].calc_response(I, false)) { Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(R); pts[i].x = pts[i].x + minMaxLocResult.maxLoc.x - 0.5 * ssize.width; pts[i].y = pts[i].y + minMaxLocResult.maxLoc.y - 0.5 * ssize.height; } } } } return(apply_simil(S, pts)); } }
void DoProcess() { if (!(mat.Value is OpenCVForUnityPlayMakerActions.Mat)) { LogError("mat is not initialized. Add Action \"newMat\"."); return; } OpenCVForUnity.CoreModule.Mat wrapped_mat = OpenCVForUnityPlayMakerActionsUtils.GetWrappedObject <OpenCVForUnityPlayMakerActions.Mat, OpenCVForUnity.CoreModule.Mat>(mat); System.Byte[] wrapped_array = OpenCVForUnityPlayMakerActionsUtils.GetWrappedObject <OpenCVForUnityPlayMakerActions.ByteArray, System.Byte[]>(array); MatUtils.copyFromMat <byte>(wrapped_mat, wrapped_array); }
public void calc_params(Point[] pts, Mat weight, float c_factor) { int n = pts.Length; //assert(V.rows == 2*n); //Debug.Log ("V.rows == 2*n " + V.rows () + " " + 2 * n); using (Mat s = (new MatOfPoint2f(pts)).reshape(1, 2 * n)) { //point set to vector format if (weight.total() == 0) { Core.gemm(V.t(), s, 1, new Mat(), 0, p); //simple projection } else { //scaled projection if (weight.rows() != n) { Debug.Log("Invalid weighting matrix"); } float[] weight_float = new float[weight.total()]; MatUtils.copyFromMat <float>(weight, weight_float); int weight_cols = weight.cols(); int K = V.cols(); using (Mat H = Mat.zeros(K, K, CvType.CV_32F)) using (Mat g = Mat.zeros(K, 1, CvType.CV_32F)) { for (int i = 0; i < n; i++) { using (Mat v = new Mat(V, new OpenCVForUnity.CoreModule.Rect(0, 2 * i, K, 2))) using (Mat tmpMat1 = new Mat()) using (Mat tmpMat2 = new Mat()) using (Mat tmpMat3 = new Mat()) { float w = weight_float[i * weight_cols]; Core.multiply(v.t(), new Scalar(w), tmpMat1); Core.gemm(tmpMat1, v, 1, new Mat(), 0, tmpMat2); Core.add(H, tmpMat2, H); Core.gemm(tmpMat1, new MatOfPoint2f(pts[i]).reshape(1, 2), 1, new Mat(), 0, tmpMat3); Core.add(g, tmpMat3, g); } } Core.solve(H, g, p, Core.DECOMP_SVD); } } } clamp(c_factor); //clamp resulting parameters }
public Point[] getConnections() { Point[] c = new Point[smodel.C.rows()]; int[] data = new int[c.Length * 2]; MatUtils.copyFromMat <int> (smodel.C, data); int len = c.Length; for (int i = 0; i < len; i++) { c [i] = new Point(data [i * 2], data [(i * 2) + 1]); } return(c); }
Point[] apply_simil(Mat S, Point[] points) { float[] S_float = new float[S.total()]; MatUtils.copyFromMat <float> (S, S_float); int S_cols = S.cols(); int n = points.Length; Point[] p = new Point[n]; for (int i = 0; i < n; i++) { p [i] = new Point(); p [i].x = S_float [0] * points [i].x + S_float [1] * points [i].y + S_float [2]; p [i].y = S_float [1 * S_cols] * points [i].x + S_float [(1 * S_cols) + 1] * points [i].y + S_float [(1 * S_cols) + 2]; } return(p); }
void DoProcess() { if (!(mat.Value is OpenCVForUnityPlayMakerActions.Mat)) { LogError("mat is not initialized. Add Action \"newMat\"."); return; } OpenCVForUnity.CoreModule.Mat wrapped_mat = OpenCVForUnityPlayMakerActionsUtils.GetWrappedObject <OpenCVForUnityPlayMakerActions.Mat, OpenCVForUnity.CoreModule.Mat>(mat); int[] tmpArray = new int[array.Length]; MatUtils.copyFromMat <int>(wrapped_mat, tmpArray); for (int i = 0; i < tmpArray.Length; i++) { array.Set(i, (float)tmpArray[i]); } array.SaveChanges(); }
public Point[] calc_shape() { using (Mat s = new Mat()) { Core.gemm(V, p, 1, new Mat(), 0, s); float[] s_float = new float[s.total()]; MatUtils.copyFromMat <float> (s, s_float); int s_cols = s.cols(); int n = s.rows() / 2; Point[] pts = new Point[n]; for (int i = 0; i < n; i++) { pts [i] = new Point(s_float [(2 * s_cols) * i], s_float [((2 * s_cols) * i) + 1]); } return(pts); } }
Mat inv_simil(Mat S) { float[] S_float = new float[S.total()]; MatUtils.copyFromMat <float> (S, S_float); int S_cols = S.cols(); Mat Si = new Mat(2, 3, CvType.CV_32F); float d = S_float [0] * S_float [(1 * S_cols) + 1] - S_float [1 * S_cols] * S_float [1]; float[] Si_float = new float[Si.total()]; MatUtils.copyFromMat <float> (Si, Si_float); int Si_cols = Si.cols(); Si_float [0] = S_float [(1 * S_cols) + 1] / d; Si_float [1] = -S_float [1] / d; Si_float [(1 * Si_cols) + 1] = S_float [0] / d; Si_float [1 * Si_cols] = -S_float [1 * S_cols] / d; MatUtils.copyToMat(Si_float, Si); Mat Ri = new Mat(Si, new OpenCVForUnity.CoreModule.Rect(0, 0, 2, 2)); Mat negaRi = new Mat(); Core.multiply(Ri, new Scalar(-1), negaRi); Mat t = new Mat(); Core.gemm(negaRi, S.col(2), 1, new Mat(negaRi.rows(), negaRi.cols(), negaRi.type()), 0, t); Mat St = Si.col(2); t.copyTo(St); return(Si); }
// Match histograms of 'src' to that of 'dst', according to both masks. public static void CalculateLUT(Mat src, Mat dst, Mat src_mask, Mat dst_mask, Texture2D LUTTex) { if (src.channels() < 3) { throw new ArgumentException("src.channels() < 3"); } if (dst.channels() < 3) { throw new ArgumentException("dst.channels() < 3"); } if (src_mask.channels() != 1) { throw new ArgumentException("src_mask.channels() != 1"); } if (dst_mask.channels() != 1) { throw new ArgumentException("dst_mask.channels() != 1"); } if (src_mask != null && src.total() != src_mask.total()) { throw new ArgumentException("src.total() != src_mask.total()"); } if (dst_mask != null && dst.total() != dst_mask.total()) { throw new ArgumentException("dst.total() != dst_mask.total()"); } if (LUTTex.width != 256 || LUTTex.height != 1 || LUTTex.format != TextureFormat.RGB24) { throw new ArgumentException("Invalid LUTTex."); } byte[] LUT = new byte[3 * 256]; double[][] src_hist = new double[3][]; for (int i = 0; i < src_hist.Length; i++) { src_hist [i] = new double[256]; } double[][] dst_hist = new double[3][]; for (int i = 0; i < dst_hist.Length; i++) { dst_hist [i] = new double[256]; } double[][] src_cdf = new double[3][]; for (int i = 0; i < src_cdf.Length; i++) { src_cdf [i] = new double[256]; } double[][] dst_cdf = new double[3][]; for (int i = 0; i < dst_cdf.Length; i++) { dst_cdf [i] = new double[256]; } double[] src_histMax = new double[3]; double[] dst_histMax = new double[3]; byte[] src_mask_byte = null; byte[] dst_mask_byte = null; if (src_mask != null) { src_mask_byte = new byte[src_mask.total() * src_mask.channels()]; MatUtils.copyFromMat <byte> (src_mask, src_mask_byte); } if (dst_mask != null) { dst_mask_byte = new byte[dst_mask.total() * dst_mask.channels()]; MatUtils.copyFromMat <byte> (dst_mask, dst_mask_byte); } byte[] src_byte = new byte[src.total() * src.channels()]; MatUtils.copyFromMat <byte> (src, src_byte); byte[] dst_byte = new byte[dst.total() * dst.channels()]; MatUtils.copyFromMat <byte> (dst, dst_byte); int pixel_i = 0; int channels = src.channels(); int total = (int)src.total(); if (src_mask_byte != null) { for (int i = 0; i < total; i++) { if (src_mask_byte [i] != 0) { byte c = src_byte [pixel_i]; src_hist [0] [c]++; if (src_hist [0] [c] > src_histMax [0]) { src_histMax [0] = src_hist [0] [c]; } c = src_byte [pixel_i + 1]; src_hist [1] [c]++; if (src_hist [1] [c] > src_histMax [1]) { src_histMax [1] = src_hist [1] [c]; } c = src_byte [pixel_i + 2]; src_hist [2] [c]++; if (src_hist [2] [c] > src_histMax [2]) { src_histMax [2] = src_hist [2] [c]; } } // Advance to next pixel pixel_i += channels; } } else { for (int i = 0; i < total; i++) { byte c = src_byte [pixel_i]; src_hist [0] [c]++; if (src_hist [0] [c] > src_histMax [0]) { src_histMax [0] = src_hist [0] [c]; } c = src_byte [pixel_i + 1]; src_hist [1] [c]++; if (src_hist [1] [c] > src_histMax [1]) { src_histMax [1] = src_hist [1] [c]; } c = src_byte [pixel_i + 2]; src_hist [2] [c]++; if (src_hist [2] [c] > src_histMax [2]) { src_histMax [2] = src_hist [2] [c]; } // Advance to next pixel pixel_i += channels; } } pixel_i = 0; channels = dst.channels(); total = (int)dst.total(); if (dst_mask_byte != null) { for (int i = 0; i < total; i++) { if (dst_mask_byte [i] != 0) { byte c = dst_byte [pixel_i]; dst_hist [0] [c]++; if (dst_hist [0] [c] > dst_histMax [0]) { dst_histMax [0] = dst_hist [0] [c]; } c = dst_byte [pixel_i + 1]; dst_hist [1] [c]++; if (dst_hist [1] [c] > dst_histMax [1]) { dst_histMax [1] = dst_hist [1] [c]; } c = dst_byte [pixel_i + 2]; dst_hist [2] [c]++; if (dst_hist [2] [c] > dst_histMax [2]) { dst_histMax [2] = dst_hist [2] [c]; } } // Advance to next pixel pixel_i += channels; } } else { for (int i = 0; i < total; i++) { byte c = dst_byte [pixel_i]; dst_hist [0] [c]++; if (dst_hist [0] [c] > dst_histMax [0]) { dst_histMax [0] = dst_hist [0] [c]; } c = dst_byte [pixel_i + 1]; dst_hist [1] [c]++; if (dst_hist [1] [c] > dst_histMax [1]) { dst_histMax [1] = dst_hist [1] [c]; } c = dst_byte [pixel_i + 2]; dst_hist [2] [c]++; if (dst_hist [2] [c] > dst_histMax [2]) { dst_histMax [2] = dst_hist [2] [c]; } // Advance to next pixel pixel_i += channels; } } //normalize hist for (int i = 0; i < 256; i++) { src_hist [0] [i] /= src_histMax [0]; src_hist [1] [i] /= src_histMax [1]; src_hist [2] [i] /= src_histMax [2]; dst_hist [0] [i] /= dst_histMax [0]; dst_hist [1] [i] /= dst_histMax [1]; dst_hist [2] [i] /= dst_histMax [2]; } // Calc cumulative distribution function (CDF) src_cdf [0] [0] = src_hist [0] [0]; src_cdf [1] [0] = src_hist [1] [0]; src_cdf [2] [0] = src_hist [2] [0]; dst_cdf [0] [0] = dst_hist [0] [0]; dst_cdf [1] [0] = dst_hist [1] [0]; dst_cdf [2] [0] = dst_hist [2] [0]; for (int i = 1; i < 256; i++) { src_cdf [0] [i] = src_cdf [0] [i - 1] + src_hist [0] [i]; src_cdf [1] [i] = src_cdf [1] [i - 1] + src_hist [1] [i]; src_cdf [2] [i] = src_cdf [2] [i - 1] + src_hist [2] [i]; dst_cdf [0] [i] = dst_cdf [0] [i - 1] + dst_hist [0] [i]; dst_cdf [1] [i] = dst_cdf [1] [i - 1] + dst_hist [1] [i]; dst_cdf [2] [i] = dst_cdf [2] [i - 1] + dst_hist [2] [i]; } // Normalize CDF for (int i = 0; i < 256; i++) { src_cdf [0] [i] /= src_cdf [0] [255]; src_cdf [1] [i] /= src_cdf [1] [255]; src_cdf [2] [i] /= src_cdf [2] [255]; dst_cdf [0] [i] /= dst_cdf [0] [255]; dst_cdf [1] [i] /= dst_cdf [1] [255]; dst_cdf [2] [i] /= dst_cdf [2] [255]; } // Create lookup table const double HISTMATCH_EPSILON = 0.000001f; for (int i = 0; i < 3; i++) { int last = 0; for (int j = 0; j < 256; j++) { double F1j = src_cdf [i] [j]; for (int k = last; k < 256; k++) { double F2k = dst_cdf [i] [k]; if (Math.Abs(F2k - F1j) < HISTMATCH_EPSILON || F2k > F1j) { LUT [(j * 3) + i] = (byte)k; last = k; break; } } } } LUTTex.LoadRawTextureData(LUT); LUTTex.Apply(false); }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame() && !imageOptimizationHelper.IsCurrentFrameSkipped()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); //get downScaleMat; Mat downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat((rgbaMat)); //grayscale Imgproc.cvtColor(downScaleRgbaMat, gray1Mat, Imgproc.COLOR_RGBA2GRAY); //blur Imgproc.blur(gray1Mat, gray2Mat, new Size(5, 5)); //edge filter Imgproc.filter2D(gray2Mat, gray1Mat, gray1Mat.depth(), kernel); //blur Imgproc.blur(gray1Mat, gray2Mat, new Size(3, 3)); //detect edge Imgproc.threshold(gray2Mat, gray2Mat, EDGE_DETECT_VALUE, 255, Imgproc.THRESH_BINARY); //copy Mat to byteArray MatUtils.copyFromMat <byte>(gray2Mat, byteArray); //set edge pointList List <Point> pointList = new List <Point>(); int w = gray1Mat.width(); int h = gray1Mat.height(); for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) { if (byteArray[x + w * y] == 255) { pointList.Add(new Point(x, y)); } } } int limit = Mathf.RoundToInt((float)(pointList.Count * POINT_RATE)); if (limit > POINT_MAX_NUM) { limit = POINT_MAX_NUM; } while (pointList.Count > limit) { pointList.RemoveAt(Random.Range(0, pointList.Count)); } //Debug.Log ("pointList.Count " + pointList.Count); //init subdiv subdiv.initDelaunay(new OpenCVForUnity.CoreModule.Rect(0, 0, downScaleRgbaMat.width(), downScaleRgbaMat.height())); for (int i = 0; i < pointList.Count; i++) { subdiv.insert(pointList[i]); } subdiv.insert(new Point(0, 0)); subdiv.insert(new Point(gray1Mat.width() / 2 - 1, 0)); subdiv.insert(new Point(gray1Mat.width() - 1, 0)); subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() / 2 - 1)); subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() - 1)); subdiv.insert(new Point(gray1Mat.width() / 2 - 1, gray1Mat.height() - 1)); subdiv.insert(new Point(0, gray1Mat.height() - 1)); subdiv.insert(new Point(0, gray1Mat.height() / 2 - 1)); using (MatOfFloat6 triangleList = new MatOfFloat6()) { subdiv.getTriangleList(triangleList); float[] pointArray = triangleList.toArray(); float downScaleRatio = imageOptimizationHelper.downscaleRatio; if (downScaleRatio < 1) { downScaleRatio = 1; } byte[] color = new byte[4]; for (int i = 0; i < pointArray.Length / 6; i++) { Point p0 = new Point(pointArray[i * 6 + 0] * downScaleRatio, pointArray[i * 6 + 1] * downScaleRatio); Point p1 = new Point(pointArray[i * 6 + 2] * downScaleRatio, pointArray[i * 6 + 3] * downScaleRatio); Point p2 = new Point(pointArray[i * 6 + 4] * downScaleRatio, pointArray[i * 6 + 5] * downScaleRatio); if (p0.x < 0 || p0.x > rgbaMat.width()) { continue; } if (p0.y < 0 || p0.y > rgbaMat.height()) { continue; } if (p1.x < 0 || p1.x > rgbaMat.width()) { continue; } if (p1.y < 0 || p1.y > rgbaMat.height()) { continue; } if (p2.x < 0 || p2.x > rgbaMat.width()) { continue; } if (p2.y < 0 || p2.y > rgbaMat.height()) { continue; } //get center of gravity int cx = (int)((p0.x + p1.x + p2.x) * 0.33333); int cy = (int)((p0.y + p1.y + p2.y) * 0.33333); // Debug.Log ("cx " + cx + " cy " + cy ); //get center of gravity color rgbaMat.get(cy, cx, color); //Debug.Log ("r " + color[0] + " g " + color[1] + " b " + color[2] + " a " + color[3]); //fill Polygon Imgproc.fillConvexPoly(rgbaMat, new MatOfPoint(p0, p1, p2), new Scalar(color[0], color[1], color[2], color[3]), Imgproc.LINE_AA, 0); //Imgproc.line (rgbaMat, p0, p1, new Scalar (64, 255, 128, 255)); //Imgproc.line (rgbaMat, p1, p2, new Scalar (64, 255, 128, 255)); //Imgproc.line (rgbaMat, p2, p0, new Scalar (64, 255, 128, 255)); } } //Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " DOWNSCALE W:" + downScaleRgbaMat.width () + " H:" + downScaleRgbaMat.height (), new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Utils.fastMatToTexture2D(rgbaMat, texture); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); if (applyComicFilter) { comicFilter.Process(rgbaMat, rgbaMat); } if (isVideoRecording && !isFinishWriting) { textPos.x = 5; textPos.y = rgbaMat.rows() - 70; Imgproc.putText(rgbaMat, exampleTitle, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false); textPos.y = rgbaMat.rows() - 50; Imgproc.putText(rgbaMat, exampleSceneTitle, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false); if (container == ContainerPreset.MP4 || container == ContainerPreset.HEVC) { textPos.y = rgbaMat.rows() - 30; Imgproc.putText(rgbaMat, settingInfo1, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false); textPos.y = rgbaMat.rows() - 10; Imgproc.putText(rgbaMat, settingInfo2, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false); } else if (container == ContainerPreset.GIF) { textPos.y = rgbaMat.rows() - 30; Imgproc.putText(rgbaMat, settingInfoGIF, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false); } else if (container == ContainerPreset.JPG) { textPos.y = rgbaMat.rows() - 30; Imgproc.putText(rgbaMat, settingInfoJPG, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false); } } // Upload the image Mat data to the Texture2D. // (The internal processing of the fastMatToTexture method restore the image Mat data to Unity coordinate system) Utils.fastMatToTexture2D(rgbaMat, texture); // Record frames if (videoRecorder != null && (isVideoRecording && !isFinishWriting) && frameCount++ % recordEveryNthFrame == 0) { #if !OPENCV_USE_UNSAFE_CODE MatUtils.copyFromMat(rgbaMat, pixelBuffer); videoRecorder.CommitFrame(pixelBuffer, recordingClock.timestamp); #else unsafe { videoRecorder.CommitFrame((void *)rgbaMat.dataAddr(), recordingClock.timestamp); } #endif } } if (isVideoPlaying && videoPlayer.isPlaying) { gameObject.GetComponent <Renderer>().sharedMaterial.mainTexture = videoPlayer.texture; } }
// Histogram Equalize seperately for the left and right sides of the face. private static void equalizeLeftAndRightHalves(Mat faceImg) { // It is common that there is stronger light from one half of the face than the other. In that case, // if you simply did histogram equalization on the whole face then it would make one half dark and // one half bright. So we will do histogram equalization separately on each face half, so they will // both look similar on average. But this would cause a sharp edge in the middle of the face, because // the left half and right half would be suddenly different. So we also histogram equalize the whole // image, and in the middle part we blend the 3 images together for a smooth brightness transition. int w = faceImg.cols(); int h = faceImg.rows(); // 1) First, equalize the whole face. using (Mat wholeFace = new Mat(h, w, CvType.CV_8UC1)) { Imgproc.equalizeHist(faceImg, wholeFace); // 2) Equalize the left half and the right half of the face separately. int midX = w / 2; using (Mat leftSide = new Mat(faceImg, new Rect(0, 0, midX, h))) using (Mat rightSide = new Mat(faceImg, new Rect(midX, 0, w - midX, h))) { Imgproc.equalizeHist(leftSide, leftSide); Imgproc.equalizeHist(rightSide, rightSide); // 3) Combine the left half and right half and whole face together, so that it has a smooth transition. byte[] wholeFace_byte = new byte[wholeFace.total() * wholeFace.elemSize()]; MatUtils.copyFromMat <byte>(wholeFace, wholeFace_byte); byte[] leftSide_byte = new byte[leftSide.total() * leftSide.elemSize()]; MatUtils.copyFromMat <byte>(leftSide, leftSide_byte); byte[] rightSide_byte = new byte[rightSide.total() * rightSide.elemSize()]; MatUtils.copyFromMat <byte>(rightSide, rightSide_byte); int leftSide_w = leftSide.cols(); int rightSide_w = rightSide.cols(); for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) { byte wv = wholeFace_byte[y * w + x]; if (x < w / 4) { // Left 25%: just use the left face. wv = leftSide_byte[y * leftSide_w + x]; } else if (x < w * 2 / 4) { // Mid-left 25%: blend the left face & whole face. byte lv = leftSide_byte[y * leftSide_w + x]; // Blend more of the whole face as it moves further right along the face. float f = (x - w * 1 / 4) / (w * 0.25f); wv = (byte)Mathf.Round((1.0f - f) * lv + f * wv); } else if (x < w * 3 / 4) { // Mid-right 25%: blend the right face & whole face. byte rv = rightSide_byte[y * rightSide_w + x - midX]; // Blend more of the right-side face as it moves further right along the face. float f = (x - w * 2 / 4) / (w * 0.25f); wv = (byte)Mathf.Round((1.0f - f) * wv + f * rv); } else { // Right 25%: just use the right face. wv = rightSide_byte[y * rightSide_w + x - midX]; } } // end x loop } //end y loop MatUtils.copyToMat(wholeFace_byte, faceImg); } } }
protected override void postprocess(Mat frame, List <Mat> outs, Net net, int backend = Dnn.DNN_BACKEND_OPENCV) { List <int> classIdsList = new List <int>(); List <float> confidencesList = new List <float>(); List <Rect2d> boxesList = new List <Rect2d>(); List <Point[]> pointsList = new List <Point[]>(); if (outs.Count == 2) { // reshape mat : outs[0]:[1, x, 4] to [x, 4], outs[1]:[1, x, 2] to [x, 2] Mat boxes_m = outs[0].reshape(1, new int[] { outs[0].size(1), outs[0].size(2) }); Mat scores_m = outs[1].reshape(1, new int[] { outs[1].size(1), outs[1].size(2) }); //Debug.Log("boxes_m: " + boxes_m); //Debug.Log("scores_m: " + scores_m); //Debug.Log("priors: " + priors); convertLocationsToBoxes(boxes_m, priors, 0.1f, 0.2f); centerFormToCornerForm(boxes_m); Mat boxes_0_4 = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(0, 4)); float[] boxes_arr = new float[boxes_0_4.rows() * boxes_0_4.cols()]; MatUtils.copyFromMat(boxes_0_4, boxes_arr); Mat scores_1_2 = new Mat(scores_m, new Range(0, scores_m.rows()), new Range(1, 2)); float[] confidences_arr = new float[scores_1_2.rows()]; MatUtils.copyFromMat(scores_1_2, confidences_arr); for (int i = 0; i < boxes_m.rows(); i++) { float confidence = confidences_arr[i]; if (confidence > confThreshold) { int boxes_index = i * 4; float left = boxes_arr[boxes_index] * frame.cols(); float top = boxes_arr[boxes_index + 1] * frame.rows(); float right = boxes_arr[boxes_index + 2] * frame.cols(); float bottom = boxes_arr[boxes_index + 3] * frame.rows(); float width = right - left + 1f; float height = bottom - top + 1f; classIdsList.Add(0); confidencesList.Add(confidence); boxesList.Add(new Rect2d(left, top, width, height)); } } if (boxes_m.cols() > 4 && boxes_m.cols() % 2 == 0) { Mat points = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(4, boxes_m.cols())); float[] points_arr = new float[points.rows() * points.cols()]; MatUtils.copyFromMat(points, points_arr); for (int i = 0; i < boxes_m.rows(); i++) { float confidence = confidences_arr[i]; if (confidence > confThreshold) { int points_index = i * points.cols(); Point[] p_arr = new Point[points.cols() / 2]; for (int index = 0; index < points.cols() / 2; index++) { float x = points_arr[points_index + index * 2] * frame.cols(); float y = points_arr[points_index + index * 2 + 1] * frame.rows(); p_arr[index] = new Point(x, y); } pointsList.Add(p_arr); } } } } MatOfRect2d boxes = new MatOfRect2d(); boxes.fromList(boxesList); MatOfFloat confidences = new MatOfFloat(); confidences.fromList(confidencesList); MatOfInt indices = new MatOfInt(); Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); //Debug.Log("indices.dump () " + indices.dump()); //Debug.Log ("indices.ToString () "+indices.ToString()); for (int i = 0; i < indices.total(); ++i) { int idx = (int)indices.get(i, 0)[0]; Rect2d box = boxesList[idx]; drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y, box.x + box.width, box.y + box.height, frame); if (pointsList.Count > 0) { drawPredPoints(pointsList[idx], frame); } } indices.Dispose(); boxes.Dispose(); confidences.Dispose(); }
private Point[] fit(Mat image, Point[] init, Size ssize, bool robust, int itol, double ftol) { int n = smodel.npts(); // assert((int(init.size())==n) && (pmodel.n_patches()==n)); // Debug.Log ("init.size())==n " + init.Length + " " + n); // Debug.Log ("pmodel.n_patches()==n " + pmodel.n_patches () + " " + n); smodel.calc_params(init, new Mat(), 3.0f); Point[] pts = smodel.calc_shape(); //find facial features in image around current estimates Point[] peaks = pmodel.calc_peaks(image, pts, ssize); //optimise if (!robust) { smodel.calc_params(peaks, new Mat(), 3.0f); //compute shape model parameters pts = smodel.calc_shape(); //update shape } else { using (Mat weight = new Mat(n, 1, CvType.CV_32F)) using (Mat weight_sort = new Mat(n, 1, CvType.CV_32F)) { float[] weight_float = new float[weight.total()]; MatUtils.copyFromMat <float> (weight, weight_float); float[] weight_sort_float = new float[weight_sort.total()]; Point[] pts_old = pts; for (int iter = 0; iter < itol; iter++) { //compute robust weight for (int i = 0; i < n; i++) { using (MatOfPoint tmpMat = new MatOfPoint(new Point(pts [i].x - peaks [i].x, pts [i].y - peaks [i].y))) { weight_float [i] = (float)Core.norm(tmpMat); } } MatUtils.copyToMat(weight_float, weight); Core.sort(weight, weight_sort, Core.SORT_EVERY_COLUMN | Core.SORT_ASCENDING); MatUtils.copyFromMat <float> (weight_sort, weight_sort_float); double var = 1.4826 * weight_sort_float [n / 2]; if (var < 0.1) { var = 0.1; } Core.pow(weight, 2, weight); Core.multiply(weight, new Scalar(-0.5 / (var * var)), weight); Core.exp(weight, weight); //compute shape model parameters smodel.calc_params(peaks, weight, 3.0f); //update shape pts = smodel.calc_shape(); //check for convergence float v = 0; for (int i = 0; i < n; i++) { using (MatOfPoint tmpMat = new MatOfPoint(new Point(pts [i].x - pts_old [i].x, pts [i].y - pts_old [i].y))) { v += (float)Core.norm(tmpMat); } } if (v < ftol) { break; } else { pts_old = pts; } } } } return(pts); }
// Calculates source image histogram and changes target_image to match source hist. private void specifyHistogram(Mat source_image, Mat target_image, Mat mask) { byte[][] LUT = new byte[3][]; for (int i = 0; i < LUT.Length; i++) { LUT[i] = new byte[256]; } double[][] source_hist = new double[3][]; for (int i = 0; i < source_hist.Length; i++) { source_hist[i] = new double[256]; } double[][] target_hist = new double[3][]; for (int i = 0; i < target_hist.Length; i++) { target_hist[i] = new double[256]; } double[][] source_cdf = new double[3][]; for (int i = 0; i < source_cdf.Length; i++) { source_cdf[i] = new double[256]; } double[][] target_cdf = new double[3][]; for (int i = 0; i < target_cdf.Length; i++) { target_cdf[i] = new double[256]; } double[] source_histMax = new double[3]; double[] target_histMax = new double[3]; byte[] mask_byte = new byte[mask.total() * mask.channels()]; MatUtils.copyFromMat <byte>(mask, mask_byte); byte[] source_image_byte = new byte[source_image.total() * source_image.channels()]; MatUtils.copyFromMat <byte>(source_image, source_image_byte); byte[] target_image_byte = new byte[target_image.total() * target_image.channels()]; MatUtils.copyFromMat <byte>(target_image, target_image_byte); int pixel_i = 0; int channels = (int)source_image.channels(); int total = (int)source_image.total(); for (int i = 0; i < total; i++) { if (mask_byte[i] != 0) { byte c = source_image_byte[pixel_i]; source_hist[0][c]++; if (source_hist[0][c] > source_histMax[0]) { source_histMax[0] = source_hist[0][c]; } c = source_image_byte[pixel_i + 1]; source_hist[1][c]++; if (source_hist[1][c] > source_histMax[1]) { source_histMax[1] = source_hist[1][c]; } c = source_image_byte[pixel_i + 2]; source_hist[2][c]++; if (source_hist[2][c] > source_histMax[2]) { source_histMax[2] = source_hist[2][c]; } c = target_image_byte[pixel_i]; target_hist[0][c]++; if (target_hist[0][c] > target_histMax[0]) { target_histMax[0] = target_hist[0][c]; } c = target_image_byte[pixel_i + 1]; target_hist[1][c]++; if (target_hist[1][c] > target_histMax[1]) { target_histMax[1] = target_hist[1][c]; } c = target_image_byte[pixel_i + 2]; target_hist[2][c]++; if (target_hist[2][c] > target_histMax[2]) { target_histMax[2] = target_hist[2][c]; } } // Advance to next pixel pixel_i += channels; } // Normalize hist for (int i = 0; i < 256; i++) { source_hist[0][i] /= source_histMax[0]; source_hist[1][i] /= source_histMax[1]; source_hist[2][i] /= source_histMax[2]; target_hist[0][i] /= target_histMax[0]; target_hist[1][i] /= target_histMax[1]; target_hist[2][i] /= target_histMax[2]; } // Calc cumulative distribution function (CDF) source_cdf[0][0] = source_hist[0][0]; source_cdf[1][0] = source_hist[1][0]; source_cdf[2][0] = source_hist[2][0]; target_cdf[0][0] = target_hist[0][0]; target_cdf[1][0] = target_hist[1][0]; target_cdf[2][0] = target_hist[2][0]; for (int i = 1; i < 256; i++) { source_cdf[0][i] = source_cdf[0][i - 1] + source_hist[0][i]; source_cdf[1][i] = source_cdf[1][i - 1] + source_hist[1][i]; source_cdf[2][i] = source_cdf[2][i - 1] + source_hist[2][i]; target_cdf[0][i] = target_cdf[0][i - 1] + target_hist[0][i]; target_cdf[1][i] = target_cdf[1][i - 1] + target_hist[1][i]; target_cdf[2][i] = target_cdf[2][i - 1] + target_hist[2][i]; } // Normalize CDF for (int i = 0; i < 256; i++) { source_cdf[0][i] /= source_cdf[0][255]; source_cdf[1][i] /= source_cdf[1][255]; source_cdf[2][i] /= source_cdf[2][255]; target_cdf[0][i] /= target_cdf[0][255]; target_cdf[1][i] /= target_cdf[1][255]; target_cdf[2][i] /= target_cdf[2][255]; } // Create lookup table (LUT) const double HISTMATCH_EPSILON = 0.000001f; for (int i = 0; i < 3; i++) { int last = 0; for (int j = 0; j < 256; j++) { double F1j = target_cdf[i][j]; for (int k = last; k < 256; k++) { double F2k = source_cdf[i][k]; if (Math.Abs(F2k - F1j) < HISTMATCH_EPSILON || F2k > F1j) { LUT[i][j] = (byte)k; last = k; break; } } } } // Repaint pixels pixel_i = 0; for (int i = 0; i < total; i++) { if (mask_byte[i] != 0) { target_image_byte[pixel_i] = LUT[0][target_image_byte[pixel_i]]; target_image_byte[pixel_i + 1] = LUT[1][target_image_byte[pixel_i + 1]]; target_image_byte[pixel_i + 2] = LUT[2][target_image_byte[pixel_i + 2]]; } // Advance to next pixel pixel_i += channels; } MatUtils.copyToMat(target_image_byte, target_image); }