void clamp(float c)
{
float[] p_float = new float[p.total()];
MatUtils.copyFromMat <float>(p, p_float);
int p_cols = p.cols();
float[] e_float = new float[e.total()];
MatUtils.copyFromMat <float>(e, e_float);
int e_cols = e.cols();
double scale = p_float[0];
int rows = e.rows();
for (int i = 0; i < rows; i++)
{
if (e_float[i * e_cols] < 0)
{
continue;
}
float v = c * (float)Math.Sqrt(e_float[i * e_cols]);
if (Math.Abs(p_float[i * p_cols] / scale) > v)
{
if (p_float[i * p_cols] > 0)
{
p_float[i * p_cols] = (float)(v * scale);
}
else
{
p_float[i * p_cols] = (float)(-v * scale);
}
}
}
MatUtils.copyToMat(p_float, p);
}
public List <Point[]> convertMatOfRectToPoints(MatOfRect rects)
{
List <OpenCVForUnity.CoreModule.Rect> R = rects.toList();
List <Point[]> points = new List <Point[]>(R.Count);
int n = reference.rows() / 2;
float[] reference_float = new float[reference.total()];
MatUtils.copyFromMat <float>(reference, reference_float);
foreach (var r in R)
{
Vector3 scale = detector_offset * r.width;
Point[] p = new Point[n];
for (int i = 0; i < n; i++)
{
p[i] = new Point();
p[i].x = scale.z * reference_float[2 * i] + r.x + 0.5 * r.width + scale.x;
p[i].y = scale.z * reference_float[(2 * i) + 1] + r.y + 0.5 * r.height + scale.y;
}
points.Add(p);
}
return(points);
}
public void draw(Mat im, Scalar pts_color, Scalar con_color)
{
int[] smodel_C_int = new int[smodel.C.total()];
MatUtils.copyFromMat <int> (smodel.C, smodel_C_int);
foreach (var point in points)
{
int n = point.Length;
if (n == 0)
{
return;
}
int rows = smodel.C.rows();
int cols = smodel.C.cols();
for (int i = 0; i < rows; i++)
{
int j = smodel_C_int [i * cols], k = smodel_C_int [(i * cols) + 1];
#if OPENCV_2
Core.line(im, point[j], point[k], con_color, 1);
#else
Imgproc.line(im, point [j], point [k], con_color, 1);
#endif
}
for (int i = 0; i < n; i++)
{
#if OPENCV_2
Core.circle(im, point [i], 1, pts_color, 2, Core.LINE_AA, 0);
#else
Imgproc.circle(im, point [i], 1, pts_color, 2, Imgproc.LINE_AA, 0);
#endif
}
}
}
// Pastes faces on original frame.
private void AlphaBlend_pixel(Mat fg, Mat bg, Mat alpha, Mat dst)
{
byte[] fg_byte = new byte[fg.total() * fg.channels()];
MatUtils.copyFromMat <byte>(fg, fg_byte);
byte[] bg_byte = new byte[bg.total() * bg.channels()];
MatUtils.copyFromMat <byte>(bg, bg_byte);
byte[] alpha_byte = new byte[alpha.total() * alpha.channels()];
MatUtils.copyFromMat <byte>(alpha, alpha_byte);
int pixel_i = 0;
int channels = (int)bg.channels();
int total = (int)bg.total();
for (int i = 0; i < total; i++)
{
if (alpha_byte[i] == 0)
{
}
else if (alpha_byte[i] == 255)
{
bg_byte[pixel_i] = fg_byte[pixel_i];
bg_byte[pixel_i + 1] = fg_byte[pixel_i + 1];
bg_byte[pixel_i + 2] = fg_byte[pixel_i + 2];
}
else
{
bg_byte[pixel_i] = (byte)(((255 - alpha_byte[i]) * bg_byte[pixel_i] + alpha_byte[i] * fg_byte[pixel_i]) >> 8);
bg_byte[pixel_i + 1] = (byte)(((255 - alpha_byte[i]) * bg_byte[pixel_i + 1] + alpha_byte[i] * fg_byte[pixel_i + 1]) >> 8);
bg_byte[pixel_i + 2] = (byte)(((255 - alpha_byte[i]) * bg_byte[pixel_i + 2] + alpha_byte[i] * fg_byte[pixel_i + 2]) >> 8);
}
pixel_i += channels;
}
MatUtils.copyToMat(bg_byte, dst);
}
public void CaptureFrame(byte[] pixelBuffer)
{
if (uprightMatrix == null)
{
return;
}
MatUtils.copyFromMat(uprightMatrix, pixelBuffer);
}
Mat calc_simil(Mat pts)
{
float[] pts_float = new float[pts.total()];
MatUtils.copyFromMat <float>(pts, pts_float);
int pts_cols = pts.cols();
//compute translation
int n = pts.rows() / 2;
float mx = 0, my = 0;
for (int i = 0; i < n; i++)
{
mx += pts_float[(2 * pts_cols) * i];
my += pts_float[((2 * pts_cols) * i) + 1];
}
using (Mat p = new Mat(2 * n, 1, CvType.CV_32F))
{
float[] p_float = new float[p.total()];
MatUtils.copyFromMat <float>(p, p_float);
int p_cols = p.cols();
mx /= n;
my /= n;
for (int i = 0; i < n; i++)
{
p_float[(2 * p_cols) * i] = pts_float[(2 * pts_cols) * i] - mx;
p_float[((2 * p_cols) * i) + 1] = pts_float[((2 * pts_cols) * i) + 1] - my;
}
MatUtils.copyToMat(p_float, p);
//compute rotation and scale
float[] reference_float = new float[reference.total()];
MatUtils.copyFromMat <float>(reference, reference_float);
int reference_cols = reference.cols();
float a = 0, b = 0, c = 0;
for (int i = 0; i < n; i++)
{
a += reference_float[(2 * reference_cols) * i] * reference_float[(2 * reference_cols) * i] +
reference_float[((2 * reference_cols) * i) + 1] * reference_float[((2 * reference_cols) * i) + 1];
b += reference_float[(2 * reference_cols) * i] * p_float[(2 * p_cols) * i] +
reference_float[((2 * reference_cols) * i) + 1] * p_float[((2 * p_cols) * i) + 1];
c += reference_float[(2 * reference_cols) * i] * p_float[((2 * p_cols) * i) + 1] -
reference_float[((2 * reference_cols) * i) + 1] * p_float[(2 * p_cols) * i];
}
b /= a;
c /= a;
float scale = (float)Math.Sqrt(b * b + c * c), theta = (float)Math.Atan2(c, b);
float sc = scale * (float)Math.Cos(theta), ss = scale * (float)Math.Sin(theta);
Mat returnMat = new Mat(2, 3, CvType.CV_32F);
returnMat.put(0, 0, sc, -ss, mx, ss, sc, my);
return(returnMat);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
// detect faces
cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE
new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size());
if (faces.total() > 0)
{
// fit landmarks for each found face
List <MatOfPoint2f> landmarks = new List <MatOfPoint2f>();
facemark.fit(grayMat, faces, landmarks);
Rect[] rects = faces.toArray();
for (int i = 0; i < rects.Length; i++)
{
//Debug.Log ("detect faces " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects[i].x, rects[i].y), new Point(rects[i].x + rects[i].width, rects[i].y + rects[i].height), new Scalar(255, 0, 0, 255), 2);
}
// draw them
for (int i = 0; i < landmarks.Count; i++)
{
MatOfPoint2f lm = landmarks[i];
float[] lm_float = new float[lm.total() * lm.channels()];
MatUtils.copyFromMat <float>(lm, lm_float);
DrawFaceLandmark(rgbaMat, ConvertArrayToPointList(lm_float), new Scalar(0, 255, 0, 255), 2);
//for (int j = 0; j < lm_float.Length; j = j + 2)
//{
// Point p = new Point(lm_float[j], lm_float[j + 1]);
// Imgproc.circle(rgbaMat, p, 2, new Scalar(255, 0, 0, 255), 1);
//}
}
}
// rgbaMat.convertTo(rgbaMat, CvType.CV_8UC3);
// Debug.Log(rgbaMat.type());
// OpenCVForUnity.XphotoModule.Xphoto.applyChannelGains(rgbaMat, effectsMat, 1.0f,220.0f, 1.0f);
// OpenCVForUnity.XphotoModule.Xphoto.oilPainting(effects, effects, 10, 10);
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
public Point[] calc_peaks(Mat im, Point[] points, Size ssize)
{
int n = points.Length;
using (Mat pt = (new MatOfPoint2f(points)).reshape(1, 2 * n)) using (Mat S = calc_simil(pt)) using (Mat Si = inv_simil(S))
{
float[] pt_float = new float[pt.total()];
MatUtils.copyFromMat <float>(pt, pt_float);
int pt_cols = pt.cols();
float[] S_float = new float[S.total()];
MatUtils.copyFromMat <float>(S, S_float);
int S_cols = S.cols();
float[] A_float = new float[2 * 3];
Point[] pts = apply_simil(Si, points);
for (int i = 0; i < n; i++)
{
Size wsize = new Size(ssize.width + patches[i].patch_size().width, ssize.height + patches[i].patch_size().height);
using (Mat A = new Mat(2, 3, CvType.CV_32F))
{
MatUtils.copyFromMat <float>(A, A_float);
int A_cols = A.cols();
A_float[0] = S_float[0];
A_float[1] = S_float[1];
A_float[1 * A_cols] = S_float[1 * S_cols];
A_float[(1 * A_cols) + 1] = S_float[(1 * S_cols) + 1];
A_float[2] = (float)(pt_float[(2 * pt_cols) * i] -
(A_float[0] * (wsize.width - 1) / 2 + A_float[1] * (wsize.height - 1) / 2));
A_float[(1 * A_cols) + 2] = (float)(pt_float[((2 * pt_cols) * i) + 1] -
(A_float[1 * A_cols] * (wsize.width - 1) / 2 + A_float[(1 * A_cols) + 1] * (wsize.height - 1) / 2));
MatUtils.copyToMat(A_float, A);
using (Mat I = new Mat())
{
Imgproc.warpAffine(im, I, A, wsize, Imgproc.INTER_LINEAR + Imgproc.WARP_INVERSE_MAP);
using (Mat R = patches[i].calc_response(I, false))
{
Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(R);
pts[i].x = pts[i].x + minMaxLocResult.maxLoc.x - 0.5 * ssize.width;
pts[i].y = pts[i].y + minMaxLocResult.maxLoc.y - 0.5 * ssize.height;
}
}
}
}
return(apply_simil(S, pts));
}
}
void DoProcess()
{
if (!(mat.Value is OpenCVForUnityPlayMakerActions.Mat))
{
LogError("mat is not initialized. Add Action \"newMat\".");
return;
}
OpenCVForUnity.CoreModule.Mat wrapped_mat = OpenCVForUnityPlayMakerActionsUtils.GetWrappedObject <OpenCVForUnityPlayMakerActions.Mat, OpenCVForUnity.CoreModule.Mat>(mat);
System.Byte[] wrapped_array = OpenCVForUnityPlayMakerActionsUtils.GetWrappedObject <OpenCVForUnityPlayMakerActions.ByteArray, System.Byte[]>(array);
MatUtils.copyFromMat <byte>(wrapped_mat, wrapped_array);
}
public void calc_params(Point[] pts, Mat weight, float c_factor)
{
int n = pts.Length;
//assert(V.rows == 2*n);
//Debug.Log ("V.rows == 2*n " + V.rows () + " " + 2 * n);
using (Mat s = (new MatOfPoint2f(pts)).reshape(1, 2 * n))
{ //point set to vector format
if (weight.total() == 0)
{
Core.gemm(V.t(), s, 1, new Mat(), 0, p); //simple projection
}
else
{
//scaled projection
if (weight.rows() != n)
{
Debug.Log("Invalid weighting matrix");
}
float[] weight_float = new float[weight.total()];
MatUtils.copyFromMat <float>(weight, weight_float);
int weight_cols = weight.cols();
int K = V.cols();
using (Mat H = Mat.zeros(K, K, CvType.CV_32F)) using (Mat g = Mat.zeros(K, 1, CvType.CV_32F))
{
for (int i = 0; i < n; i++)
{
using (Mat v = new Mat(V, new OpenCVForUnity.CoreModule.Rect(0, 2 * i, K, 2))) using (Mat tmpMat1 = new Mat()) using (Mat tmpMat2 = new Mat()) using (Mat tmpMat3 = new Mat())
{
float w = weight_float[i * weight_cols];
Core.multiply(v.t(), new Scalar(w), tmpMat1);
Core.gemm(tmpMat1, v, 1, new Mat(), 0, tmpMat2);
Core.add(H, tmpMat2, H);
Core.gemm(tmpMat1, new MatOfPoint2f(pts[i]).reshape(1, 2), 1, new Mat(), 0, tmpMat3);
Core.add(g, tmpMat3, g);
}
}
Core.solve(H, g, p, Core.DECOMP_SVD);
}
}
}
clamp(c_factor); //clamp resulting parameters
}
public Point[] getConnections()
{
Point[] c = new Point[smodel.C.rows()];
int[] data = new int[c.Length * 2];
MatUtils.copyFromMat <int> (smodel.C, data);
int len = c.Length;
for (int i = 0; i < len; i++)
{
c [i] = new Point(data [i * 2], data [(i * 2) + 1]);
}
return(c);
}
Point[] apply_simil(Mat S, Point[] points)
{
float[] S_float = new float[S.total()];
MatUtils.copyFromMat <float> (S, S_float);
int S_cols = S.cols();
int n = points.Length;
Point[] p = new Point[n];
for (int i = 0; i < n; i++)
{
p [i] = new Point();
p [i].x = S_float [0] * points [i].x + S_float [1] * points [i].y + S_float [2];
p [i].y = S_float [1 * S_cols] * points [i].x + S_float [(1 * S_cols) + 1] * points [i].y + S_float [(1 * S_cols) + 2];
}
return(p);
}
void DoProcess()
{
if (!(mat.Value is OpenCVForUnityPlayMakerActions.Mat))
{
LogError("mat is not initialized. Add Action \"newMat\".");
return;
}
OpenCVForUnity.CoreModule.Mat wrapped_mat = OpenCVForUnityPlayMakerActionsUtils.GetWrappedObject <OpenCVForUnityPlayMakerActions.Mat, OpenCVForUnity.CoreModule.Mat>(mat);
int[] tmpArray = new int[array.Length];
MatUtils.copyFromMat <int>(wrapped_mat, tmpArray);
for (int i = 0; i < tmpArray.Length; i++)
{
array.Set(i, (float)tmpArray[i]);
}
array.SaveChanges();
}
public Point[] calc_shape()
{
using (Mat s = new Mat()) {
Core.gemm(V, p, 1, new Mat(), 0, s);
float[] s_float = new float[s.total()];
MatUtils.copyFromMat <float> (s, s_float);
int s_cols = s.cols();
int n = s.rows() / 2;
Point[] pts = new Point[n];
for (int i = 0; i < n; i++)
{
pts [i] = new Point(s_float [(2 * s_cols) * i], s_float [((2 * s_cols) * i) + 1]);
}
return(pts);
}
}
Mat inv_simil(Mat S)
{
float[] S_float = new float[S.total()];
MatUtils.copyFromMat <float> (S, S_float);
int S_cols = S.cols();
Mat Si = new Mat(2, 3, CvType.CV_32F);
float d = S_float [0] * S_float [(1 * S_cols) + 1] - S_float [1 * S_cols] * S_float [1];
float[] Si_float = new float[Si.total()];
MatUtils.copyFromMat <float> (Si, Si_float);
int Si_cols = Si.cols();
Si_float [0] = S_float [(1 * S_cols) + 1] / d;
Si_float [1] = -S_float [1] / d;
Si_float [(1 * Si_cols) + 1] = S_float [0] / d;
Si_float [1 * Si_cols] = -S_float [1 * S_cols] / d;
MatUtils.copyToMat(Si_float, Si);
Mat Ri = new Mat(Si, new OpenCVForUnity.CoreModule.Rect(0, 0, 2, 2));
Mat negaRi = new Mat();
Core.multiply(Ri, new Scalar(-1), negaRi);
Mat t = new Mat();
Core.gemm(negaRi, S.col(2), 1, new Mat(negaRi.rows(), negaRi.cols(), negaRi.type()), 0, t);
Mat St = Si.col(2);
t.copyTo(St);
return(Si);
}
// Match histograms of 'src' to that of 'dst', according to both masks.
public static void CalculateLUT(Mat src, Mat dst, Mat src_mask, Mat dst_mask, Texture2D LUTTex)
{
if (src.channels() < 3)
{
throw new ArgumentException("src.channels() < 3");
}
if (dst.channels() < 3)
{
throw new ArgumentException("dst.channels() < 3");
}
if (src_mask.channels() != 1)
{
throw new ArgumentException("src_mask.channels() != 1");
}
if (dst_mask.channels() != 1)
{
throw new ArgumentException("dst_mask.channels() != 1");
}
if (src_mask != null && src.total() != src_mask.total())
{
throw new ArgumentException("src.total() != src_mask.total()");
}
if (dst_mask != null && dst.total() != dst_mask.total())
{
throw new ArgumentException("dst.total() != dst_mask.total()");
}
if (LUTTex.width != 256 || LUTTex.height != 1 || LUTTex.format != TextureFormat.RGB24)
{
throw new ArgumentException("Invalid LUTTex.");
}
byte[] LUT = new byte[3 * 256];
double[][] src_hist = new double[3][];
for (int i = 0; i < src_hist.Length; i++)
{
src_hist [i] = new double[256];
}
double[][] dst_hist = new double[3][];
for (int i = 0; i < dst_hist.Length; i++)
{
dst_hist [i] = new double[256];
}
double[][] src_cdf = new double[3][];
for (int i = 0; i < src_cdf.Length; i++)
{
src_cdf [i] = new double[256];
}
double[][] dst_cdf = new double[3][];
for (int i = 0; i < dst_cdf.Length; i++)
{
dst_cdf [i] = new double[256];
}
double[] src_histMax = new double[3];
double[] dst_histMax = new double[3];
byte[] src_mask_byte = null;
byte[] dst_mask_byte = null;
if (src_mask != null)
{
src_mask_byte = new byte[src_mask.total() * src_mask.channels()];
MatUtils.copyFromMat <byte> (src_mask, src_mask_byte);
}
if (dst_mask != null)
{
dst_mask_byte = new byte[dst_mask.total() * dst_mask.channels()];
MatUtils.copyFromMat <byte> (dst_mask, dst_mask_byte);
}
byte[] src_byte = new byte[src.total() * src.channels()];
MatUtils.copyFromMat <byte> (src, src_byte);
byte[] dst_byte = new byte[dst.total() * dst.channels()];
MatUtils.copyFromMat <byte> (dst, dst_byte);
int pixel_i = 0;
int channels = src.channels();
int total = (int)src.total();
if (src_mask_byte != null)
{
for (int i = 0; i < total; i++)
{
if (src_mask_byte [i] != 0)
{
byte c = src_byte [pixel_i];
src_hist [0] [c]++;
if (src_hist [0] [c] > src_histMax [0])
{
src_histMax [0] = src_hist [0] [c];
}
c = src_byte [pixel_i + 1];
src_hist [1] [c]++;
if (src_hist [1] [c] > src_histMax [1])
{
src_histMax [1] = src_hist [1] [c];
}
c = src_byte [pixel_i + 2];
src_hist [2] [c]++;
if (src_hist [2] [c] > src_histMax [2])
{
src_histMax [2] = src_hist [2] [c];
}
}
// Advance to next pixel
pixel_i += channels;
}
}
else
{
for (int i = 0; i < total; i++)
{
byte c = src_byte [pixel_i];
src_hist [0] [c]++;
if (src_hist [0] [c] > src_histMax [0])
{
src_histMax [0] = src_hist [0] [c];
}
c = src_byte [pixel_i + 1];
src_hist [1] [c]++;
if (src_hist [1] [c] > src_histMax [1])
{
src_histMax [1] = src_hist [1] [c];
}
c = src_byte [pixel_i + 2];
src_hist [2] [c]++;
if (src_hist [2] [c] > src_histMax [2])
{
src_histMax [2] = src_hist [2] [c];
}
// Advance to next pixel
pixel_i += channels;
}
}
pixel_i = 0;
channels = dst.channels();
total = (int)dst.total();
if (dst_mask_byte != null)
{
for (int i = 0; i < total; i++)
{
if (dst_mask_byte [i] != 0)
{
byte c = dst_byte [pixel_i];
dst_hist [0] [c]++;
if (dst_hist [0] [c] > dst_histMax [0])
{
dst_histMax [0] = dst_hist [0] [c];
}
c = dst_byte [pixel_i + 1];
dst_hist [1] [c]++;
if (dst_hist [1] [c] > dst_histMax [1])
{
dst_histMax [1] = dst_hist [1] [c];
}
c = dst_byte [pixel_i + 2];
dst_hist [2] [c]++;
if (dst_hist [2] [c] > dst_histMax [2])
{
dst_histMax [2] = dst_hist [2] [c];
}
}
// Advance to next pixel
pixel_i += channels;
}
}
else
{
for (int i = 0; i < total; i++)
{
byte c = dst_byte [pixel_i];
dst_hist [0] [c]++;
if (dst_hist [0] [c] > dst_histMax [0])
{
dst_histMax [0] = dst_hist [0] [c];
}
c = dst_byte [pixel_i + 1];
dst_hist [1] [c]++;
if (dst_hist [1] [c] > dst_histMax [1])
{
dst_histMax [1] = dst_hist [1] [c];
}
c = dst_byte [pixel_i + 2];
dst_hist [2] [c]++;
if (dst_hist [2] [c] > dst_histMax [2])
{
dst_histMax [2] = dst_hist [2] [c];
}
// Advance to next pixel
pixel_i += channels;
}
}
//normalize hist
for (int i = 0; i < 256; i++)
{
src_hist [0] [i] /= src_histMax [0];
src_hist [1] [i] /= src_histMax [1];
src_hist [2] [i] /= src_histMax [2];
dst_hist [0] [i] /= dst_histMax [0];
dst_hist [1] [i] /= dst_histMax [1];
dst_hist [2] [i] /= dst_histMax [2];
}
// Calc cumulative distribution function (CDF)
src_cdf [0] [0] = src_hist [0] [0];
src_cdf [1] [0] = src_hist [1] [0];
src_cdf [2] [0] = src_hist [2] [0];
dst_cdf [0] [0] = dst_hist [0] [0];
dst_cdf [1] [0] = dst_hist [1] [0];
dst_cdf [2] [0] = dst_hist [2] [0];
for (int i = 1; i < 256; i++)
{
src_cdf [0] [i] = src_cdf [0] [i - 1] + src_hist [0] [i];
src_cdf [1] [i] = src_cdf [1] [i - 1] + src_hist [1] [i];
src_cdf [2] [i] = src_cdf [2] [i - 1] + src_hist [2] [i];
dst_cdf [0] [i] = dst_cdf [0] [i - 1] + dst_hist [0] [i];
dst_cdf [1] [i] = dst_cdf [1] [i - 1] + dst_hist [1] [i];
dst_cdf [2] [i] = dst_cdf [2] [i - 1] + dst_hist [2] [i];
}
// Normalize CDF
for (int i = 0; i < 256; i++)
{
src_cdf [0] [i] /= src_cdf [0] [255];
src_cdf [1] [i] /= src_cdf [1] [255];
src_cdf [2] [i] /= src_cdf [2] [255];
dst_cdf [0] [i] /= dst_cdf [0] [255];
dst_cdf [1] [i] /= dst_cdf [1] [255];
dst_cdf [2] [i] /= dst_cdf [2] [255];
}
// Create lookup table
const double HISTMATCH_EPSILON = 0.000001f;
for (int i = 0; i < 3; i++)
{
int last = 0;
for (int j = 0; j < 256; j++)
{
double F1j = src_cdf [i] [j];
for (int k = last; k < 256; k++)
{
double F2k = dst_cdf [i] [k];
if (Math.Abs(F2k - F1j) < HISTMATCH_EPSILON || F2k > F1j)
{
LUT [(j * 3) + i] = (byte)k;
last = k;
break;
}
}
}
}
LUTTex.LoadRawTextureData(LUT);
LUTTex.Apply(false);
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame() && !imageOptimizationHelper.IsCurrentFrameSkipped())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//get downScaleMat;
Mat downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat((rgbaMat));
//grayscale
Imgproc.cvtColor(downScaleRgbaMat, gray1Mat, Imgproc.COLOR_RGBA2GRAY);
//blur
Imgproc.blur(gray1Mat, gray2Mat, new Size(5, 5));
//edge filter
Imgproc.filter2D(gray2Mat, gray1Mat, gray1Mat.depth(), kernel);
//blur
Imgproc.blur(gray1Mat, gray2Mat, new Size(3, 3));
//detect edge
Imgproc.threshold(gray2Mat, gray2Mat, EDGE_DETECT_VALUE, 255, Imgproc.THRESH_BINARY);
//copy Mat to byteArray
MatUtils.copyFromMat <byte>(gray2Mat, byteArray);
//set edge pointList
List <Point> pointList = new List <Point>();
int w = gray1Mat.width();
int h = gray1Mat.height();
for (int y = 0; y < h; y++)
{
for (int x = 0; x < w; x++)
{
if (byteArray[x + w * y] == 255)
{
pointList.Add(new Point(x, y));
}
}
}
int limit = Mathf.RoundToInt((float)(pointList.Count * POINT_RATE));
if (limit > POINT_MAX_NUM)
{
limit = POINT_MAX_NUM;
}
while (pointList.Count > limit)
{
pointList.RemoveAt(Random.Range(0, pointList.Count));
}
//Debug.Log ("pointList.Count " + pointList.Count);
//init subdiv
subdiv.initDelaunay(new OpenCVForUnity.CoreModule.Rect(0, 0, downScaleRgbaMat.width(), downScaleRgbaMat.height()));
for (int i = 0; i < pointList.Count; i++)
{
subdiv.insert(pointList[i]);
}
subdiv.insert(new Point(0, 0));
subdiv.insert(new Point(gray1Mat.width() / 2 - 1, 0));
subdiv.insert(new Point(gray1Mat.width() - 1, 0));
subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() / 2 - 1));
subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() - 1));
subdiv.insert(new Point(gray1Mat.width() / 2 - 1, gray1Mat.height() - 1));
subdiv.insert(new Point(0, gray1Mat.height() - 1));
subdiv.insert(new Point(0, gray1Mat.height() / 2 - 1));
using (MatOfFloat6 triangleList = new MatOfFloat6())
{
subdiv.getTriangleList(triangleList);
float[] pointArray = triangleList.toArray();
float downScaleRatio = imageOptimizationHelper.downscaleRatio;
if (downScaleRatio < 1)
{
downScaleRatio = 1;
}
byte[] color = new byte[4];
for (int i = 0; i < pointArray.Length / 6; i++)
{
Point p0 = new Point(pointArray[i * 6 + 0] * downScaleRatio, pointArray[i * 6 + 1] * downScaleRatio);
Point p1 = new Point(pointArray[i * 6 + 2] * downScaleRatio, pointArray[i * 6 + 3] * downScaleRatio);
Point p2 = new Point(pointArray[i * 6 + 4] * downScaleRatio, pointArray[i * 6 + 5] * downScaleRatio);
if (p0.x < 0 || p0.x > rgbaMat.width())
{
continue;
}
if (p0.y < 0 || p0.y > rgbaMat.height())
{
continue;
}
if (p1.x < 0 || p1.x > rgbaMat.width())
{
continue;
}
if (p1.y < 0 || p1.y > rgbaMat.height())
{
continue;
}
if (p2.x < 0 || p2.x > rgbaMat.width())
{
continue;
}
if (p2.y < 0 || p2.y > rgbaMat.height())
{
continue;
}
//get center of gravity
int cx = (int)((p0.x + p1.x + p2.x) * 0.33333);
int cy = (int)((p0.y + p1.y + p2.y) * 0.33333);
// Debug.Log ("cx " + cx + " cy " + cy );
//get center of gravity color
rgbaMat.get(cy, cx, color);
//Debug.Log ("r " + color[0] + " g " + color[1] + " b " + color[2] + " a " + color[3]);
//fill Polygon
Imgproc.fillConvexPoly(rgbaMat, new MatOfPoint(p0, p1, p2), new Scalar(color[0], color[1], color[2], color[3]), Imgproc.LINE_AA, 0);
//Imgproc.line (rgbaMat, p0, p1, new Scalar (64, 255, 128, 255));
//Imgproc.line (rgbaMat, p1, p2, new Scalar (64, 255, 128, 255));
//Imgproc.line (rgbaMat, p2, p0, new Scalar (64, 255, 128, 255));
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " DOWNSCALE W:" + downScaleRgbaMat.width () + " H:" + downScaleRgbaMat.height (), new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (applyComicFilter)
{
comicFilter.Process(rgbaMat, rgbaMat);
}
if (isVideoRecording && !isFinishWriting)
{
textPos.x = 5;
textPos.y = rgbaMat.rows() - 70;
Imgproc.putText(rgbaMat, exampleTitle, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false);
textPos.y = rgbaMat.rows() - 50;
Imgproc.putText(rgbaMat, exampleSceneTitle, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false);
if (container == ContainerPreset.MP4 || container == ContainerPreset.HEVC)
{
textPos.y = rgbaMat.rows() - 30;
Imgproc.putText(rgbaMat, settingInfo1, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false);
textPos.y = rgbaMat.rows() - 10;
Imgproc.putText(rgbaMat, settingInfo2, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false);
}
else if (container == ContainerPreset.GIF)
{
textPos.y = rgbaMat.rows() - 30;
Imgproc.putText(rgbaMat, settingInfoGIF, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false);
}
else if (container == ContainerPreset.JPG)
{
textPos.y = rgbaMat.rows() - 30;
Imgproc.putText(rgbaMat, settingInfoJPG, textPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, textColor, 1, Imgproc.LINE_AA, false);
}
}
// Upload the image Mat data to the Texture2D.
// (The internal processing of the fastMatToTexture method restore the image Mat data to Unity coordinate system)
Utils.fastMatToTexture2D(rgbaMat, texture);
// Record frames
if (videoRecorder != null && (isVideoRecording && !isFinishWriting) && frameCount++ % recordEveryNthFrame == 0)
{
#if !OPENCV_USE_UNSAFE_CODE
MatUtils.copyFromMat(rgbaMat, pixelBuffer);
videoRecorder.CommitFrame(pixelBuffer, recordingClock.timestamp);
#else
unsafe
{
videoRecorder.CommitFrame((void *)rgbaMat.dataAddr(), recordingClock.timestamp);
}
#endif
}
}
if (isVideoPlaying && videoPlayer.isPlaying)
{
gameObject.GetComponent <Renderer>().sharedMaterial.mainTexture = videoPlayer.texture;
}
}
// Histogram Equalize seperately for the left and right sides of the face.
private static void equalizeLeftAndRightHalves(Mat faceImg)
{
// It is common that there is stronger light from one half of the face than the other. In that case,
// if you simply did histogram equalization on the whole face then it would make one half dark and
// one half bright. So we will do histogram equalization separately on each face half, so they will
// both look similar on average. But this would cause a sharp edge in the middle of the face, because
// the left half and right half would be suddenly different. So we also histogram equalize the whole
// image, and in the middle part we blend the 3 images together for a smooth brightness transition.
int w = faceImg.cols();
int h = faceImg.rows();
// 1) First, equalize the whole face.
using (Mat wholeFace = new Mat(h, w, CvType.CV_8UC1))
{
Imgproc.equalizeHist(faceImg, wholeFace);
// 2) Equalize the left half and the right half of the face separately.
int midX = w / 2;
using (Mat leftSide = new Mat(faceImg, new Rect(0, 0, midX, h)))
using (Mat rightSide = new Mat(faceImg, new Rect(midX, 0, w - midX, h)))
{
Imgproc.equalizeHist(leftSide, leftSide);
Imgproc.equalizeHist(rightSide, rightSide);
// 3) Combine the left half and right half and whole face together, so that it has a smooth transition.
byte[] wholeFace_byte = new byte[wholeFace.total() * wholeFace.elemSize()];
MatUtils.copyFromMat <byte>(wholeFace, wholeFace_byte);
byte[] leftSide_byte = new byte[leftSide.total() * leftSide.elemSize()];
MatUtils.copyFromMat <byte>(leftSide, leftSide_byte);
byte[] rightSide_byte = new byte[rightSide.total() * rightSide.elemSize()];
MatUtils.copyFromMat <byte>(rightSide, rightSide_byte);
int leftSide_w = leftSide.cols();
int rightSide_w = rightSide.cols();
for (int y = 0; y < h; y++)
{
for (int x = 0; x < w; x++)
{
byte wv = wholeFace_byte[y * w + x];
if (x < w / 4)
{ // Left 25%: just use the left face.
wv = leftSide_byte[y * leftSide_w + x];
}
else if (x < w * 2 / 4)
{ // Mid-left 25%: blend the left face & whole face.
byte lv = leftSide_byte[y * leftSide_w + x];
// Blend more of the whole face as it moves further right along the face.
float f = (x - w * 1 / 4) / (w * 0.25f);
wv = (byte)Mathf.Round((1.0f - f) * lv + f * wv);
}
else if (x < w * 3 / 4)
{ // Mid-right 25%: blend the right face & whole face.
byte rv = rightSide_byte[y * rightSide_w + x - midX];
// Blend more of the right-side face as it moves further right along the face.
float f = (x - w * 2 / 4) / (w * 0.25f);
wv = (byte)Mathf.Round((1.0f - f) * wv + f * rv);
}
else
{ // Right 25%: just use the right face.
wv = rightSide_byte[y * rightSide_w + x - midX];
}
} // end x loop
} //end y loop
MatUtils.copyToMat(wholeFace_byte, faceImg);
}
}
}
protected override void postprocess(Mat frame, List <Mat> outs, Net net, int backend = Dnn.DNN_BACKEND_OPENCV)
{
List <int> classIdsList = new List <int>();
List <float> confidencesList = new List <float>();
List <Rect2d> boxesList = new List <Rect2d>();
List <Point[]> pointsList = new List <Point[]>();
if (outs.Count == 2)
{
// reshape mat : outs[0]:[1, x, 4] to [x, 4], outs[1]:[1, x, 2] to [x, 2]
Mat boxes_m = outs[0].reshape(1, new int[] { outs[0].size(1), outs[0].size(2) });
Mat scores_m = outs[1].reshape(1, new int[] { outs[1].size(1), outs[1].size(2) });
//Debug.Log("boxes_m: " + boxes_m);
//Debug.Log("scores_m: " + scores_m);
//Debug.Log("priors: " + priors);
convertLocationsToBoxes(boxes_m, priors, 0.1f, 0.2f);
centerFormToCornerForm(boxes_m);
Mat boxes_0_4 = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(0, 4));
float[] boxes_arr = new float[boxes_0_4.rows() * boxes_0_4.cols()];
MatUtils.copyFromMat(boxes_0_4, boxes_arr);
Mat scores_1_2 = new Mat(scores_m, new Range(0, scores_m.rows()), new Range(1, 2));
float[] confidences_arr = new float[scores_1_2.rows()];
MatUtils.copyFromMat(scores_1_2, confidences_arr);
for (int i = 0; i < boxes_m.rows(); i++)
{
float confidence = confidences_arr[i];
if (confidence > confThreshold)
{
int boxes_index = i * 4;
float left = boxes_arr[boxes_index] * frame.cols();
float top = boxes_arr[boxes_index + 1] * frame.rows();
float right = boxes_arr[boxes_index + 2] * frame.cols();
float bottom = boxes_arr[boxes_index + 3] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add(0);
confidencesList.Add(confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
if (boxes_m.cols() > 4 && boxes_m.cols() % 2 == 0)
{
Mat points = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(4, boxes_m.cols()));
float[] points_arr = new float[points.rows() * points.cols()];
MatUtils.copyFromMat(points, points_arr);
for (int i = 0; i < boxes_m.rows(); i++)
{
float confidence = confidences_arr[i];
if (confidence > confThreshold)
{
int points_index = i * points.cols();
Point[] p_arr = new Point[points.cols() / 2];
for (int index = 0; index < points.cols() / 2; index++)
{
float x = points_arr[points_index + index * 2] * frame.cols();
float y = points_arr[points_index + index * 2 + 1] * frame.rows();
p_arr[index] = new Point(x, y);
}
pointsList.Add(p_arr);
}
}
}
}
MatOfRect2d boxes = new MatOfRect2d();
boxes.fromList(boxesList);
MatOfFloat confidences = new MatOfFloat();
confidences.fromList(confidencesList);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
//Debug.Log("indices.dump () " + indices.dump());
//Debug.Log ("indices.ToString () "+indices.ToString());
for (int i = 0; i < indices.total(); ++i)
{
int idx = (int)indices.get(i, 0)[0];
Rect2d box = boxesList[idx];
drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y,
box.x + box.width, box.y + box.height, frame);
if (pointsList.Count > 0)
{
drawPredPoints(pointsList[idx], frame);
}
}
indices.Dispose();
boxes.Dispose();
confidences.Dispose();
}
private Point[] fit(Mat image,
Point[] init,
Size ssize,
bool robust,
int itol,
double ftol)
{
int n = smodel.npts();
// assert((int(init.size())==n) && (pmodel.n_patches()==n));
// Debug.Log ("init.size())==n " + init.Length + " " + n);
// Debug.Log ("pmodel.n_patches()==n " + pmodel.n_patches () + " " + n);
smodel.calc_params(init, new Mat(), 3.0f);
Point[] pts = smodel.calc_shape();
//find facial features in image around current estimates
Point[] peaks = pmodel.calc_peaks(image, pts, ssize);
//optimise
if (!robust)
{
smodel.calc_params(peaks, new Mat(), 3.0f); //compute shape model parameters
pts = smodel.calc_shape(); //update shape
}
else
{
using (Mat weight = new Mat(n, 1, CvType.CV_32F)) using (Mat weight_sort = new Mat(n, 1, CvType.CV_32F)) {
float[] weight_float = new float[weight.total()];
MatUtils.copyFromMat <float> (weight, weight_float);
float[] weight_sort_float = new float[weight_sort.total()];
Point[] pts_old = pts;
for (int iter = 0; iter < itol; iter++)
{
//compute robust weight
for (int i = 0; i < n; i++)
{
using (MatOfPoint tmpMat = new MatOfPoint(new Point(pts [i].x - peaks [i].x, pts [i].y - peaks [i].y))) {
weight_float [i] = (float)Core.norm(tmpMat);
}
}
MatUtils.copyToMat(weight_float, weight);
Core.sort(weight, weight_sort, Core.SORT_EVERY_COLUMN | Core.SORT_ASCENDING);
MatUtils.copyFromMat <float> (weight_sort, weight_sort_float);
double var = 1.4826 * weight_sort_float [n / 2];
if (var < 0.1)
{
var = 0.1;
}
Core.pow(weight, 2, weight);
Core.multiply(weight, new Scalar(-0.5 / (var * var)), weight);
Core.exp(weight, weight);
//compute shape model parameters
smodel.calc_params(peaks, weight, 3.0f);
//update shape
pts = smodel.calc_shape();
//check for convergence
float v = 0;
for (int i = 0; i < n; i++)
{
using (MatOfPoint tmpMat = new MatOfPoint(new Point(pts [i].x - pts_old [i].x, pts [i].y - pts_old [i].y))) {
v += (float)Core.norm(tmpMat);
}
}
if (v < ftol)
{
break;
}
else
{
pts_old = pts;
}
}
}
}
return(pts);
}
// Calculates source image histogram and changes target_image to match source hist.
private void specifyHistogram(Mat source_image, Mat target_image, Mat mask)
{
byte[][] LUT = new byte[3][];
for (int i = 0; i < LUT.Length; i++)
{
LUT[i] = new byte[256];
}
double[][] source_hist = new double[3][];
for (int i = 0; i < source_hist.Length; i++)
{
source_hist[i] = new double[256];
}
double[][] target_hist = new double[3][];
for (int i = 0; i < target_hist.Length; i++)
{
target_hist[i] = new double[256];
}
double[][] source_cdf = new double[3][];
for (int i = 0; i < source_cdf.Length; i++)
{
source_cdf[i] = new double[256];
}
double[][] target_cdf = new double[3][];
for (int i = 0; i < target_cdf.Length; i++)
{
target_cdf[i] = new double[256];
}
double[] source_histMax = new double[3];
double[] target_histMax = new double[3];
byte[] mask_byte = new byte[mask.total() * mask.channels()];
MatUtils.copyFromMat <byte>(mask, mask_byte);
byte[] source_image_byte = new byte[source_image.total() * source_image.channels()];
MatUtils.copyFromMat <byte>(source_image, source_image_byte);
byte[] target_image_byte = new byte[target_image.total() * target_image.channels()];
MatUtils.copyFromMat <byte>(target_image, target_image_byte);
int pixel_i = 0;
int channels = (int)source_image.channels();
int total = (int)source_image.total();
for (int i = 0; i < total; i++)
{
if (mask_byte[i] != 0)
{
byte c = source_image_byte[pixel_i];
source_hist[0][c]++;
if (source_hist[0][c] > source_histMax[0])
{
source_histMax[0] = source_hist[0][c];
}
c = source_image_byte[pixel_i + 1];
source_hist[1][c]++;
if (source_hist[1][c] > source_histMax[1])
{
source_histMax[1] = source_hist[1][c];
}
c = source_image_byte[pixel_i + 2];
source_hist[2][c]++;
if (source_hist[2][c] > source_histMax[2])
{
source_histMax[2] = source_hist[2][c];
}
c = target_image_byte[pixel_i];
target_hist[0][c]++;
if (target_hist[0][c] > target_histMax[0])
{
target_histMax[0] = target_hist[0][c];
}
c = target_image_byte[pixel_i + 1];
target_hist[1][c]++;
if (target_hist[1][c] > target_histMax[1])
{
target_histMax[1] = target_hist[1][c];
}
c = target_image_byte[pixel_i + 2];
target_hist[2][c]++;
if (target_hist[2][c] > target_histMax[2])
{
target_histMax[2] = target_hist[2][c];
}
}
// Advance to next pixel
pixel_i += channels;
}
// Normalize hist
for (int i = 0; i < 256; i++)
{
source_hist[0][i] /= source_histMax[0];
source_hist[1][i] /= source_histMax[1];
source_hist[2][i] /= source_histMax[2];
target_hist[0][i] /= target_histMax[0];
target_hist[1][i] /= target_histMax[1];
target_hist[2][i] /= target_histMax[2];
}
// Calc cumulative distribution function (CDF)
source_cdf[0][0] = source_hist[0][0];
source_cdf[1][0] = source_hist[1][0];
source_cdf[2][0] = source_hist[2][0];
target_cdf[0][0] = target_hist[0][0];
target_cdf[1][0] = target_hist[1][0];
target_cdf[2][0] = target_hist[2][0];
for (int i = 1; i < 256; i++)
{
source_cdf[0][i] = source_cdf[0][i - 1] + source_hist[0][i];
source_cdf[1][i] = source_cdf[1][i - 1] + source_hist[1][i];
source_cdf[2][i] = source_cdf[2][i - 1] + source_hist[2][i];
target_cdf[0][i] = target_cdf[0][i - 1] + target_hist[0][i];
target_cdf[1][i] = target_cdf[1][i - 1] + target_hist[1][i];
target_cdf[2][i] = target_cdf[2][i - 1] + target_hist[2][i];
}
// Normalize CDF
for (int i = 0; i < 256; i++)
{
source_cdf[0][i] /= source_cdf[0][255];
source_cdf[1][i] /= source_cdf[1][255];
source_cdf[2][i] /= source_cdf[2][255];
target_cdf[0][i] /= target_cdf[0][255];
target_cdf[1][i] /= target_cdf[1][255];
target_cdf[2][i] /= target_cdf[2][255];
}
// Create lookup table (LUT)
const double HISTMATCH_EPSILON = 0.000001f;
for (int i = 0; i < 3; i++)
{
int last = 0;
for (int j = 0; j < 256; j++)
{
double F1j = target_cdf[i][j];
for (int k = last; k < 256; k++)
{
double F2k = source_cdf[i][k];
if (Math.Abs(F2k - F1j) < HISTMATCH_EPSILON || F2k > F1j)
{
LUT[i][j] = (byte)k;
last = k;
break;
}
}
}
}
// Repaint pixels
pixel_i = 0;
for (int i = 0; i < total; i++)
{
if (mask_byte[i] != 0)
{
target_image_byte[pixel_i] = LUT[0][target_image_byte[pixel_i]];
target_image_byte[pixel_i + 1] = LUT[1][target_image_byte[pixel_i + 1]];
target_image_byte[pixel_i + 2] = LUT[2][target_image_byte[pixel_i + 2]];
}
// Advance to next pixel
pixel_i += channels;
}
MatUtils.copyToMat(target_image_byte, target_image);
}
