// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame() && !imageOptimizationHelper.IsCurrentFrameSkipped())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//get downScaleMat;
Mat downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat((rgbaMat));
//grayscale
Imgproc.cvtColor(downScaleRgbaMat, gray1Mat, Imgproc.COLOR_RGBA2GRAY);
//blur
Imgproc.blur(gray1Mat, gray2Mat, new Size(5, 5));
//edge filter
Imgproc.filter2D(gray2Mat, gray1Mat, gray1Mat.depth(), kernel);
//blur
Imgproc.blur(gray1Mat, gray2Mat, new Size(3, 3));
//detect edge
Imgproc.threshold(gray2Mat, gray2Mat, EDGE_DETECT_VALUE, 255, Imgproc.THRESH_BINARY);
//copy Mat to byteArray
Utils.copyFromMat <byte> (gray2Mat, byteArray);
//set edge pointList
List <Point> pointList = new List <Point> ();
int w = gray1Mat.width();
int h = gray1Mat.height();
for (int y = 0; y < h; y++)
{
for (int x = 0; x < w; x++)
{
if (byteArray [x + w * y] == 255)
{
pointList.Add(new Point(x, y));
}
}
}
int limit = Mathf.RoundToInt((float)(pointList.Count * POINT_RATE));
if (limit > POINT_MAX_NUM)
{
limit = POINT_MAX_NUM;
}
while (pointList.Count > limit)
{
pointList.RemoveAt(Random.Range(0, pointList.Count));
}
// Debug.Log ("pointList.Count " + pointList.Count);
//init subdiv
subdiv.initDelaunay(new OpenCVForUnity.CoreModule.Rect(0, 0, downScaleRgbaMat.width(), downScaleRgbaMat.height()));
for (int i = 0; i < pointList.Count; i++)
{
subdiv.insert(pointList [i]);
}
subdiv.insert(new Point(0, 0));
subdiv.insert(new Point(gray1Mat.width() / 2 - 1, 0));
subdiv.insert(new Point(gray1Mat.width() - 1, 0));
subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() / 2 - 1));
subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() - 1));
subdiv.insert(new Point(gray1Mat.width() / 2 - 1, gray1Mat.height() - 1));
subdiv.insert(new Point(0, gray1Mat.height() - 1));
subdiv.insert(new Point(0, gray1Mat.height() / 2 - 1));
using (MatOfFloat6 triangleList = new MatOfFloat6()) {
subdiv.getTriangleList(triangleList);
float[] pointArray = triangleList.toArray();
float downScaleRatio = imageOptimizationHelper.downscaleRatio;
if (downScaleRatio < 1)
{
downScaleRatio = 1;
}
byte[] color = new byte[4];
for (int i = 0; i < pointArray.Length / 6; i++)
{
Point p0 = new Point(pointArray [i * 6 + 0] * downScaleRatio, pointArray [i * 6 + 1] * downScaleRatio);
Point p1 = new Point(pointArray [i * 6 + 2] * downScaleRatio, pointArray [i * 6 + 3] * downScaleRatio);
Point p2 = new Point(pointArray [i * 6 + 4] * downScaleRatio, pointArray [i * 6 + 5] * downScaleRatio);
if (p0.x < 0 || p0.x > rgbaMat.width())
{
continue;
}
if (p0.y < 0 || p0.y > rgbaMat.height())
{
continue;
}
if (p1.x < 0 || p1.x > rgbaMat.width())
{
continue;
}
if (p1.y < 0 || p1.y > rgbaMat.height())
{
continue;
}
if (p2.x < 0 || p2.x > rgbaMat.width())
{
continue;
}
if (p2.y < 0 || p2.y > rgbaMat.height())
{
continue;
}
//get center of gravity
int cx = (int)((p0.x + p1.x + p2.x) * 0.33333);
int cy = (int)((p0.y + p1.y + p2.y) * 0.33333);
// Debug.Log ("cx " + cx + " cy " + cy );
//get center of gravity color
rgbaMat.get(cy, cx, color);
// Debug.Log ("r " + color[0] + " g " + color[1] + " b " + color[2] + " a " + color[3]);
//fill Polygon
Imgproc.fillConvexPoly(rgbaMat, new MatOfPoint(p0, p1, p2), new Scalar(color [0], color [1], color [2], color [3]), Imgproc.LINE_AA, 0);
// Imgproc.line (rgbaMat, p0, p1, new Scalar (64, 255, 128, 255));
// Imgproc.line (rgbaMat, p1, p2, new Scalar (64, 255, 128, 255));
// Imgproc.line (rgbaMat, p2, p0, new Scalar (64, 255, 128, 255));
}
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " DOWNSCALE W:" + downScaleRgbaMat.width () + " H:" + downScaleRgbaMat.height (), new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying () && webCamTextureToMatHelper.DidUpdateThisFrame ()) {
Mat rgbaMat = webCamTextureToMatHelper.GetMat ();
//get downScaleMat;
Mat downScaleRgbaMat = webCamTextureToMatHelper.GetDownScaleMat (rgbaMat);
//grayscale
Imgproc.cvtColor (downScaleRgbaMat, gray1Mat, Imgproc.COLOR_RGBA2GRAY);
//blur
Imgproc.blur (gray1Mat, gray2Mat, new Size (5, 5));
//edge filter
Imgproc.filter2D (gray2Mat, gray1Mat, gray1Mat.depth (), kernel);
//blur
Imgproc.blur (gray1Mat, gray2Mat, new Size (3, 3));
//detect edge
Imgproc.threshold (gray2Mat, gray2Mat, EDGE_DETECT_VALUE, 255, Imgproc.THRESH_BINARY);
//copy Mat to byteArray
Utils.copyFromMat<byte> (gray2Mat, byteArray);
//set edge pointList
List<Point> pointList = new List<Point> ();
int w = gray1Mat.width ();
int h = gray1Mat.height ();
for (int y = 0; y < h; y++) {
for (int x = 0; x < w; x++) {
if (byteArray [x + w * y] == 255)
pointList.Add (new Point (x, y));
}
}
int limit = Mathf.RoundToInt ((float)(pointList.Count * POINT_RATE));
if (limit > POINT_MAX_NUM)
limit = POINT_MAX_NUM;
while (pointList.Count > limit) {
pointList.RemoveAt (Random.Range (0, pointList.Count));
}
// Debug.Log ("pointList.Count " + pointList.Count);
//init subdiv
subdiv.initDelaunay (new OpenCVForUnity.Rect (0, 0, downScaleRgbaMat.width (), downScaleRgbaMat.height ()));
for (int i = 0; i < pointList.Count; i++) {
subdiv.insert (pointList [i]);
}
subdiv.insert (new Point (0, 0));
subdiv.insert (new Point (gray1Mat.width () / 2 - 1, 0));
subdiv.insert (new Point (gray1Mat.width () - 1, 0));
subdiv.insert (new Point (gray1Mat.width () - 1, gray1Mat.height () / 2 - 1));
subdiv.insert (new Point (gray1Mat.width () - 1, gray1Mat.height () - 1));
subdiv.insert (new Point (gray1Mat.width () / 2 - 1, gray1Mat.height () - 1));
subdiv.insert (new Point (0, gray1Mat.height () - 1));
subdiv.insert (new Point (0, gray1Mat.height () / 2 - 1));
using (MatOfFloat6 triangleList = new MatOfFloat6()) {
subdiv.getTriangleList (triangleList);
float[] pointArray = triangleList.toArray ();
int downScaleRatio = webCamTextureToMatHelper.DOWNSCALE_RATIO;
if (downScaleRatio < 1)
downScaleRatio = 1;
byte[] color = new byte[4];
for (int i = 0; i < pointArray.Length/6; i++) {
Point p0 = new Point (pointArray [i * 6 + 0] * downScaleRatio, pointArray [i * 6 + 1] * downScaleRatio);
Point p1 = new Point (pointArray [i * 6 + 2] * downScaleRatio, pointArray [i * 6 + 3] * downScaleRatio);
Point p2 = new Point (pointArray [i * 6 + 4] * downScaleRatio, pointArray [i * 6 + 5] * downScaleRatio);
if (p0.x < 0 || p0.x > rgbaMat.width ())
continue;
if (p0.y < 0 || p0.y > rgbaMat.height ())
continue;
if (p1.x < 0 || p1.x > rgbaMat.width ())
continue;
if (p1.y < 0 || p1.y > rgbaMat.height ())
continue;
if (p2.x < 0 || p2.x > rgbaMat.width ())
continue;
if (p2.y < 0 || p2.y > rgbaMat.height ())
continue;
//get center of gravity
int cx = (int)((p0.x + p1.x + p2.x) * 0.33333);
int cy = (int)((p0.y + p1.y + p2.y) * 0.33333);
// Debug.Log ("cx " + cx + " cy " + cy );
//get center of gravity color
rgbaMat.get (cy, cx, color);
// Debug.Log ("r " + color[0] + " g " + color[1] + " b " + color[2] + " a " + color[3]);
//fill Polygon
Imgproc.fillConvexPoly (rgbaMat, new MatOfPoint (p0, p1, p2), new Scalar (color [0], color [1], color [2], color [3]), Imgproc.LINE_AA, 0);
// Imgproc.line (rgbaMat, p0, p1, new Scalar (64, 255, 128, 255));
// Imgproc.line (rgbaMat, p1, p2, new Scalar (64, 255, 128, 255));
// Imgproc.line (rgbaMat, p2, p0, new Scalar (64, 255, 128, 255));
}
}
Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " DOWNSCALE W:" + downScaleRgbaMat.width () + " H:" + downScaleRgbaMat.height (), new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D (rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors ());
}
}
void TriangleDivide()
{
//init subdiv
Subdiv2D subdiv = new Subdiv2D();
OpenCVForUnity.Rect cv_rect = new OpenCVForUnity.Rect(0, 0, rgbaMat.width(), rgbaMat.height());
subdiv.initDelaunay(cv_rect);
for (int i = 0; i < pointList.Count; i++)
{
subdiv.insert(pointList[i]);
}
subdiv.insert(new Point(0, 0));
subdiv.insert(new Point(rgbaMat.width() / 4 - 1, 0)); //
subdiv.insert(new Point(rgbaMat.width() / 2 - 1, 0)); //
subdiv.insert(new Point(rgbaMat.width() / 2 - 1, rgbaMat.height() / 2 - 1)); //
subdiv.insert(new Point(rgbaMat.width() / 2 - 1, rgbaMat.height() - 1)); //
subdiv.insert(new Point(rgbaMat.width() / 4 - 1, rgbaMat.height() - 1)); //
subdiv.insert(new Point(0, rgbaMat.height() - 1));
subdiv.insert(new Point(0, rgbaMat.height() / 2 - 1));
//using (MatOfFloat6 triangleList = new MatOfFloat6())
MatOfFloat6 triangleList = new MatOfFloat6();
{
subdiv.getTriangleList(triangleList);
float[] pointArray = triangleList.toArray();
//Debug.Log(pointArray.Length); //11000+
float downScaleRatio = 1;
byte[] color = new byte[4];
for (int i = 0; i < pointArray.Length / 6; i++)
{
Point p0 = new Point(pointArray[i * 6 + 0] * downScaleRatio, pointArray[i * 6 + 1] * downScaleRatio);
Point p1 = new Point(pointArray[i * 6 + 2] * downScaleRatio, pointArray[i * 6 + 3] * downScaleRatio);
Point p2 = new Point(pointArray[i * 6 + 4] * downScaleRatio, pointArray[i * 6 + 5] * downScaleRatio);
if (p0.x < 0 || p0.x > rgbaMat.width())
{
continue;
}
if (p0.y < 0 || p0.y > rgbaMat.height())
{
continue;
}
if (p1.x < 0 || p1.x > rgbaMat.width())
{
continue;
}
if (p1.y < 0 || p1.y > rgbaMat.height())
{
continue;
}
if (p2.x < 0 || p2.x > rgbaMat.width())
{
continue;
}
if (p2.y < 0 || p2.y > rgbaMat.height())
{
continue;
}
//get center of gravity
int cx = (int)((p0.x + p1.x + p2.x) * 0.33333);
int cy = (int)((p0.y + p1.y + p2.y) * 0.33333);
//Debug.Log ("cx " + cx + " cy " + cy );
//get center of gravity color
rgbaMat.get(cy, cx, color);
//Debug.Log ("r " + color[0] + " g " + color[1] + " b " + color[2] + " a " + color[3]);
//fill Polygon
//Imgproc.fillConvexPoly(rgbaMat, new MatOfPoint(p0, p1, p2), new Scalar(color[0], color[1], color[2], color[3]), Imgproc.LINE_AA, 0);
Imgproc.line(rgbaMat, p0, p1, new Scalar(64, 255, 128, 255));
Imgproc.line(rgbaMat, p1, p2, new Scalar(64, 255, 128, 255));
Imgproc.line(rgbaMat, p2, p0, new Scalar(64, 255, 128, 255));
//--------------------------------------------------------------------//
//Debug.Log(pointArray.Length / 6); //154 //（68dlib点+周围8点 => 154个三角形）
//Point[] pt = new Point[3];
//pt[0] = new Point(t[0], t[1]);
//pt[1] = new Point(t[2], t[3]);
//pt[2] = new Point(t[4], t[5]);
//Debug.Log("[0]" + pt[0] + "[1]" + pt[1] + "[2]" + pt[2]);
Point[] pt = new Point[3] {
p0, p1, p2
};
//Debug.Log("[0]" + p0 + "[1]" + p1 + "[2]" + p2);
//Debug.Log(cv_rect); //800*400，rgba图的尺寸
if (cv_rect.contains(p0) && cv_rect.contains(p1) && cv_rect.contains(p2))
{
int count = 0;
int[] index = new int[3];
for (int j = 0; j < 3; j++)
{
for (int k = 0; k < pointList.Count; k++)
{
if (Mathf.Abs((float)pt[j].x - (float)pointList[k].x) < 1.0 && Mathf.Abs((float)pt[j].y - (float)pointList[k].y) < 1.0) //重合
{
index[j] = k;
count++;
}
}
}
if (count == 3)
{
//154中筛选109个，满足3个点重合
//delaunayTri.push_back(index);
}
}
//--------------------------------------------------------------------//
}
//subdiv.getVoronoiFacetList();
}
}
private void WarpFaces()
{
// Create the image matrices
Mat imgSrc = CreateSourceImageMat();
Mat imgDest = CreateDestImageMat(imgSrc);
// Generate the face data (in this case, load the parameters)
//var sourceFace = JsonConvert.DeserializeObject<RootObject>(sourceImageFace);
//var warpToFace = JsonConvert.DeserializeObject<RootObject>(this.warpToFace);
// Create the landmark point lists
var pointList1 = new List <Point>();
var pointList2 = new List <Point>();
AddMatchingLandmarks(sourceFace.faceLandmarks, warpToFace.faceLandmarks, pointList1, pointList2);
if (pointList1.Count != pointList2.Count)
{
Debug.LogErrorFormat("Counts are not equal. Lists ahve {0} and {1} elements.", pointList1.Count, pointList2.Count);
}
// Declare the int source width/height
int sw = (int)imgSrc.size().width;
int sh = (int)imgSrc.size().height;
int dw = warpToImage.width;
int dh = warpToImage.height;
// Create a subdivision for the source image
var sourceSubDiv = new Subdiv2D(new OpenCVForUnity.Rect(0, 0, sw, sh));
// Add border points to the subdivision
// and add all the other points
Add8PointBorder(sourceSubDiv, sw, sh);
foreach (var p in pointList1)
{
sourceSubDiv.insert(p);
}
// Create a list of triangles
// from the subdivision
MatOfFloat6 sourceTriMat = new MatOfFloat6();
sourceSubDiv.getTriangleList(sourceTriMat);
var sourceTriList = new List <float>(sourceTriMat.toArray());
// OpenCV for Unity subdiv has a bug and we need to ensure
// every triangle is inside the bounds of the source
CapToMax(sourceTriList, sw - 1, sh - 1);
// Copy it
var destTriList = new List <float>();
foreach (float f in sourceTriList)
{
destTriList.Add(f);
}
// Find the points from the source, and
// move them to the destination this way
// the triangles correlate to each other
MovePointsInList(sourceFace.faceLandmarks.FaceLandmarkList(), warpToFace.faceLandmarks.FaceLandmarkList(), destTriList);
MovePointsInList(Get8PointBorder(sw, sh), Get8PointBorder(dw, dh), destTriList);
// Now we have source triangles and positions
// and destination triangles and positions
// warp the source triangles ot the destination
WarpImages(imgSrc, imgDest, sourceTriList, destTriList);
// Everything is warped, so now create the texture from
// the matrix
Texture2D texture = MatToTexture2D(imgDest);
// Assign the texture
target.GetComponent <Renderer>().material.mainTexture = texture;
}