void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
ARGvrHead.trackRotation = true;
}
else
{
ARGvrHead.trackRotation = false;
}
}
// Update is called once per frame
void Update()
{
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
//Touch
int touchCount = Input.touchCount;
if (touchCount == 1)
{
Touch t = Input.GetTouch(0);
if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId))
{
storedTouchPoint = new Point(t.position.x, t.position.y);
//Debug.Log ("touch X " + t.position.x);
//Debug.Log ("touch Y " + t.position.y);
}
}
#else
//Mouse
#endif
timer -= Time.deltaTime;
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
HandPoseEstimationProcess(rgbaMat);
}
}
// Update is called once per frame
void Update()
{
if (Input.GetKeyDown(KeyCode.Escape))
{
#if UNITY_5_3 || UNITY_5_3_OR_NEWER
SceneManager.LoadScene("FacemojiStart");
#else
Application.LoadLevel("FacemojiStart");
#endif
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
if (points.Count > 0)
{
live2DModelUpdate(points);
//currentFacePoints = points;
break;
}
}
}
}
void Update()
{
//アプリ起動時とカメラ切替時，カメラ起動まで待つ
if (_remainingWaitingTime > 0)
{
_remainingWaitingTime -= Time.deltaTime;
return;
}
_waitingingIndicator.SetActive(false);
if (!_toMatHelper.IsPlaying() || !_toMatHelper.DidUpdateThisFrame())
{
return;
}
//背景を保存する．StartでやるとWebCamのPlayが間に合わない？
if (!_invCvtr.IsSavedBg)
{
_invCvtr.SaveBgr(_toMatHelper.GetMat());
}
//透明人間に変換して表示
var invImg = _invCvtr.CvtToInvisible(_toMatHelper.GetMat());
if (_isRecording)
{
_movieTaker?.Write(invImg);
}
Utils.fastMatToTexture2D(invImg, _quadTex);
}
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Run();
}
}
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
void Update()
{
if (_WebCamTextureToMatHelper.IsPlaying() && _WebCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = _WebCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, _GrayMat, Imgproc.COLOR_RGBA2GRAY);
_ImageDetector.FindARMarker(_GrayMat);
Utils.fastMatToTexture2D(rgbaMat, _WebCamTexture);
}
}
void Update()
{
if (!_webCamTextureToMatHelper.IsPlaying() || !_webCamTextureToMatHelper.DidUpdateThisFrame())
{
return;
}
var rgbaMat = _webCamTextureToMatHelper.GetMat();
SetFire(rgbaMat);
Utils.fastMatToTexture2D(rgbaMat, _texture);
}
IEnumerator worker()
{
bool inProcess = false;
while (true)
{
yield return(null);
if (inProcess)
{
yield return(null);
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
color_filter.Clear();
Mat t_rgbaMat = webCamTextureToMatHelper.GetMat();
tagramDetect(t_rgbaMat, (TangramResultModel trm, List <MyShape> lms) =>
{
if (trm != null)
{
if (this.handler != null)
{
handler(trm, lms, tangramFeatureModelList);
}
if (debug == true)
{
string s = "";
for (var i = 0; i < trm.datas.Length; i++)
{
s += trm.datas[i] + " ";
}
if (uitext != null && debug == true)
{
uitext.text = s;
}
}
}
});
}
}
}
// Update is called once per frame
void Update()
{
Debug.Log("called update func");
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Color32[] rgbabuf = webCamTextureToMatHelper.GetBufferColors();
if (rgbabuf != null && faceLandmarkDetector != null && texture != null)
{
Debug.Log("on Update above SetImage");
faceLandmarkDetector.SetImage <Color32> (rgbabuf, texture.width, texture.height, 4, true);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
Debug.Log("face : " + rect);
//detect landmark points
faceLandmarkDetector.DetectLandmark(rect);
//draw landmark points
faceLandmarkDetector.DrawDetectLandmarkResult <Color32> (rgbabuf, texture.width, texture.height, 4, true, 255, 255, 255, 255);
//faceLandmarkDetector.DrawDetectLandmarkResult<Color32>(drawbuf, texture.width, texture.height, 4, true, 255, 255, 255, 255);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
if (points.Count > 0)
{
live2DModelUpdate(points);
}
}
if (isHideCameraImage == false)
{
texture.SetPixels32(rgbabuf);
texture.Apply(false);
}
}
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
frame = webCamTextureToMatHelper.GetMat();
frame.copyTo(img_orig);
drawing = img_orig.clone();
int lowThreshold = 50;// (int)200;// slider.value;
const int ratio = 1;
const int kernel_size = 3;
Imgproc.cvtColor(img_orig, img_lab, Imgproc.COLOR_BGR2Lab);
double omrSize = img_orig.cols() * img_orig.rows();
Imgproc.cvtColor(img_orig, img_gray, Imgproc.COLOR_RGBA2GRAY);
Imgproc.GaussianBlur(img_gray, img_gray, new Size(15, 15), 1.5, 1.5); //Gaussian blur
Imgproc.erode(img_gray, img_gray, new Mat(), new Point(-1, -1), 1); //Erosion
// Imgproc.dilate(img_gray, img_gray, new Mat(), new Point(-1, -1), 10, 1, new Scalar(10)); //Dilation
Imgproc.Canny(img_gray, img_edges, lowThreshold, lowThreshold * ratio, kernel_size, false);
//Shape detection
List <MatOfPoint> contours = new List <MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(img_edges, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0));
//Texture2D tex = new Texture2D(img_edges.width(), img_edges.height(), TextureFormat.RGB24, false);
//Utils.matToTexture2D(img_edges, tex);
//byte[] bytes1 = tex.EncodeToJPG();
//File.WriteAllBytes("D:/2019/OMR/" + "test213123.png", bytes1);
List <MatOfPoint> hulls = new List <MatOfPoint>();
for (int i = 0; i < contours.Count; i++)
{
MatOfInt hull_temp = new MatOfInt();
Imgproc.convexHull(contours[i], hull_temp);
int[] arrIndex = hull_temp.toArray();
Point[] arrContour = contours[i].toArray();
Point[] arrPoints = new Point[arrIndex.Length];
for (int k = 0; k < arrIndex.Length; k++)
{
arrPoints[k] = arrContour[arrIndex[k]];
}
MatOfPoint temp = new MatOfPoint();
temp.fromArray(arrPoints);
//Filter outliers
if (Imgproc.contourArea(temp) > omrSize / 3 && Imgproc.contourArea(temp) < (omrSize * 4) / 5)
{
hulls.Add(temp);
}
}
List <MatOfPoint2f> hull2f = new List <MatOfPoint2f>();
for (int i = 0; i < hulls.Count; i++)
{
MatOfPoint2f newPoint = new MatOfPoint2f(hulls[i].toArray());
hull2f.Add(newPoint);
}
for (int i = 0; i < hulls.Count; i++)
{
//Approximate polygon
MatOfPoint2f approx = new MatOfPoint2f();
Imgproc.approxPolyDP(hull2f[i], approx, 0.01 * Imgproc.arcLength(hull2f[i], true), true);
List <Point> approx_polygon = approx.toList();
// approx_polygon = Scannerproc.filterPolygon(approx_polygon);
// Debug.Log(approx_polygon.Count);
if (!Scannerproc.isSquare(approx_polygon))
{
continue;
}
else
{
nowRectPoints.Clear();
nowRectPoints.AddRange(approx_polygon);
perspectiveAlign();
}
//Center of mass
int cx = 0,
cy = 0;
for (int k = 0; k < approx_polygon.Count; k++)
{
cx += (int)approx_polygon[k].x;
cy += (int)approx_polygon[k].y;
}
cx /= approx_polygon.Count;
cy /= approx_polygon.Count;
Scannerproc.drawShape(drawing, approx_polygon, new Scalar(0, 255, 0));
}
if (showTextureOnScreen)
{
showCurrentTextureOnScreen();
}
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//convert image to greyscale
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (isAutoResetMode || faceTracker.getPoints().Count <= 0)
{
// Debug.Log ("detectFace");
//convert image to greyscale
using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0
// | Objdetect.CASCADE_FIND_BIGGEST_OBJECT
| Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
if (faces.rows() > 0)
{
// Debug.Log ("faces " + faces.dump ());
List <OpenCVForUnity.CoreModule.Rect> rectsList = faces.toList();
List <Point[]> pointsList = faceTracker.getPoints();
if (isAutoResetMode)
{
//add initial face points from MatOfRect
if (pointsList.Count <= 0)
{
faceTracker.addPoints(faces);
// Debug.Log ("reset faces ");
}
else
{
for (int i = 0; i < rectsList.Count; i++)
{
OpenCVForUnity.CoreModule.Rect trackRect = new OpenCVForUnity.CoreModule.Rect(rectsList [i].x + rectsList [i].width / 3, rectsList [i].y + rectsList [i].height / 2, rectsList [i].width / 3, rectsList [i].height / 3);
//It determines whether nose point has been included in trackRect.
if (i < pointsList.Count && !trackRect.contains(pointsList [i] [67]))
{
rectsList.RemoveAt(i);
pointsList.RemoveAt(i);
// Debug.Log ("remove " + i);
}
//uncomment below for rectangle around face
Imgproc.rectangle(rgbaMat, new Point(trackRect.x, trackRect.y), new Point(trackRect.x + trackRect.width, trackRect.y + trackRect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
else
{
faceTracker.addPoints(faces);
}
//draw face rect
for (int i = 0; i < rectsList.Count; i++)
{
//uncomment below for rectangle around face
Imgproc.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsList [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
}
}
else
{
if (isAutoResetMode)
{
faceTracker.reset();
}
}
}
}
//track face points.if face points <= 0, always return false.
if (faceTracker.track(grayMat, faceTrackerParams))
{
//GameObject.FindGameObjectWithTag("left hand").transform.localScale = new Vector3(0.05f, 0.05f, 50);
//GameObject.FindGameObjectWithTag("right hand").transform.localScale = new Vector3(0.05f, 0.05f, 50);
//facecount = 0;
if (facerec > 15)
{
GameObject.FindGameObjectWithTag("left hand").transform.localScale = new Vector3(0.2f, 0.2f, 50);
GameObject.FindGameObjectWithTag("right hand").transform.localScale = new Vector3(0.2f, 0.2f, 50);
facecount = 0;
}
else
{
facerec++;
}
//uncomment below for rectangle around face
//faceTracker.draw(rgbaMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
}
else
{
//facecount prevents flickering of hand from poor face recognition
if (facecount > 15)
{
facerec = 0;
GameObject.FindGameObjectWithTag("left hand").transform.localScale = new Vector3(0f, 0f, 0);
GameObject.FindGameObjectWithTag("right hand").transform.localScale = new Vector3(0f, 0f, 0); facecount++;
}
else
{
facecount++;
}
}
//Imgproc.putText (rgbaMat, "'Tap' or 'Space Key' to Reset", new Point (5, rgbaMat.rows () - 5), Imgproc.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.fastMatToTexture2D(rgbaMat, texture);
}
//facetrac resets upon screen click and space bar
if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0)
{
faceTracker.reset();
}
if (Input.GetKeyDown(KeyCode.Escape))
{
if (SpeechRecognizer.IsRecording())
{
SpeechRecognizer.StopIfRecording();
//resultText.text = "I stopped recording";
}
Application.Quit();
//Application.LoadLevel ("MainActivity.class");
}
}
void Update()
{
if (!running)
{
return;
}
if (Input.GetMouseButtonDown(0))
{
Debug.Log($"Mouse Position: {Input.mousePosition} -> World: {Camera.main.ScreenToWorldPoint(Input.mousePosition)}");
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//Writes into the mat
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGB2GRAY); //COLOR_RGBA2GRAY
//Applies gaussian blur for better results
Imgproc.GaussianBlur(grayMat, grayMat, new Size(3, 3), 2);
using (Mat circles = new Mat()) {
//Circle detection using the hough gradient
Imgproc.HoughCircles(grayMat, circles, Imgproc.CV_HOUGH_GRADIENT, dp, minDist, param1, param2, minRadius, maxRadius);
Point pt = new Point();
//Limits the circle drawing when too much circles are detected
if ((int)circles.total() > 5)
{
for (int i = 0; i < circles.rows(); i++)
{
double[] data = circles.get(i, 0);
pt.x = data [0];
pt.y = data [1];
double rho = data [2];
Imgproc.circle(rgbaMat, pt, (int)rho, GlobalValues.DETECTION_COLOR, GlobalValues.RINGS_RADIUS);
}
}
else //Tennis ball tracking starts here
{
for (int i = 0; i < circles.rows(); i++)
{
for (var j = 0; j < circles.cols(); j++)
{
//Get the data from the API
double[] data = circles.get(i, j);
pt.x = data [0];
pt.y = data [1];
double rho = data [2];
//Convert to worldspace
Vector2 pos = new Vector2((float)data[0], webCamTextureToMatHelper.GetWebCamTexture().height - (float)data[1]);
Vector3 worldPos = Camera.main.ScreenToWorldPoint(AdjustToResolution(pos));
//Drawings for debug purposes
Debug.DrawRay(worldPos, Vector3.up * 10, Color.magenta, 1f);
Debug.DrawRay(worldPos, Vector3.down * 10, Color.magenta, 1f);
Debug.DrawRay(worldPos, Vector3.left * 10, Color.magenta, 1f);
Debug.DrawRay(worldPos, Vector3.right * 10, Color.magenta, 1f);
//If the ball went outside the detection threshold
if (ball_tracker.AwaitingForRegainFocus(worldPos))
{
//Flash a blue cirlcle to indicate the player where to start
if (Mathf.Sin(Time.time * GlobalValues.CHECK_POINT_BLINKING_FRECUENCY) > 0)
{
var last_pos = ball_tracker.GetLastPosition();
var screen_pos = InvertAdjustToResolution(Camera.main.WorldToScreenPoint(last_pos));
screen_pos.y = webCamTextureToMatHelper.GetWebCamTexture().height - screen_pos.y;
Imgproc.circle(rgbaMat, new Point(screen_pos.x, screen_pos.y), (int)rho, GlobalValues.CHECK_POINT_COLOR, GlobalValues.RINGS_RADIUS);
}
} //Otherwise Update the ball tracker
else if (ball_tracker.Update(worldPos))
{
Imgproc.circle(rgbaMat, pt, (int)rho, GlobalValues.TRACKING_COLOR, GlobalValues.RINGS_RADIUS);
}
}
}
}
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// CV_8UC3
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : bgrMat.cols(),
inpHeight > 0 ? inpHeight : bgrMat.rows());
Mat blob = Dnn.blobFromImage(bgrMat, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(bgrMat, bgrMat, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
//Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
postprocess(rgbaMat, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
}
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
/// <summary>
/// Update is called once per frame
/// </summary>
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
// Detect faces on resize image 在调整图像时察觉脸
//detect face rects 发现脸的矩形
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
;
if (detectResult.Count > 0)
{
//detect landmark points 检测具有界标意义的点
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult[0]);
//将points绘制在mat上
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
imagePoints.fromArray(
new Point((points[38].x + points[41].x) / 2, (points[38].y + points[41].y) / 2), //l eye (Interpupillary breadth)
new Point((points[43].x + points[46].x) / 2, (points[43].y + points[46].y) / 2), //r eye (Interpupillary breadth)
new Point(points[30].x, points[30].y), //nose (Nose top)
new Point(points[48].x, points[48].y), //l mouth (Mouth breadth)
new Point(points[54].x, points[54].y), //r mouth (Mouth breadth)
new Point(points[0].x, points[0].y), //l ear (Bitragion breadth)
new Point(points[16].x, points[16].y) //r ear (Bitragion breadth)
);
// Estimate head pose. 估计头部姿势
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
//从3D到2D点的pose中找到一个物体的姿势 rvec是旋转 tvec是平移向量
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_z = tvec.get(2, 0)[0];
if (double.IsNaN(tvec_z) || tvec_z < 0)
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
if (!double.IsNaN(tvec_z) && points.Count == 68)
{
cubismParameterDictionary.Clear();
Calib3d.Rodrigues(rvec, rotMat);
transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0)[0], (float)rotMat.get(0, 1)[0], (float)rotMat.get(0, 2)[0], (float)tvec.get(0, 0)[0]));
transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0)[0], (float)rotMat.get(1, 1)[0], (float)rotMat.get(1, 2)[0], (float)tvec.get(1, 0)[0]));
transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0)[0], (float)rotMat.get(2, 1)[0], (float)rotMat.get(2, 2)[0], (float)tvec.get(2, 0)[0]));
transformationM.SetRow(3, new Vector4(0, 0, 0, 1));
ARM = invertYM * transformationM * invertZM;
Vector3 forward;
forward.x = ARM.m02;
forward.y = ARM.m12;
forward.z = ARM.m22;
Vector3 upwards;
upwards.x = ARM.m01;
upwards.y = ARM.m11;
upwards.z = ARM.m21;
Vector3 angles = Quaternion.LookRotation(forward, upwards).eulerAngles;
float rotateX = angles.x > 180 ? angles.x - 360 : angles.x;
cubismParameterDictionary.Add(ParamAngleY, (float)Math.Round(rotateX));
float rotateY = angles.y > 180 ? angles.y - 360 : angles.y;
cubismParameterDictionary.Add(ParamAngleX, (float)Math.Round(-rotateY) * 2);
float rotateZ = angles.z > 180 ? angles.z - 360 : angles.z;
cubismParameterDictionary.Add(ParamAngleZ, (float)Math.Round(-rotateZ) * 2);
//Debug.("X" + rotateX + "Y" + rotateY + "Z" + rotateZ);
//ParamAngleX.BlendToValue(BlendMode,(float)(Math.Round(-rotateY) * 2));
//ParamAngleY.BlendToValue(BlendMode, (float)Math.Round(rotateX));
//ParamAngleZ.BlendToValue(BlendMode, (float)Math.Round(-rotateZ) * 2);
float eyeOpen_L = Mathf.Clamp(Mathf.Abs(points[43].y - points[47].y) / (Mathf.Abs(points[43].x - points[44].x) * 0.75f), -0.1f, 2.0f);
if (eyeOpen_L >= 0.8f)
{
eyeOpen_L = 1f;
}
else
{
eyeOpen_L = 0;
}
float eyeOpen_R = Mathf.Clamp(Mathf.Abs(points[38].y - points[40].y) / (Mathf.Abs(points[37].x - points[38].x) * 0.75f), -0.1f, 2.0f);
if (eyeOpen_R >= 0.8f)
{
eyeOpen_R = 1f;
}
else
{
eyeOpen_R = 0;
}
// ParamEyeROpen.BlendToValue(BlendMode, eyeOpen_R);
cubismParameterDictionary.Add(ParamEyeROpen, eyeOpen_R);
// ParamEyeLOpen.BlendToValue(BlendMode, eyeOpen_L);
cubismParameterDictionary.Add(ParamEyeLOpen, eyeOpen_L);
// ParamEyeBallX.BlendToValue(BlendMode, (float)rotateY / 30f);
cubismParameterDictionary.Add(ParamEyeBallX, rotateY / 30f);
// ParamEyeBallX.BlendToValue(BlendMode, (float)-rotateX / 30f - 0.25f);
cubismParameterDictionary.Add(ParamEyeBallY, (float)-rotateX / 30f - 0.25f);
float RY = Mathf.Abs(points[19].y - points[27].y) / Mathf.Abs(points[27].y - points[29].y);
RY -= 1;
RY *= 4f;
float LY = Mathf.Abs(points[24].y - points[27].y) / Mathf.Abs(points[27].y - points[29].y);
LY -= 1;
LY *= 4f;
// ParamBrowRY.BlendToValue(BlendMode, RY);
cubismParameterDictionary.Add(ParamBrowRY, RY);
// ParamBrowLY.BlendToValue(BlendMode, LY);
cubismParameterDictionary.Add(ParamBrowLY, LY);
float mouthOpen = Mathf.Clamp01(Mathf.Abs(points[62].y - points[66].y) / (Mathf.Abs(points[51].y - points[62].y) + Mathf.Abs(points[66].y - points[57].y)));
if (mouthOpen < 0.6f)
{
mouthOpen = 0;
}
// ParamMouthOpenY.BlendToValue(BlendMode, mouthOpen);
cubismParameterDictionary.Add(ParamMouthOpenY, mouthOpen);
float mouthSize = Mathf.Abs(points[48].x - points[54].x) / (Mathf.Abs(points[31].x - points[35].x));
// ParamMouthForm.BlendToValue(BlendMode, Mathf.Clamp(mouthSize, -1.0f, 1.0f));
cubismParameterDictionary.Add(ParamMouthForm, Mathf.Clamp(mouthSize, -1.0f, 1.0f));
}
}
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
OpenCVForUnity.CoreModule.Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
}
/*
* Parameters:
* prevImg first 8-bit input image
* nextImg second input image
* prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
* nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
* status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
* err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
//if (mMOBStatus.rows() > 0)
//{
// List<Point> cornersPrev = mMOP2fptsPrev.toList();
// List<Point> cornersThis = mMOP2fptsThis.toList();
// List<byte> byteStatus = mMOBStatus.toList();
// int x = 0;
// int y = byteStatus.Count - 1;
// for (x = 0; x < y; x++)
// {
// if (byteStatus[x] == 1)
// {
// Point pt = cornersThis[x];
// Point pt2 = cornersPrev[x];
// Imgproc.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1);
// Imgproc.line(rgbaMat, pt, pt2, colorRed, iLineThickness);
// }
// }
//}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.fastMatToTexture2D(rgbaMat, texture);
if (DateTime.Now >= dataAtual.AddSeconds(5))
{
webCamTextureToMatHelper.Pause();
TakeSnapshot();
DetectShapes();
dataAtual = DateTime.Now;
webCamTextureToMatHelper.Play();
}
}
}
void Update()
{
if (Input.GetKeyDown(KeyCode.Escape))
{
OnBackButton();
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
markerDetector.processFrame(rgbaMat, 1);
foreach (MarkerSettings settings in markerSettings)
{
if (!settings.shouldNotSetToInactivePerFrame)
{
settings.setAllARGameObjectsDisable();
}
else
{
GameObject ARGameObject = settings.getARGameObject();
if (ARGameObject != null)
{
DelayableSetActive obj = ARGameObject.GetComponent <DelayableSetActive> ();
if (obj != null)
{
obj.SetActive(false, 0.5f);
}
}
}
}
List <Marker> findMarkers = markerDetector.getFindMarkers();
for (int i = 0; i < findMarkers.Count; i++)
{
Marker marker = findMarkers [i];
foreach (MarkerSettings settings in markerSettings)
{
if (marker.id == settings.getMarkerId())
{
transformationM = marker.transformation;
//Debug.Log ("transformationM " + transformationM.ToString ());
ARM = ARCamera.transform.localToWorldMatrix * invertYM * transformationM * invertZM;
//Debug.Log ("arM " + arM.ToString ());
GameObject ARGameObject = settings.getARGameObject();
if (ARGameObject != null)
{
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
DelayableSetActive obj = ARGameObject.GetComponent <DelayableSetActive> ();
if (obj != null)
{
obj.SetActive(true);
}
else
{
ARGameObject.SetActive(true);
}
}
}
}
}
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Image Processing function - called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
//load camera feed into matrix
Mat frame = webCamTextureToMatHelper.GetMat();
//clone frame to new variable
Mat cameraFeed = frame.clone();
//apply blurring methods to image
Imgproc.GaussianBlur(cameraFeed, cameraFeed, new Size(5, 5), 0);
Imgproc.medianBlur(cameraFeed, cameraFeed, 3);
//convert to hsv colour space
Mat hsv_image = new Mat();
Imgproc.cvtColor(cameraFeed, hsv_image, Imgproc.COLOR_BGR2HSV);
//create thresholds for colour isolation
Mat blue_hue_range = new Mat();
Mat red_hue_range = new Mat();
Mat lower_red = new Mat();
Mat upper_red = new Mat();
//upper and lower red colour thresholds
Core.inRange(hsv_image, new Scalar(0, 100, 100), new Scalar(10, 200, 200), lower_red);
Core.inRange(hsv_image, new Scalar(160, 100, 100), new Scalar(179, 255, 255), upper_red);
//add red thresholds together
Core.addWeighted(lower_red, 1.0, upper_red, 1.0, 0.0, red_hue_range);
Core.inRange(hsv_image, new Scalar(115, 100, 100), new Scalar(135, 200, 200), blue_hue_range);
//add red and blue thresholds together
Mat hue_image = new Mat();
Core.addWeighted(blue_hue_range, 1.0, red_hue_range, 1.0, 0.0, hue_image);
//noise reduction on hsv image
Imgproc.GaussianBlur(hue_image, hue_image, new Size(9, 9), 5);
Mat erodeElement = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
Mat dilateElement = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(8, 8));
Imgproc.erode(hue_image, hue_image, erodeElement);
Imgproc.dilate(hue_image, hue_image, dilateElement);
//find contours in image
System.Collections.Generic.List <MatOfPoint> circles = new System.Collections.Generic.List <MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(hue_image, circles, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0));
//find circles and draw if radius is > 30
for (int i = 0; i < circles.Count; i++)
{
Point pt = new Point();
float[] radius = new float[1];
Imgproc.minEnclosingCircle(new MatOfPoint2f(circles[i].toArray()), pt, radius);
int r = (int)radius[0];
if (r > 30)
{
Imgproc.circle(frame, pt, r, new Scalar(0, 255, 0), 3);
}
}
//output either frame with circles drawn or hsv feed depending on status of change camera button
if (camMode == false)
{
Utils.matToTexture2D(frame, texture, webCamTextureToMatHelper.GetBufferColors());
}
else
{
Utils.matToTexture2D(hue_image, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
}