// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net.empty() || classes == null)
{
Imgproc.putText(rgbaMat, "model file or class names list file is not loaded.", new Point(5, rgbaMat.rows() - 50), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "The model and class names list can be downloaded here:", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
blob = Dnn.blobFromImage(rgbaMat, 1, new Size(224, 224), new Scalar(104, 117, 123), false);
net.setInput(blob);
Mat prob = net.forward();
Core.MinMaxLocResult minmax = Core.minMaxLoc(prob.reshape(1, 1));
// Debug.Log ("Best match " + (int)minmax.maxLoc.x);
// Debug.Log ("Best match class " + classes [(int)minmax.maxLoc.x]);
// Debug.Log ("Probability: " + minmax.maxVal * 100 + "%");
prob.Dispose();
Imgproc.putText(rgbaMat, "Best match class " + classes [(int)minmax.maxLoc.x], new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (cascade != null)
{
cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE
new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size());
}
OpenCVForUnity.Rect[] rects = faces.toArray();
for (int i = 0; i < rects.Length; i++)
{
// Debug.Log ("detect faces " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2);
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
//first find blue objects
Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV);
Core.inRange(hsvMat, blue.getHSVmin(), blue.getHSVmax(), thresholdMat);
morphOps(thresholdMat);
trackFilteredObject(blue, thresholdMat, hsvMat, rgbMat);
//then yellows
Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV);
Core.inRange(hsvMat, yellow.getHSVmin(), yellow.getHSVmax(), thresholdMat);
morphOps(thresholdMat);
trackFilteredObject(yellow, thresholdMat, hsvMat, rgbMat);
//then reds
Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV);
Core.inRange(hsvMat, red.getHSVmin(), red.getHSVmax(), thresholdMat);
morphOps(thresholdMat);
trackFilteredObject(red, thresholdMat, hsvMat, rgbMat);
//then greens
Imgproc.cvtColor(rgbMat, hsvMat, Imgproc.COLOR_RGB2HSV);
Core.inRange(hsvMat, green.getHSVmin(), green.getHSVmax(), thresholdMat);
morphOps(thresholdMat);
trackFilteredObject(green, thresholdMat, hsvMat, rgbMat);
Imgproc.putText(rgbMat, "W:" + rgbMat.width() + " H:" + rgbMat.height() + " SO:" + Screen.orientation, new Point(5, rgbMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
using (Mat circles = new Mat()) {
Imgproc.HoughCircles(grayMat, circles, Imgproc.CV_HOUGH_GRADIENT, 2, 10, 160, 50, 10, 40);
Point pt = new Point();
for (int i = 0; i < circles.cols(); i++)
{
double[] data = circles.get(0, i);
pt.x = data [0];
pt.y = data [1];
double rho = data [2];
Imgproc.circle(rgbaMat, pt, (int)rho, new Scalar(255, 0, 0, 255), 5);
}
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
blob = Dnn.blobFromImage(bgrMat, inScaleFactor, new Size(inWidth, inHeight), new Scalar(meanVal, meanVal, meanVal), false, false);
net.setInput(blob);
Mat prob = net.forward();
prob = prob.reshape(1, (int)prob.total() / 7);
float[] data = new float[7];
float confidenceThreshold = 0.2f;
for (int i = 0; i < prob.rows(); i++)
{
prob.get(i, 0, data);
float confidence = data [2];
if (confidence > confidenceThreshold)
{
int class_id = (int)(data [1]);
float left = data [3] * rgbaMat.cols();
float top = data [4] * rgbaMat.rows();
float right = data [5] * rgbaMat.cols();
float bottom = data [6] * rgbaMat.rows();
Imgproc.rectangle(rgbaMat, new Point(left, top), new Point(right, bottom),
new Scalar(0, 255, 0, 255), 2);
string label = classNames [class_id] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(rgbaMat, new Point(left, top),
new Point(left + labelSize.width, top + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255, 255), Core.FILLED);
Imgproc.putText(rgbaMat, label, new Point(left, top + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255));
}
}
prob.Dispose();
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0)
{
rgbaMat.copyTo(bgMat);
setBgTexture(bgMat);
}
//set fgMaskMat
findFgMaskMat(rgbaMat, bgMat, thresh);
//set bgMaskMat
Core.bitwise_not(fgMaskMat, bgMaskMat);
//copy greenMat using bgMaskMat
greenMat.copyTo(rgbaMat, bgMaskMat);
Imgproc.putText(rgbaMat, "SPACE KEY or TOUCH SCREEN: Reset backgroud img", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net.empty() || classes == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
blob = Dnn.blobFromImage(bgrMat, 1, new Size(224, 224), new Scalar(104, 117, 123), false, true);
net.setInput(blob);
Mat prob = net.forward();
Core.MinMaxLocResult minmax = Core.minMaxLoc(prob.reshape(1, 1));
// Debug.Log ("Best match " + (int)minmax.maxLoc.x);
// Debug.Log ("Best match class " + classes [(int)minmax.maxLoc.x]);
// Debug.Log ("Probability: " + minmax.maxVal * 100 + "%");
prob.Dispose();
Imgproc.putText(rgbaMat, "Best match class " + classes [(int)minmax.maxLoc.x], new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
// Utils.webCamTextureToMat (webCamTexture, grayMat, colors);
bgMat.copyTo(dstMat);
Imgproc.GaussianBlur(grayMat, lineMat, new Size(3, 3), 0);
grayMat.get(0, 0, grayPixels);
for (int i = 0; i < grayPixels.Length; i++)
{
maskPixels [i] = 0;
if (grayPixels [i] < 70)
{
grayPixels [i] = 0;
maskPixels [i] = 1;
}
else if (70 <= grayPixels [i] && grayPixels [i] < 120)
{
grayPixels [i] = 100;
}
else
{
grayPixels [i] = 255;
maskPixels [i] = 1;
}
}
grayMat.put(0, 0, grayPixels);
maskMat.put(0, 0, maskPixels);
grayMat.copyTo(dstMat, maskMat);
Imgproc.Canny(lineMat, lineMat, 20, 120);
lineMat.copyTo(maskMat);
Core.bitwise_not(lineMat, lineMat);
lineMat.copyTo(dstMat, maskMat);
// Imgproc.putText (dstMat, "W:" + dstMat.width () + " H:" + dstMat.height () + " SO:" + Screen.orientation, new Point (5, dstMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (0), 2, Imgproc.LINE_AA, false);
// Imgproc.cvtColor(dstMat,rgbaMat,Imgproc.COLOR_GRAY2RGBA);
// Utils.matToTexture2D (rgbaMat, texture);
Utils.matToTexture2D(dstMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
copyMat = rgbaMat.clone();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (cascade != null)
{
cascade.detectMultiScale(grayMat,
faces,
1.1, 2, 2,
new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2),
new Size());
}
OpenCVForUnity.Rect[] rects = faces.toArray();
if (rects.Length == 0)
{
noFaceFrameCount++;
if (rectsLast != null && noFaceFrameCount <= maxNegativeFrames && rectsLast.Length > 0)
{
blurBackground(rectsLast, rgbaMat);
}
else
{
//clear last rect fields
rowRangeTopLast = null;
rowRangeButtomLast = null;
colRangeleftLast = null;
colRangeRightLast = null;
}
}
if (rects != null && rects.Length > 0)
{
blurBackground(rects, rgbaMat);
noFaceFrameCount = 0;
rectsLast = null;
rectsLast = new OpenCVForUnity.Rect[rects.Length];
for (int i = 0; i < rects.Length; i++)
{
rectsLast[i] = rects[i];
}
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
/// <summary>
/// Raises the web cam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(webCamTextureMat, texture, webCamTextureToMatHelper.GetBufferColors());
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
gameObject.transform.localScale = new Vector3(webCamTextureMat.cols(), webCamTextureMat.rows(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
if (fpsMonitor != null)
{
fpsMonitor.Add("width", webCamTextureMat.width().ToString());
fpsMonitor.Add("height", webCamTextureMat.height().ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
float width = webCamTextureMat.width();
float height = webCamTextureMat.height();
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
grayMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC1);
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
backgroundSubstractorMOG2.apply(rgbMat, fgmaskMat);
Core.bitwise_not(fgmaskMat, fgmaskMat);
rgbaMat.setTo(new Scalar(0, 0, 0, 0), fgmaskMat);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGB2HSV);
Scalar lower_red = new Scalar(145, 42, 154);
Scalar lower_blue = new Scalar(90, 50, 50);
Scalar upper_red = new Scalar(255, 255, 255);
Scalar upper_blue = new Scalar(130, 255, 255);
Core.inRange(grayMat, lower_red, upper_red, redframe_threshold);
Core.inRange(grayMat, lower_blue, upper_blue, blueframe_threshold);
Core.bitwise_or(redframe_threshold, blueframe_threshold, frame_threshold);
Size size = new Size(5, 5);
Imgproc.erode(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size));
Imgproc.dilate(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size));
Imgproc.erode(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size));
Imgproc.dilate(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size));
Imgproc.erode(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size));
Imgproc.dilate(frame_threshold, frame_threshold, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, size));
using (Mat circles = new Mat()) {
Imgproc.HoughCircles(frame_threshold, circles, Imgproc.CV_HOUGH_GRADIENT, 1, frame_threshold.rows() / 2, 20, 15, 15, 100);
Point pt = new Point();
for (int i = 0; i < circles.cols(); i++)
{
double[] data = circles.get(0, i);
pt.x = data [0];
pt.y = data [1];
double rho = data [2];
// Imgproc.circle (rgbaMat, pt, 3, new Scalar (255, 0, 255), 5);
Imgproc.circle(rgbaMat, pt, (int)rho, new Scalar(255, 0, 0, 255), 5);
}
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
bool isTouched = false;
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
if (Input.touchCount == 1)
{
Touch t = Input.GetTouch(0);
if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId))
{
isTouched = true;
}
}
#else
if (Input.GetKeyUp(KeyCode.Space))
{
isTouched = true;
}
#endif
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (isTouched)
{
rgbaMat.copyTo(bgMat);
setBgTexture(bgMat);
}
//set fgMaskMat
findFgMaskMat(rgbaMat, bgMat, thresh);
//set bgMaskMat
Core.bitwise_not(fgMaskMat, bgMaskMat);
//copy greenMat using bgMaskMat
greenMat.copyTo(rgbaMat, bgMaskMat);
//Imgproc.putText (rgbaMat, "SPACE KEY or TOUCH SCREEN: Reset backgroud image.", new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.6, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
//Touch
int touchCount = Input.touchCount;
if (touchCount == 1)
{
Touch t = Input.GetTouch(0);
if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId))
{
storedTouchPoint = new Point(t.position.x, t.position.y);
//Debug.Log ("touch X " + t.position.x);
//Debug.Log ("touch Y " + t.position.y);
}
}
#else
//Mouse
if (Input.GetMouseButtonUp(0) && !EventSystem.current.IsPointerOverGameObject())
{
storedTouchPoint = new Point(Input.mousePosition.x, Input.mousePosition.y);
//Debug.Log ("mouse X " + Input.mousePosition.x);
//Debug.Log ("mouse Y " + Input.mousePosition.y);
}
#endif
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (storedTouchPoint != null)
{
onTouch(rgbaMat, convertScreenPoint(storedTouchPoint, gameObject, Camera.main));
storedTouchPoint = null;
}
handPoseEstimationProcess(rgbaMat);
Imgproc.putText(rgbaMat, "Please touch the area of the open hand.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
//Here starts the OpenCV script
/*
* // ENTER YOUR OPENCV CODE HERES
*/
//
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
// detect markers and estimate pose
Aruco.detectMarkers(rgbMat, dictionary, corners, ids, detectorParams, rejected);
// Aruco.detectMarkers (imgMat, dictionary, corners, ids);
if (estimatePose && ids.total() > 0)
{
Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs);
}
// draw results
if (ids.total() > 0)
{
Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(255, 0, 0));
if (estimatePose)
{
for (int i = 0; i < ids.total(); i++)
{
Aruco.drawAxis(rgbMat, camMatrix, distCoeffs, rvecs, tvecs, markerLength * 0.5f);
//This example can display ARObject on only first detected marker.
if (i == 0)
{
// position
double[] tvec = tvecs.get(i, 0);
Vector4 pos = new Vector4((float)tvec [0], (float)tvec [1], (float)tvec [2], 0); // from OpenCV
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARGameObject.transform.localPosition = new Vector3(pos.x, -pos.y, pos.z);
// rotation
double[] rv = rvecs.get(i, 0);
Mat rvec = new Mat(3, 1, CvType.CV_64FC1);
rvec.put(0, 0, rv[0]);
rvec.put(1, 0, rv[1]);
rvec.put(2, 0, rv[2]);
Calib3d.Rodrigues(rvec, rotMat);
Vector3 forward = new Vector3((float)rotMat.get(0, 2) [0], (float)rotMat.get(1, 2) [0], (float)rotMat.get(2, 2) [0]); // from OpenCV
Vector3 up = new Vector3((float)rotMat.get(0, 1) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(2, 1) [0]); // from OpenCV
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
Quaternion rot = Quaternion.LookRotation(new Vector3(forward.x, -forward.y, forward.z), new Vector3(up.x, -up.y, up.z));
ARGameObject.transform.localRotation = rot;
// Apply Z axis inverted matrix.
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
transformationM = ARGameObject.transform.localToWorldMatrix * invertZM;
// Apply camera transform matrix.
transformationM = ARCamera.transform.localToWorldMatrix * transformationM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref transformationM);
}
}
}
}
if (showRejected && rejected.Count > 0)
{
Aruco.drawDetectedMarkers(rgbMat, rejected, new Mat(), new Scalar(0, 0, 255));
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
//! [Resizing without keeping aspect ratio]
Imgproc.resize(bgrMat, resized, new Size(network_width, network_height));
//! [Resizing without keeping aspect ratio]
//! [Prepare blob]
inputBlob = Dnn.blobFromImage(resized, 1 / 255.0, new Size(), new Scalar(0), true, true); //Convert Mat to batch of images
//! [Prepare blob]
//! [Set input blob]
net.setInput(inputBlob, "data"); //set the network input
//! [Set input blob]
// TickMeter tm = new TickMeter ();
// tm.start ();
//! [Make forward pass]
Mat detectionMat = net.forward("detection_out"); //compute output
//! [Make forward pass]
// tm.stop ();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
// Debug.Log ("detectionMat.ToString(): " + detectionMat.ToString ());
float[] position = new float[5];
float[] confidences = new float[80];
float confidenceThreshold = 0.24f;
for (int i = 0; i < detectionMat.rows(); i++)
{
detectionMat.get(i, 0, position);
detectionMat.get(i, 5, confidences);
int maxIdx = confidences.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I;
float confidence = confidences [maxIdx];
if (confidence > confidenceThreshold)
{
float x = position [0];
float y = position [1];
float width = position [2];
float height = position [3];
int xLeftBottom = (int)((x - width / 2) * rgbaMat.cols());
int yLeftBottom = (int)((y - height / 2) * rgbaMat.rows());
int xRightTop = (int)((x + width / 2) * rgbaMat.cols());
int yRightTop = (int)((y + height / 2) * rgbaMat.rows());
// Debug.Log ("confidence: " + confidence);
//
// Debug.Log (" " + xLeftBottom
// + " " + yLeftBottom
// + " " + xRightTop
// + " " + yRightTop);
Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop),
new Scalar(0, 255, 0, 255), 2);
if (maxIdx < classNames.Count)
{
string label = classNames [maxIdx] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom),
new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255, 255), Core.FILLED);
Imgproc.putText(rgbaMat, label, new Point(xLeftBottom, yLeftBottom + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255));
}
}
}
detectionMat.Dispose();
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
net.setInput(Dnn.blobFromImage(bgrMat, 1.0, new Size(inWidth, inHeight), new Scalar(104, 177, 123), false, false), "data");
Mat detections = net.forward("detection_out");
detections = detections.reshape(1, (int)detections.total() / 7);
float[] data = new float[7];
for (int i = 0; i < detections.rows(); i++)
{
detections.get(i, 0, data);
float confidence = data [2];
if (confidence > confThreshold)
{
float xLeftBottom = data [3] * bgrMat.cols();
float yLeftBottom = data [4] * bgrMat.rows();
float xRightTop = data [5] * bgrMat.cols();
float yRightTop = data [6] * bgrMat.rows();
Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop),
new Scalar(0, 255, 0, 255), 2);
string label = "face: " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom),
new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255, 255), Core.FILLED);
Imgproc.putText(rgbaMat, label, new Point(xLeftBottom, yLeftBottom + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255));
}
}
detections.Dispose();
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
// detect markers.
Aruco.detectMarkers(rgbMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs);
// refine marker detection.
if (refineMarkerDetection)
{
switch (markerType)
{
case (int)MarkerType.GridBoard:
Aruco.refineDetectedMarkers(rgbMat, gridBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams);
break;
case (int)MarkerType.ChArUcoBoard:
Aruco.refineDetectedMarkers(rgbMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams);
break;
}
}
// if at least one marker detected
if (ids.total() > 0)
{
if (markerType != (int)MarkerType.ChArUcoDiamondMarker)
{
if (markerType == (int)MarkerType.ChArUcoBoard)
{
Aruco.interpolateCornersCharuco(corners, ids, rgbMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers);
// draw markers.
Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(0, 255, 0));
if (charucoIds.total() > 0)
{
Aruco.drawDetectedCornersCharuco(rgbMat, charucoCorners, charucoIds, new Scalar(0, 0, 255));
}
}
else
{
// draw markers.
Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(0, 255, 0));
}
// estimate pose.
if (applyEstimationPose)
{
switch (markerType)
{
case (int)MarkerType.CanonicalMarker:
EstimatePoseCanonicalMarker(rgbMat);
break;
case (int)MarkerType.GridBoard:
EstimatePoseGridBoard(rgbMat);
break;
case (int)MarkerType.ChArUcoBoard:
EstimatePoseChArUcoBoard(rgbMat);
break;
}
}
}
else
{
// detect diamond markers.
Aruco.detectCharucoDiamond(rgbMat, corners, ids, diamondSquareLength / diamondMarkerLength, diamondCorners, diamondIds, camMatrix, distCoeffs);
// draw markers.
Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(0, 255, 0));
// draw diamond markers.
Aruco.drawDetectedDiamonds(rgbMat, diamondCorners, diamondIds, new Scalar(0, 0, 255));
// estimate pose.
if (applyEstimationPose)
{
EstimatePoseChArUcoDiamondMarker(rgbMat);
}
}
}
if (showRejectedCorners && rejectedCorners.Count > 0)
{
Aruco.drawDetectedMarkers(rgbMat, rejectedCorners, new Mat(), new Scalar(255, 0, 0));
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
/// Mouse interaction, not developed yet
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
//Touch
int touchCount = Input.touchCount;
if (touchCount == 1)
{
Touch t = Input.GetTouch(0);
if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId))
{
storedTouchPoint = new Point(t.position.x, t.position.y);
//Debug.Log ("touch X " + t.position.x);
//Debug.Log ("touch Y " + t.position.y);
}
}
#else
//Mouse
if (Input.GetMouseButtonUp(0) && !EventSystem.current.IsPointerOverGameObject())
{
storedTouchPoint = new Point(Input.mousePosition.x, Input.mousePosition.y);
//Debug.Log ("mouse X " + Input.mousePosition.x);
//Debug.Log ("mouse Y " + Input.mousePosition.y);
}
#endif
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
//Here starts the OpenCV script
//onTouch para el 1er clic
if (selectedPointList.Count == 1)
{
if (storedTouchPoint != null)
{
ConvertScreenPointToTexturePoint(storedTouchPoint, storedTouchPoint, gameObject, rgbaMat.cols(), rgbaMat.rows());
OnTouch(rgbaMat, storedTouchPoint); //rgab o gray¡
Debug.Log("primer clic por " + storedTouchPoint.x + ";" + storedTouchPoint.y);
storedTouchPoint = null;
}
}
//error PlayerLoop called recursively! on iOS.reccomend WebCamTexture.
if (selectedPointList.Count != 1)
{
//onTouch para el 2do clic
if (storedTouchPoint != null)
{
ConvertScreenPointToTexturePoint(storedTouchPoint, storedTouchPoint, gameObject, rgbaMat.cols(), rgbaMat.rows());
OnTouch(rgbaMat, storedTouchPoint);
storedTouchPoint = null;
}
//si ya es 2, creeme otra trackinArea
if (selectedPointList.Count < 2)
{
foreach (var point in selectedPointList)
{
Imgproc.circle(rgbaMat, point, 6, new Scalar(0, 0, 255), 2);
}
}
else
{
//key line! DM
using (MatOfPoint selectedPointMat = new MatOfPoint(selectedPointList.ToArray()))
{
//TODO 13n cambiar essa region por varias regiones, que le pasara
OpenCVForUnity.Rect region = Imgproc.boundingRect(selectedPointMat); //si se necesita al fin y al cabo
SelectTracker(tracker_type); //16n monotracker = TrackerKCF.create(); //8n
trackerInitialized = monotracker.init(grayMat, new Rect2d(region.x, region.y, region.width, region.height));
}
selectedPointList.Clear();
}
// aca ya no lo inicializa, sino que lo actualiza
if (trackerInitialized)
{
bool updated = monotracker.update(grayMat, bbox);//8n //pero nunca le pasa la bbox ni las coordenadas, unicamente al momento de tracker.init
if (bbox.width != 0 && bbox.height != 0)
{
Debug.Log("tracking por aca:" + bbox.x + ";" + bbox.y);
Imgproc.rectangle(rgbaMat, bbox.tl(), bbox.br(), new Scalar(255, 255, 255, 255), 2, 1, 0);
previousBox = new Rect2d(bbox.x, bbox.y, bbox.width, bbox.height);
}
else
{
Debug.Log("Se perdio en:" + previousBox.x + ";" + previousBox.y);
Imgproc.rectangle(rgbaMat, previousBox.tl(), previousBox.br(), new Scalar(255, 0, 0, 255), 2, 1, 0); //8n
}
}
// bool updated = trackers.update (rgbaMat, objects);
// Debug.Log ("updated " + updated);
// if (!updated && bbox.width > 0)
// OnResetTrackerButtonClick ();
// }
if (selectedPointList.Count != 1)
{
//Imgproc.putText (rgbaMat, "Please touch the screen, and select tracking regions.", new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "Touch the screen to select a new tracking region.";
}
}
else
{
//Imgproc.putText (rgbaMat, "Please select the end point of the new tracking region.", new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "Please select the end point of the new tracking region.";
}
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
}
// Update is called once per frame
void Update()
{
#if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
//Touch
int touchCount = Input.touchCount;
if (touchCount == 1)
{
Touch t = Input.GetTouch(0);
if (t.phase == TouchPhase.Ended && !EventSystem.current.IsPointerOverGameObject(t.fingerId))
{
storedTouchPoint = new Point(t.position.x, t.position.y);
//Debug.Log ("touch X " + t.position.x);
//Debug.Log ("touch Y " + t.position.y);
}
}
#else
//Mouse
if (Input.GetMouseButtonUp(0) && !EventSystem.current.IsPointerOverGameObject())
{
storedTouchPoint = new Point(Input.mousePosition.x, Input.mousePosition.y);
//Debug.Log ("mouse X " + Input.mousePosition.x);
//Debug.Log ("mouse Y " + Input.mousePosition.y);
}
#endif
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, hsvMat, Imgproc.COLOR_RGBA2RGB);
Imgproc.cvtColor(hsvMat, hsvMat, Imgproc.COLOR_RGB2HSV);
if (storedTouchPoint != null)
{
ConvertScreenPointToTexturePoint(storedTouchPoint, storedTouchPoint, gameObject, rgbaMat.cols(), rgbaMat.rows());
OnTouch(rgbaMat, storedTouchPoint);
storedTouchPoint = null;
}
Point[] points = roiPointList.ToArray();
if (shouldStartCamShift)
{
shouldStartCamShift = false;
using (MatOfPoint roiPointMat = new MatOfPoint(points)) {
roiRect = Imgproc.boundingRect(roiPointMat);
}
if (roiHistMat != null)
{
roiHistMat.Dispose();
roiHistMat = null;
}
roiHistMat = new Mat();
using (Mat roiHSVMat = new Mat(hsvMat, roiRect))
using (Mat maskMat = new Mat()) {
Imgproc.calcHist(new List <Mat> (new Mat[] { roiHSVMat }), new MatOfInt(0), maskMat, roiHistMat, new MatOfInt(16), new MatOfFloat(0, 180));
Core.normalize(roiHistMat, roiHistMat, 0, 255, Core.NORM_MINMAX);
//Debug.Log ("roiHist " + roiHistMat.ToString ());
}
}
else if (points.Length == 4)
{
using (Mat backProj = new Mat()) {
Imgproc.calcBackProject(new List <Mat> (new Mat[] { hsvMat }), new MatOfInt(0), roiHistMat, backProj, new MatOfFloat(0, 180), 1.0);
RotatedRect r = Video.CamShift(backProj, roiRect, termination);
r.points(points);
}
}
if (points.Length < 4)
{
for (int i = 0; i < points.Length; i++)
{
Imgproc.circle(rgbaMat, points [i], 6, new Scalar(0, 0, 255, 255), 2);
}
}
else
{
for (int i = 0; i < 4; i++)
{
Imgproc.line(rgbaMat, points [i], points [(i + 1) % 4], new Scalar(255, 0, 0, 255), 2);
}
Imgproc.rectangle(rgbaMat, roiRect.tl(), roiRect.br(), new Scalar(0, 255, 0, 255), 2);
}
Imgproc.putText(rgbaMat, "Please touch the 4 points surrounding the tracking object.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (mMOP2fptsPrev.rows() == 0)
{
// first time through the loop so we need prev and this mats
// plus prev points
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// copy that to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get prev corners
Imgproc.goodFeaturesToTrack(matOpFlowPrev, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsPrev.fromArray(MOPcorners.toArray());
// get safe copy of this corners
mMOP2fptsPrev.copyTo(mMOP2fptsSafe);
}
else
{
// we've been through before so
// this mat is valid. Copy it to prev mat
matOpFlowThis.copyTo(matOpFlowPrev);
// get this mat
Imgproc.cvtColor(rgbaMat, matOpFlowThis, Imgproc.COLOR_RGBA2GRAY);
// get the corners for this mat
Imgproc.goodFeaturesToTrack(matOpFlowThis, MOPcorners, iGFFTMax, 0.05, 20);
mMOP2fptsThis.fromArray(MOPcorners.toArray());
// retrieve the corners from the prev mat
// (saves calculating them again)
mMOP2fptsSafe.copyTo(mMOP2fptsPrev);
// and save this corners for next time through
mMOP2fptsThis.copyTo(mMOP2fptsSafe);
}
/*
* Parameters:
* prevImg first 8-bit input image
* nextImg second input image
* prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision floating-point numbers.
* nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
* status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
* err output vector of errors; each element of the vector is set to an error for the corresponding feature, type of the error measure can be set in flags parameter; if the flow wasn't found then the error is not defined (use the status parameter to find such cases).
*/
Video.calcOpticalFlowPyrLK(matOpFlowPrev, matOpFlowThis, mMOP2fptsPrev, mMOP2fptsThis, mMOBStatus, mMOFerr);
if (mMOBStatus.rows() > 0)
{
List <Point> cornersPrev = mMOP2fptsPrev.toList();
List <Point> cornersThis = mMOP2fptsThis.toList();
List <byte> byteStatus = mMOBStatus.toList();
int x = 0;
int y = byteStatus.Count - 1;
for (x = 0; x < y; x++)
{
if (byteStatus [x] == 1)
{
Point pt = cornersThis [x];
Point pt2 = cornersPrev [x];
Imgproc.circle(rgbaMat, pt, 5, colorRed, iLineThickness - 1);
Imgproc.line(rgbaMat, pt, pt2, colorRed, iLineThickness);
}
}
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
// detect markers.
Aruco.detectMarkers(rgbMat, dictionary, corners, ids, detectorParams, rejected);
// estimate pose.
if (applyEstimationPose && ids.total() > 0)
{
Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs);
}
if (ids.total() > 0)
{
Aruco.drawDetectedMarkers(rgbMat, corners, ids, new Scalar(255, 0, 0));
if (applyEstimationPose)
{
for (int i = 0; i < ids.total(); i++)
{
Aruco.drawAxis(rgbMat, camMatrix, distCoeffs, rvecs, tvecs, markerLength * 0.5f);
// This example can display ARObject on only first detected marker.
if (i == 0)
{
// position
double[] tvec = tvecs.get(i, 0);
// rotation
double[] rv = rvecs.get(i, 0);
Mat rvec = new Mat(3, 1, CvType.CV_64FC1);
rvec.put(0, 0, rv [0]);
rvec.put(1, 0, rv [1]);
rvec.put(2, 0, rv [2]);
Calib3d.Rodrigues(rvec, rotMat);
transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0) [0], (float)rotMat.get(0, 1) [0], (float)rotMat.get(0, 2) [0], (float)tvec [0]));
transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(1, 2) [0], (float)tvec [1]));
transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0) [0], (float)rotMat.get(2, 1) [0], (float)rotMat.get(2, 2) [0], (float)tvec [2]));
transformationM.SetRow(3, new Vector4(0, 0, 0, 1));
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z axis inverted matrix.
ARM = ARM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
}
}
}
if (showRejected && rejected.Count > 0)
{
Aruco.drawDetectedMarkers(rgbMat, rejected, new Mat(), new Scalar(0, 0, 255));
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame() && !imageOptimizationHelper.IsCurrentFrameSkipped())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//get downScaleMat;
Mat downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat((rgbaMat));
//grayscale
Imgproc.cvtColor(downScaleRgbaMat, gray1Mat, Imgproc.COLOR_RGBA2GRAY);
//blur
Imgproc.blur(gray1Mat, gray2Mat, new Size(5, 5));
//edge filter
Imgproc.filter2D(gray2Mat, gray1Mat, gray1Mat.depth(), kernel);
//blur
Imgproc.blur(gray1Mat, gray2Mat, new Size(3, 3));
//detect edge
Imgproc.threshold(gray2Mat, gray2Mat, EDGE_DETECT_VALUE, 255, Imgproc.THRESH_BINARY);
//copy Mat to byteArray
Utils.copyFromMat <byte> (gray2Mat, byteArray);
//set edge pointList
List <Point> pointList = new List <Point> ();
int w = gray1Mat.width();
int h = gray1Mat.height();
for (int y = 0; y < h; y++)
{
for (int x = 0; x < w; x++)
{
if (byteArray [x + w * y] == 255)
{
pointList.Add(new Point(x, y));
}
}
}
int limit = Mathf.RoundToInt((float)(pointList.Count * POINT_RATE));
if (limit > POINT_MAX_NUM)
{
limit = POINT_MAX_NUM;
}
while (pointList.Count > limit)
{
pointList.RemoveAt(Random.Range(0, pointList.Count));
}
// Debug.Log ("pointList.Count " + pointList.Count);
//init subdiv
subdiv.initDelaunay(new OpenCVForUnity.Rect(0, 0, downScaleRgbaMat.width(), downScaleRgbaMat.height()));
for (int i = 0; i < pointList.Count; i++)
{
subdiv.insert(pointList [i]);
}
subdiv.insert(new Point(0, 0));
subdiv.insert(new Point(gray1Mat.width() / 2 - 1, 0));
subdiv.insert(new Point(gray1Mat.width() - 1, 0));
subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() / 2 - 1));
subdiv.insert(new Point(gray1Mat.width() - 1, gray1Mat.height() - 1));
subdiv.insert(new Point(gray1Mat.width() / 2 - 1, gray1Mat.height() - 1));
subdiv.insert(new Point(0, gray1Mat.height() - 1));
subdiv.insert(new Point(0, gray1Mat.height() / 2 - 1));
using (MatOfFloat6 triangleList = new MatOfFloat6()) {
subdiv.getTriangleList(triangleList);
float[] pointArray = triangleList.toArray();
float downScaleRatio = imageOptimizationHelper.downscaleRatio;
if (downScaleRatio < 1)
{
downScaleRatio = 1;
}
byte[] color = new byte[4];
for (int i = 0; i < pointArray.Length / 6; i++)
{
Point p0 = new Point(pointArray [i * 6 + 0] * downScaleRatio, pointArray [i * 6 + 1] * downScaleRatio);
Point p1 = new Point(pointArray [i * 6 + 2] * downScaleRatio, pointArray [i * 6 + 3] * downScaleRatio);
Point p2 = new Point(pointArray [i * 6 + 4] * downScaleRatio, pointArray [i * 6 + 5] * downScaleRatio);
if (p0.x < 0 || p0.x > rgbaMat.width())
{
continue;
}
if (p0.y < 0 || p0.y > rgbaMat.height())
{
continue;
}
if (p1.x < 0 || p1.x > rgbaMat.width())
{
continue;
}
if (p1.y < 0 || p1.y > rgbaMat.height())
{
continue;
}
if (p2.x < 0 || p2.x > rgbaMat.width())
{
continue;
}
if (p2.y < 0 || p2.y > rgbaMat.height())
{
continue;
}
//get center of gravity
int cx = (int)((p0.x + p1.x + p2.x) * 0.33333);
int cy = (int)((p0.y + p1.y + p2.y) * 0.33333);
// Debug.Log ("cx " + cx + " cy " + cy );
//get center of gravity color
rgbaMat.get(cy, cx, color);
// Debug.Log ("r " + color[0] + " g " + color[1] + " b " + color[2] + " a " + color[3]);
//fill Polygon
Imgproc.fillConvexPoly(rgbaMat, new MatOfPoint(p0, p1, p2), new Scalar(color [0], color [1], color [2], color [3]), Imgproc.LINE_AA, 0);
// Imgproc.line (rgbaMat, p0, p1, new Scalar (64, 255, 128, 255));
// Imgproc.line (rgbaMat, p1, p2, new Scalar (64, 255, 128, 255));
// Imgproc.line (rgbaMat, p2, p0, new Scalar (64, 255, 128, 255));
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " DOWNSCALE W:" + downScaleRgbaMat.width () + " H:" + downScaleRgbaMat.height (), new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
/*Text Detection*/
Imgproc.cvtColor(rgbMat, binaryMat, Imgproc.COLOR_RGB2GRAY);
Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
Core.absdiff(binaryMat, new Scalar(255), maskMat);
List <MatOfPoint> regions = new List <MatOfPoint> ();
OpenCVForUnity.Text.detectRegions(binaryMat, er_filter1, er_filter2, regions);
// Debug.Log ("regions.Count " + regions.Count);
MatOfRect groups_rects = new MatOfRect();
List <OpenCVForUnity.Rect> rects = new List <OpenCVForUnity.Rect> ();
if (regions.Count > 0)
{
OpenCVForUnity.Text.erGrouping(rgbMat, binaryMat, regions, groups_rects);
}
for (int i = 0; i < regions.Count; i++)
{
regions [i].Dispose();
}
regions.Clear();
rects.AddRange(groups_rects.toList());
groups_rects.Dispose();
/*Text Recognition (OCR)*/
List <Mat> detections = new List <Mat> ();
for (int i = 0; i < (int)rects.Count; i++)
{
Mat group_img = new Mat();
maskMat.submat(rects [i]).copyTo(group_img);
Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0));
detections.Add(group_img);
}
// Debug.Log ("detections.Count " + detections.Count);
// Debug.Log ("rects.Count " + rects.Count);
//#Visualization
for (int i = 0; i < rects.Count; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 255, 255, 255), 1);
Imgproc.putText(rgbaMat, "" + i, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false);
}
for (int i = 0; i < detections.Count; i++)
{
string output = decoder.run(detections [i], 0);
Debug.Log("output " + output);
if (string.IsNullOrEmpty(output))
{
Debug.LogError("IsNullOrEmpty output " + output);
}
else
{
Imgproc.putText(rgbaMat, " " + output, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false);
}
}
for (int i = 0; i < detections.Count; i++)
{
detections [i].Dispose();
}
detections.Clear();
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (cascade != null)
{
cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE
new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size());
}
OpenCVForUnity.Rect[] rects = faces.toArray();
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2);
//make the coordinates for the rectangle accessible
elPosX = rects [i].x + rects [i].width;
elPosY = rects [i].y + rects [i].height;
}
if (rects.Length > 0)
{
//yay we see a face
Debug.Log("Update: Face detected!!!");
//start the party (once)
//partyObject.SetActive (true);
//set the hat graphic true
//TODO: need to figure out a way for this to not flicker so much
//m_partyHat.SetActive (true);
//update the position of the hat
// TODO: need to provide an offset based on the size of the rectangle (i think!)
m_partyHat.transform.position = new Vector3(elPosX, elPosY, 0);
}
else
{
//set the hat graphic false
Debug.Log("Update: No face yet!");
//m_partyHat.SetActive (false);
}
//float elPosX = rects [0].x;
//float elPosX = 400f;
//float elPosY = rects [0].y;
//float elPosY = 100f;
//Debug.Log (elPosX + ", " + elPosY);
//m_partyHat.transform.position = new Vector3 (elPosX, elPosY, 0);
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
