| public static InvokeOnAppThread ( AppCallbackItem item, bool waitUntilDone ) : void | ||
| item | AppCallbackItem | Item to execute. |
| waitUntilDone | bool | Wait until item is executed. |
| return | void | |
private void trackFilteredObject(ObjectTracker ot, Mat threshold)
{
Mat temp = new Mat();
threshold.copyTo(temp);
List <MatOfPoint> contours = new List <MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(temp, contours, hierarchy, Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE);
if (hierarchy.rows() > 0)
{
for (int index = 0; index >= 0; index = (int)hierarchy.get(0, index)[0])
{
Moments moment = Imgproc.moments(contours[index]);
double area = moment.m00;
if (area > 10 * 10)
{
int x = (int)(moment.get_m10() / area);
int y = (int)(moment.get_m01() / area);
Vector2 point = new Vector2(x, y);
Vector3 dirRay = LocatableCameraUtils.PixelCoordToWorldCoord(_cameraToWorldMatrix, _projectionMatrix, _resolution, point);
Application.InvokeOnAppThread(() => {
ot.Sphere.transform.position = Camera.main.transform.position + new Vector3(0, ot.offset, 0);
SphereCollider collider = ot.Sphere.GetComponent <SphereCollider>();
// We inverse the ray source and dir to make the sphere collider work
Vector3 newPosRay = Camera.main.transform.position + dirRay * (collider.radius * 2);
Ray ray = new Ray(newPosRay, -dirRay);
RaycastHit hit;
if (Physics.Raycast(ray, out hit, collider.radius * 3))
{
Vector3 pos = hit.point;
ot.gameObject.transform.position = pos;
}
}, false);
}
}
}
}
private void OnFrameSampleAcquired(VideoCaptureSample sample)
{
// upload image bytes
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
// transform matrix
float[] cameraToWorldMatrixAsFloat;
float[] projectionMatrixAsFloat;
if (sample.TryGetCameraToWorldMatrix(out cameraToWorldMatrixAsFloat) == false || sample.TryGetProjectionMatrix(out projectionMatrixAsFloat) == false)
{
Debug.Log("Failed to get camera to world or projection matrix");
return;
}
_cameraToWorldMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(cameraToWorldMatrixAsFloat);
_projectionMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(projectionMatrixAsFloat);
//focal_x = _projectionMatrix[0, 0] * _resolution.width / 2;
//focal_y = _projectionMatrix[1, 1] * _resolution.height / 2;
//cx = _resolution.width / 2 + _resolution.width / 2 * _projectionMatrix[0, 2];
//cy = _resolution.height / 2 + _resolution.height / 2 * _projectionMatrix[1, 2];
// Debug.Log("focal_x: " + focal_x.ToString("f5") + " focal_y: " + focal_y.ToString("f5") + " cx: " + cx.ToString("f5") + " cy: " + cy.ToString("f5"));
sample.Dispose();
// Opencv mat conversion: to RGB mat
Mat frameBGRA = new Mat(_resolution.height, _resolution.width, CvType.CV_8UC4);
//Array.Reverse(_latestImageBytes);
frameBGRA.put(0, 0, _latestImageBytes);
//Core.flip(frameBGRA, frameBGRA, 0);
Mat frameBGR = new Mat(_resolution.height, _resolution.width, CvType.CV_8UC3);
Imgproc.cvtColor(frameBGRA, frameBGR, Imgproc.COLOR_BGRA2BGR);
Mat RGB = new Mat();
Imgproc.cvtColor(frameBGR, RGB, Imgproc.COLOR_BGR2RGB);
// Target detection: marker location in webcam
Vector4 location = markerDetection.DetectMarkers(RGB);
// Application thread. Because some operations are only allowed in main thread, not event thread
Application.InvokeOnAppThread(() =>
{
if (MarkerDetection.success == true)// if detected
{
Debug.Log("catch!");
// target: marker -> webcam -> world.
Vector3 target = _cameraToWorldMatrix * location;
Debug.Log("target: " + target.ToString("f5"));
Target.transform.position = target;
// render with the right matrix
Matrix4x4 V = Camera.main.GetStereoViewMatrix(Camera.StereoscopicEye.Right);
Matrix4x4 P = GL.GetGPUProjectionMatrix(Camera.main.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right), false); // openGL to DX
Matrix4x4 M = Target.transform.localToWorldMatrix;
Target.GetComponent <Renderer>().material.SetMatrix("MATRIX_MVP", P * V * M); // render with custom pipeline
Target.GetComponent <Renderer>().enabled = true; // show now
Vector4 vertex = P * V * M * new Vector4(0f, 0f, 0f, 1f); // vertex location on the rendering plane
//Vector3 cam = V.GetColumn(3);
//// ----------------------------------------------------
//// collide a cam-target ray with display collider
//Vector3 camToTarget = target - cam;
//Ray ray = new Ray(cam, camToTarget);
//RaycastHit hit;
//if (Physics.Raycast(ray, out hit, 2f))
//{
// Vector3 pos = hit.point; // physical hitted point
// Target.transform.position = pos;
// Debug.Log("hit pos: " + pos.ToString("f5"));
// Matrix4x4 M = Target.transform.localToWorldMatrix;
// Target.GetComponent<Renderer>().material.SetMatrix("MATRIX_MVP", P * V * M); // render with custom pipeline
// Target.GetComponent<Renderer>().enabled = true; // show now
// // Vector4 vertex = P * V * M * new Vector4(0f, 0f, 0f, 1f); // vertex location on the rendering plane
//}
// ---------------------------------------------------
// get eye position in eyecam
string[] eyedata = UDPCommunication.message.Split(',');
Vector4 eye_pos_e = new Vector4(float.Parse(eyedata[0]) / 1000, float.Parse(eyedata[1]) / 1000, float.Parse(eyedata[2]) / 1000, 1.0f); // in [m]
Debug.Log("eye in eyecam: " + eye_pos_e.ToString("f5"));
// eye: eyecam -> webcam -> world.
Vector3 eye_pos_w = _cameraToWorldMatrix * _eyecamTowebcam * eye_pos_e;
Debug.Log("eye in world: " + eye_pos_w.ToString("f5"));
// ----------------------------------------------------
// collide a eye-target ray with display collider
Vector3 eyeToTarget = target - eye_pos_w;
Ray ray_revised = new Ray(eye_pos_w, eyeToTarget);
RaycastHit hit_revised;
if (Physics.Raycast(ray_revised, out hit_revised, 2f))
{
Vector4 pos_revised = hit_revised.point; // physical hitted point
//Revised.transform.position = pos_revised;
Debug.Log("hit_revised pos: " + pos_revised.ToString("f5"));
pos_revised.w = 1.0f;
// calculate hitted vertex, scale w to the same depth
Vector4 vertex_hit = P * V * pos_revised;
float scale = vertex.w / vertex_hit.w;
Vector4 vertex_hit_scaled = new Vector4(vertex_hit.x * scale, vertex_hit.y * scale, vertex_hit.z, vertex_hit.w * scale);
// retrieve the world location
Vector3 pos_scaled = V.inverse * P.inverse * vertex_hit_scaled;
Revised.transform.position = pos_scaled;
// position the revised target and render it
Matrix4x4 M_revised = Revised.transform.localToWorldMatrix;
Revised.GetComponent <Renderer>().material.SetMatrix("MATRIX_MVP", P * V * M_revised);
Revised.GetComponent <Renderer>().enabled = true;
Debug.Log("webcameraToWorldMatrix:\n" + _cameraToWorldMatrix.ToString("f5"));
Debug.Log("WorldToRightMatrix:\n" + V.ToString("f5"));
Debug.Log("RightGLProjectionMatrix:\n" + P.ToString("f5"));
Debug.Log("detected target location: " + Target.transform.position.ToString("f5"));
Debug.Log("revised target location: " + Revised.transform.position.ToString("f5"));
Debug.Log("rendering vertex: " + vertex.ToString("f5"));
Debug.Log("hit vertex: " + vertex_hit.ToString("f5"));
Debug.Log("revised rendering vertex: " + vertex_hit_scaled.ToString("f5"));
}
}
else
{
Revised.GetComponent <Renderer>().enabled = false;
Target.GetComponent <Renderer>().enabled = false; // hide
}
}, false);
}
