/// <summary>
/// Gets all images needed for any of the listeners for any of the OnImageUpdated events, converts them to an OpenCV Mat, and calls the events.
/// Called from zedManager.OnGrab() whenever the ZED updates the image.
/// You can extend this event to add other kinds of ZED Images, like VIEW.RIGHT, and add more events appropriately.
/// </summary>
private void OnZEDGrabbed()
{
if (!isInit)
{
return; //We haven't set up the camera mat yet, so we can't call any of the events that need it.
}
if (OnImageUpdated_LeftBGRA != null && OnImageUpdated_LeftBGRA.GetInvocationList().Length > 0) //Regular left color image as BGRA.
{
DeployGrabbedEvent(zedManager.GetLeftCamera(), ref zedLeftBGRAMat, VIEW.LEFT, ZEDMat.MAT_TYPE.MAT_8U_C4, ref cvLeftBGRAMat, OnImageUpdated_LeftBGRA);
}
if (OnImageUpdated_LeftBGR != null && OnImageUpdated_LeftBGR.GetInvocationList().Length > 0) //Regular left color image as BGRA.
{
DeployGrabbedEvent(zedManager.GetLeftCamera(), ref zedLeftBGRAMat, VIEW.LEFT, ZEDMat.MAT_TYPE.MAT_8U_C4, ref cvLeftBGRMat, OnImageUpdated_LeftBGR, OpenCVConversion.BGRA2BGR);
}
if (OnImageUpdated_LeftRGBA != null && OnImageUpdated_LeftRGBA.GetInvocationList().Length > 0) //Regular left color image as BGRA.
{
DeployGrabbedEvent(zedManager.GetLeftCamera(), ref zedLeftBGRAMat, VIEW.LEFT, ZEDMat.MAT_TYPE.MAT_8U_C4, ref cvLeftRGBAMat, OnImageUpdated_LeftRGBA, OpenCVConversion.BGRA2RGBA);
}
if (OnImageUpdated_LeftRGB != null && OnImageUpdated_LeftRGB.GetInvocationList().Length > 0) //Regular left color image as BGRA.
{
DeployGrabbedEvent(zedManager.GetLeftCamera(), ref zedLeftBGRAMat, VIEW.LEFT, ZEDMat.MAT_TYPE.MAT_8U_C4, ref cvLeftRGBMat, OnImageUpdated_LeftRGB, OpenCVConversion.BGRA2RGB);
}
if (OnImageUpdated_LeftGrayscale != null && OnImageUpdated_LeftGrayscale.GetInvocationList().Length > 0) //Left grayscale image.
{
DeployGrabbedEvent(zedManager.GetLeftCamera(), ref zedLeftGrayMat, VIEW.LEFT_GREY, ZEDMat.MAT_TYPE.MAT_8U_C1, ref cvLeftGrayMat, OnImageUpdated_LeftGrayscale);
}
}
/// <summary>
/// Checks nearby for valid places for the drone to move.
/// Valid places must be in front of the player, and not intersect any objects within a reasonable tolerance.
/// Use radiusCheckRate and percentageThreshold to tweak what counts as a valid location.
/// </summary>
/// <param name="newpos"></param>
/// <returns></returns>
private bool FindNewMovePosition(out Vector3 newpos)
{
//We can't move if the ZED isn't initialized.
if (!zedManager.IsZEDReady)
{
newpos = transform.position;
return(false);
}
Vector3 randomPosition;
// Look Around For a New Position
//If the Drone is on the screen, search around a smaller radius.
//Note that we only check the primary ZEDManager because we only want the drone to spawn in front of the player anyway.
if (ZEDSupportFunctions.CheckScreenView(transform.position, zedManager.GetLeftCamera()))
{
randomPosition = UnityEngine.Random.onUnitSphere * UnityEngine.Random.Range(2f, 3f) + transform.position;
}
else //if the drone is outside, look around a bigger radius to find a position which is inside the screen.
{
randomPosition = UnityEngine.Random.onUnitSphere * UnityEngine.Random.Range(4f, 5f) + transform.position;
}
// Look For Any Collisions Through The ZED
bool hit = ZEDSupportFunctions.HitTestAtPoint(zedManager.zedCamera, zedManager.GetLeftCamera(), randomPosition);
if (!hit)
{
newpos = transform.position;
return(false);
}
//If we spawn the drone at that world point, it'll spawn inside a wall. Bring it closer by a distance of ClearRadius.
Quaternion directiontoDrone = Quaternion.LookRotation(zedManager.GetLeftCameraTransform().position - randomPosition, Vector3.up);
Vector3 newPosition = randomPosition + directiontoDrone * Vector3.forward * ClearRadius;
//Check the new position isn't too close from the camera.
float dist = Vector3.Distance(zedManager.GetLeftCamera().transform.position, randomPosition);
if (dist < 1f)
{
newpos = transform.position;
return(false);
}
//Also check nearby points in a sphere of radius to make sure the whole drone has a clear space.
if (ZEDSupportFunctions.HitTestOnSphere(zedManager.zedCamera, zedManager.GetLeftCamera(), newPosition, 1f, radiusCheckRate, percentageThreshold))
{
newpos = transform.position;
return(false);
}
//Return true if it's made it this far and out the location we chose.
newpos = newPosition;
return(true);
}
// Use this for initialization
void Awake()
{
if (!text2D)
{
text2D = labelRoot.GetComponentInChildren <Text>();
}
thisFrameScale = labelRoot.localScale;
labelStartLossyScale = labelRoot.lossyScale;
if (!shaderIndexIDsSet)
{
FindShaderIndexes();
}
#if !ZED_LWRP && !ZED_HDRP
Camera.onPreCull += OnCameraPreRender;
#elif ZED_LWRP
RenderPipelineManager.beginCameraRendering += LWRPBeginCamera;
#elif ZED_HDRP
ZEDManager manager = FindObjectOfType <ZEDManager>();
labelFacingCamera = manager.GetLeftCamera();
RenderPipelineManager.beginFrameRendering += HDRPBeginFrame;
#endif
}
private void DrawAsteroids(ZEDManager manager)
{
Camera leftcamera = manager.GetLeftCamera();
Camera rightcamera = manager.GetRightCamera();
//Update positions and draw asteroids of type 1
UpdatePosition(listPositionsOrigin, listPositions);
Graphics.DrawMeshInstanced(asteroidsType1.GetComponent <MeshFilter>().sharedMesh,
0, asteroidsType1.GetComponent <MeshRenderer>().sharedMaterial,
listPositions,
listPositions.Length,
null,
UnityEngine.Rendering.ShadowCastingMode.Off,
false,
gameObject.layer,
leftcamera);
if (manager.IsStereoRig)
{
Graphics.DrawMeshInstanced(asteroidsType1.GetComponent <MeshFilter>().sharedMesh,
0,
asteroidsType1.GetComponent <MeshRenderer>().sharedMaterial,
listPositions,
listPositions.Length,
null,
UnityEngine.Rendering.ShadowCastingMode.Off,
false,
gameObject.layer,
rightcamera);
}
//Update positions and draw asteroids of type 2
UpdatePosition(listPositionsOrigin2, listPositions2);
Graphics.DrawMeshInstanced(asteroidsType2.GetComponent <MeshFilter>().sharedMesh,
0,
asteroidsType2.GetComponent <MeshRenderer>().sharedMaterial,
listPositions2,
listPositions2.Length,
null,
UnityEngine.Rendering.ShadowCastingMode.Off,
false,
gameObject.layer,
leftcamera);
if (manager.IsStereoRig)
{
Graphics.DrawMeshInstanced(asteroidsType2.GetComponent <MeshFilter>().sharedMesh,
0,
asteroidsType2.GetComponent <MeshRenderer>().sharedMaterial,
listPositions2,
listPositions2.Length,
null,
UnityEngine.Rendering.ShadowCastingMode.Off,
false,
gameObject.layer,
rightcamera);
}
}
private void Start()
{
if (!zedManager)
{
zedManager = FindObjectOfType <ZEDManager>();
}
if (zedManager)
{
zedLeftCam = zedManager.GetLeftCamera();
}
}
/// <summary>
/// Returns the viewport positions of the four corners of the object's 2D bounding box on the capturing camera.
/// <para>Like most of Unity, the Y values are relative to the bottom of the image, which is unlike the
/// raw imageBoundingBox data from the ObjectData struct.</para>
/// 0 ------- 1
/// | obj |
/// 3-------- 2
/// </summary>
/// <param name="scaleForCanvasUnityError">Adds an optional scaling factor to handle a bug in 2018.3 and greater where the
/// canvas set to Screen Space - Camera has very weird offsets when its projection matrix has certain small changes made to it directly.</param>
public Vector2[] Get2DBoundingBoxCorners_Viewport(float scaleForCanvasUnityError = 1)
{
Vector2[] imagecoords = Get2DBoundingBoxCorners_Image(scaleForCanvasUnityError);
Vector2[] viewportcorners = new Vector2[4];
for (int i = 0; i < 4; i++)
{
viewportcorners[i] = detectingZEDManager.GetLeftCamera().ScreenToViewportPoint(imagecoords[i]);
}
return(viewportcorners);
}
// Raycast from screen space to world space
// The GetWorldPositionAtPixel function is innacurate when using ZED_Rig_Mono for some reason. May be a bug with the Unity plugin.
// Raycast from the camera in the direction of the world space position we determined from the provided screen space position
public RaycastHit raycastFromScreenSpace(Vector2 screenPosition, ZEDManager manager)
{
Camera leftcamera = manager.GetLeftCamera();
RaycastHit hit;
int raycastLength = 20;
Vector3 worldPos;
ZEDSupportFunctions.GetWorldPositionAtPixel(zedCamera, screenPosition, leftcamera, out worldPos);
Vector3 direction = (worldPos - leftcamera.transform.position).normalized;
// Debug.DrawRay(leftcamera.transform.position, direction * raycastLength, Color.green, 30f);
Physics.Raycast(leftcamera.transform.position, direction * raycastLength, out hit);
return(hit);
}
/// <summary>
/// Detects the plane around screen-space coordinates specified.
/// </summary>
/// <returns><c>true</c>, if plane at hit was detected, <c>false</c> otherwise.</returns>
/// <param name="screenPos">Position of the pixel in screen space (2D).</param>
public bool DetectPlaneAtHit(ZEDManager manager, Vector2 screenPos)
{
if (!manager.IsZEDReady)
{
return(false); //Do nothing if the ZED isn't finished initializing.
}
sl.ZEDCamera zedcam = manager.zedCamera;
Camera leftcamera = manager.GetLeftCamera();
ZEDPlaneGameObject.PlaneData plane = new ZEDPlaneGameObject.PlaneData();
if (zedcam.findPlaneAtHit(ref plane, screenPos) == sl.ERROR_CODE.SUCCESS) //We found a plane.
{
int numVertices, numTriangles = 0;
zedcam.convertHitPlaneToMesh(planeMeshVertices, planeMeshTriangles, out numVertices, out numTriangles);
if (numVertices > 0 && numTriangles > 0)
{
GameObject newhitGO = new GameObject(); //Make a new GameObject to hold the new plane.
newhitGO.transform.SetParent(holder.transform);
Vector3[] worldPlaneVertices = new Vector3[numVertices];
int[] worldPlaneTriangles = new int[numTriangles];
TransformCameraToLocalMesh(leftcamera.transform, planeMeshVertices, planeMeshTriangles, worldPlaneVertices, worldPlaneTriangles, numVertices, numTriangles, plane.PlaneCenter);
//Move the GameObject to the center of the plane. Note that the plane data's center is relative to the camera.
newhitGO.transform.position = leftcamera.transform.position; //Add the camera's world position
newhitGO.transform.position += leftcamera.transform.rotation * plane.PlaneCenter; //Add the center of the plane
ZEDPlaneGameObject hitPlane = newhitGO.AddComponent <ZEDPlaneGameObject>();
if (overrideMaterial != null)
{
hitPlane.Create(leftcamera, plane, worldPlaneVertices, worldPlaneTriangles, planeHitCount + 1, overrideMaterial);
}
else
{
hitPlane.Create(leftcamera, plane, worldPlaneVertices, worldPlaneTriangles, planeHitCount + 1);
}
hitPlane.SetPhysics(addPhysicsOption);
//hitPlane.SetVisible(isVisibleInSceneOption);
hitPlaneList.Add(hitPlane);
planeHitCount++;
return(true);
}
}
return(false);
}
/// <summary>
/// Tests the depth of both the real and virtual in the center of the screen, and returns the world position of the closest one.
/// </summary>
/// <param name="crosshairpoint">Where the crosshair should be rendered.</param>
/// <param name="collisionnormal">The normal vector of the surface aimed at, for rotating the crosshair accordingly if desired.</param>
/// <returns>False if there is no valid object, real or virtual, on which to place the crosshair. </returns>
private bool FindCrosshairPosition(out Vector3 crosshairpoint, out Vector3 collisionnormal)
{
//Find the distance to the real world. The bool will be false if there is an error reading the depth at the center of the screen.
Vector3 realpoint = Vector3.zero;
float realdistance = 20f; //Arbitrary distance to put the crosshair if it hits nothing at all. Chosen by ZED's max range.
bool foundrealdistance = false;
if (ZEDSupportFunctions.HitTestOnRay(zedManager.zedCamera, zedManager.GetLeftCamera(), laserPointerBeadHolder.transform.position,
laserPointerBeadHolder.transform.rotation, 5f, 0.01f, out realpoint))
{
realdistance = Vector3.Distance(laserPointerBeadHolder.transform.position, realpoint);
foundrealdistance = true;
}
//Find the distance to the virtual. The bool will be false if there are no colliders ahead of you.
RaycastHit hitinfo;
bool foundvirtualdistance = Physics.Raycast(laserPointerBeadHolder.transform.position, laserPointerBeadHolder.transform.rotation * Vector3.forward, out hitinfo);
//If we didn't find either, return false so the laser and bead can be disabled.
if (!foundrealdistance && !foundvirtualdistance)
{
crosshairpoint = Vector3.zero;
collisionnormal = Vector3.zero;
return(false);
}
//Decide if we use the real or virtual distance
if (!foundvirtualdistance || realdistance < hitinfo.distance)
{
//The real world is closer. Give the position of the real world pixel and return true.
crosshairpoint = realpoint;
ZEDSupportFunctions.GetNormalAtWorldLocation(zedManager.zedCamera, realpoint, sl.REFERENCE_FRAME.WORLD, zedManager.GetLeftCamera(), out collisionnormal);
return(true);
}
else
{
//The virtual world is closer, or they're tied. Return the world posiiton where the raycast hit the virtual collider.
crosshairpoint = hitinfo.point;
collisionnormal = hitinfo.normal;
return(true);
}
}
private void Awake()
{
if (zedManager == null)
{
zedManager = FindObjectOfType <ZEDManager>();
//If this happenend when only using a primary ZED for collisions but there are multiple ZEDs, collisions will be
//calculated using an arbitrary camera. Warn the user.
if (testCollisionsUsingAllZEDs == false && ZEDManager.GetInstances().Count > 1)
{
Debug.LogWarning("Warning: ZEDProjectile's zedManager value was not specified, resulting in assigning to first available " +
" camera, but there are multiple cameras in the scene. This can cause strange collision test behavior.");
}
}
if (!leftcamera)
{
leftcamera = zedManager.GetLeftCamera();
}
}
// Use this for initialization
void Awake()
{
if (!zedManager)
{
ZEDManager.GetInstance(sl.ZED_CAMERA_ID.CAMERA_ID_01);
}
if (!cam)
{
cam = zedManager.GetLeftCamera();
}
zedManager.OnZEDReady += DrawFrustum;
if (!nearPlaneLineRend)
{
nearPlaneLineRend = GetComponent <LineRenderer>();
}
CreateEdgeLinesAndPlanes();
}
private void OnZEDReady()
{
if (startObjectDetectionAutomatically && !zedManager.IsObjectDetectionRunning)
{
zedManager.StartObjectDetection();
}
//Enforce some specific settings on the canvas that are needed for things to line up.
canvas.renderMode = RenderMode.ScreenSpaceCamera;
canvas.worldCamera = zedManager.GetLeftCamera();
//Canvas needs to have its plane distance set within the camera's view frustum.
canvas.planeDistance = 1;
CanvasScaler scaler = canvas.GetComponent <CanvasScaler>();
if (!scaler)
{
scaler = canvas.gameObject.AddComponent <CanvasScaler>();
}
scaler.uiScaleMode = CanvasScaler.ScaleMode.ScaleWithScreenSize;
scaler.referenceResolution = new Vector2(zedManager.zedCamera.ImageWidth, zedManager.zedCamera.ImageHeight);
}
/// <summary>
/// Detects the floor plane. Replaces the current floor plane, if there is one, unlike DetectPlaneAtHit().
/// If a floor is detected, also assigns the user's height from the floor to estimatedPlayerHeight.
/// <para>Note that this can't be called the first frame the ZED is ready. Use a coroutine to wait one frame,
/// or until ZEDManager.isZEDReady is true.</para>
/// </summary>
/// <returns><c>true</c>, if floor plane was detected, <c>false</c> otherwise.</returns>
public bool DetectFloorPlane(ZEDManager manager, bool auto)
{
if (!manager.IsZEDReady)
{
return(false); //Do nothing if the ZED isn't finished initializing.
}
sl.ZEDCamera zedcam = manager.zedCamera;
Camera leftcamera = manager.GetLeftCamera();
ZEDPlaneGameObject.PlaneData plane = new ZEDPlaneGameObject.PlaneData();
if (zedcam.findFloorPlane(ref plane, out estimatedPlayerHeight, Quaternion.identity, Vector3.zero) == sl.ERROR_CODE.SUCCESS) //We found a plane.
{
int numVertices, numTriangles = 0;
zedcam.convertFloorPlaneToMesh(planeMeshVertices, planeMeshTriangles, out numVertices, out numTriangles);
if (numVertices > 0 && numTriangles > 0)
{
Vector3[] worldPlaneVertices = new Vector3[numVertices];
int[] worldPlaneTriangles = new int[numTriangles];
TransformCameraToLocalMesh(leftcamera.transform, planeMeshVertices, planeMeshTriangles, worldPlaneVertices, worldPlaneTriangles, numVertices, numTriangles, plane.PlaneCenter);
hasDetectedFloor = true;
if (!floorPlaneGO)
{ //Make the GameObject.
floorPlaneGO = new GameObject("Floor Plane");
floorPlaneGO.transform.SetParent(holder.transform);
}
if (!floorPlaneGO) //Make the GameObject.
{
floorPlaneGO = new GameObject("Floor Plane");
floorPlaneGO.transform.SetParent(holder.transform);
}
//Move the GameObject to the center of the plane. Note that the plane data's center is relative to the camera.
floorPlaneGO.transform.position = leftcamera.transform.position; //Add the camera's world position
floorPlaneGO.transform.position += leftcamera.transform.rotation * plane.PlaneCenter; //Add the center of the plane
if (!floorPlane) //Add a new ZEDPlaneGameObject to the floor plane if it doesn't already exist.
{
floorPlane = floorPlaneGO.AddComponent <ZEDPlaneGameObject>();
}
if (!floorPlane.IsCreated) //Call ZEDPlaneGameObject.Create() on the floor ZEDPlaneGameObject if it hasn't yet been run.
{
if (overrideMaterial != null)
{
floorPlane.Create(leftcamera, plane, worldPlaneVertices, worldPlaneTriangles, 0, overrideMaterial);
}
else
{
floorPlane.Create(leftcamera, plane, worldPlaneVertices, worldPlaneTriangles, 0);
}
floorPlane.SetPhysics(addPhysicsOption);
}
else //Update the ZEDPlaneGameObject with the new plane's data.
{
floorPlane.UpdateFloorPlane(!auto, plane, worldPlaneVertices, worldPlaneTriangles, overrideMaterial);
floorPlane.SetPhysics(addPhysicsOption);
}
return(true);
}
}
return(false);
}
/// <summary>
/// Creates canvas(es) and canvas elements depending on whether the ZED rig is mono (ZED_Rig_Mono)
/// or stereo (ZED_Rig_Stereo).
/// </summary>
private void Awake()
{
zedManager = GetComponent <ZEDManager>();
oldInitStatus = sl.ERROR_CODE.ERROR_CODE_LAST;
if (!zedManager.IsStereoRig) //Without VR, we use a Screen Space - Overlay canvas.
{
//Instantiate the mono warning prefab and set basic settings for it.
warningmono = Instantiate(Resources.Load("PrefabsUI/Warning") as GameObject, transform);
warningmono.SetActive(true);
warningmono.GetComponent <Canvas>().renderMode = RenderMode.ScreenSpaceCamera;
//Set the target camera to whichever mono camera in the rig has the highest depth.
Camera highestdepthzedcam = zedManager.GetLeftCamera();
if (zedManager.GetRightCamera() != null && (highestdepthzedcam == null || zedManager.GetRightCamera().depth > highestdepthzedcam.depth))
{
highestdepthzedcam = zedManager.GetRightCamera();
}
warningmono.GetComponent <Canvas>().worldCamera = highestdepthzedcam;
textmono = warningmono.GetComponentInChildren <UnityEngine.UI.Text>();
textmono.color = Color.white;
if (!sl.ZEDCamera.CheckPlugin())
{
textmono.text = ZEDLogMessage.Error2Str(ZEDLogMessage.ERROR.SDK_NOT_INSTALLED);
}
imagemono = warningmono.transform.GetChild(0).GetChild(1).gameObject;
imagemono.transform.parent.gameObject.SetActive(true);
ready = false;
}
else //In VR, we use two Screen Space - Camera canvases, one for each eye.
{
//Instantiate the left warning prefab and set basic settings for it.
warningleft = Instantiate(Resources.Load("PrefabsUI/Warning_VR") as GameObject, zedManager.GetLeftCameraTransform());
warningleft.SetActive(true);
warningleft.GetComponent <Canvas>().worldCamera = zedManager.GetLeftCamera();
warningleft.GetComponent <Canvas>().planeDistance = 1;
textleft = warningleft.GetComponentInChildren <UnityEngine.UI.Text>();
textleft.color = Color.white;
imageleft = warningleft.transform.GetChild(0).GetChild(1).gameObject;
imageleft.transform.parent.gameObject.SetActive(true);
//Instantiate the right warning prefab and set basic settings for it.
warningright = Instantiate(Resources.Load("PrefabsUI/Warning_VR") as GameObject, zedManager.GetRightCameraTransform());
warningright.SetActive(true);
warningright.GetComponent <Canvas>().worldCamera = zedManager.GetRightCamera();
warningright.GetComponent <Canvas>().planeDistance = 1;
textright = warningright.GetComponentInChildren <UnityEngine.UI.Text>();
textright.color = Color.white;
imageright = warningright.transform.GetChild(0).GetChild(1).gameObject;
imageright.transform.parent.gameObject.SetActive(true);
if (!sl.ZEDCamera.CheckPlugin()) //Warn the use there's no SDK installed.
{
textleft.text = ZEDLogMessage.Error2Str(ZEDLogMessage.ERROR.SDK_NOT_INSTALLED);
textright.text = ZEDLogMessage.Error2Str(ZEDLogMessage.ERROR.SDK_NOT_INSTALLED);
}
ready = false;
}
}
