private void OnConnectionClosed(ZView sender, IntPtr connection)
{
ZView.ConnectionCloseAction action = sender.GetConnectionCloseAction(connection);
ZView.ConnectionCloseReason reason = sender.GetConnectionCloseReason(connection);
Debug.Log(string.Format("<color=green>Connection Closed: (Action={0}, Reason={1})</color>", action, reason));
}
private void OnVideoRecordingQualityChanged(ZView sender, IntPtr connection)
{
ZView.VideoRecordingQuality videoRecordingQuality =
(ZView.VideoRecordingQuality)sender.GetSettingUInt32(connection, ZView.SettingKey.VideoRecordingQuality);
Debug.Log(string.Format("<color=red>Video Recording Quality Changed: {0}</color>", videoRecordingQuality));
}
private void OnConnectionAccepted(ZView sender, IntPtr connection)
{
string connectedNodeName = sender.GetConnectedNodeName(connection);
string connectedNodeStatus = sender.GetConnectedNodeStatus(connection);
Debug.Log(string.Format("<color=green>Connection Accepted: {0} ({1})</color>", connectedNodeName, connectedNodeStatus));
}
//////////////////////////////////////////////////////////////////
// Virtual Camera Overrides
//////////////////////////////////////////////////////////////////
public override void SetUp(ZView zView, IntPtr connection, ZView.ModeSetupPhase phase)
{
switch (phase)
{
case ZView.ModeSetupPhase.Initialization:
this.UpdateImageResolution(zView, connection);
break;
case ZView.ModeSetupPhase.Completion:
// Grab the image dimensions from the connection settings.
UInt16 imageWidth = zView.GetSettingUInt16(connection, ZView.SettingKey.ImageWidth);
UInt16 imageHeight = zView.GetSettingUInt16(connection, ZView.SettingKey.ImageHeight);
// Create the render texture.
_renderTexture = new RenderTexture((int)imageWidth, (int)imageHeight, 24, RenderTextureFormat.ARGB32);
_renderTexture.filterMode = FilterMode.Point;
_renderTexture.name = "RenderTextureStandard";
_renderTexture.Create();
// Cache the render texture's native texture pointer. Per Unity documentation,
// calling GetNativeTexturePtr() when using multi-threaded rendering will
// synchronize with the rendering thread (which is a slow operation). So, only
// call and cache once upon initialization.
_nativeTexturePtr = _renderTexture.GetNativeTexturePtr();
break;
default:
break;
}
}
public override void Render(ZView zView, IntPtr connection, IntPtr receivedFrame)
{
// Check to see if the image width or height changed and update them
// accordingly.
this.UpdateImageResolution(zView, connection);
// Cache the camera's culling mask to be restored after it renders the frame.
int cullingMask = _camera.cullingMask;
// Copy the center eye camera's attributes to the standard mode primary camera.
if (CoreCamera != null)
{
_camera.CopyFrom(CoreCamera.GetComponent <Camera>());
_camera.transform.position = CoreCamera.transform.position;
_camera.transform.rotation = CoreCamera.transform.rotation;
_camera.projectionMatrix = CoreCamera.projectionMatrix;
}
// Render the scene.
_camera.cullingMask = cullingMask & ~(zView.StandardModeIgnoreLayers);
_camera.targetTexture = _renderTexture;
_camera.Render();
// Restore the camera's culling mask.
_camera.cullingMask = cullingMask;
}
private void OnVideoRecordingError(ZView sender, IntPtr connection)
{
ZView.PluginError error = sender.GetVideoRecordingError(connection);
Debug.LogError(string.Format("<color=red>Video Recording Error: {0}</color>", error));
sender.ClearVideoRecordingError(connection);
}
//////////////////////////////////////////////////////////////////
// Private Methods
//////////////////////////////////////////////////////////////////
private void UpdateImageResolution(ZView zView, IntPtr connection)
{
// Get the current viewport size.
Vector2 viewportSize = ZCoreProxy.Instance.GetViewportSize();
UInt16 imageWidth = (UInt16)viewportSize.x;
UInt16 imageHeight = (UInt16)viewportSize.y;
// Set image width and height.
if (imageWidth != _imageWidth || imageHeight != _imageHeight)
{
// Begin settings batch.
try
{
zView.BeginSettingsBatch(connection);
}
catch (PluginException e)
{
Debug.LogError(string.Format("Failed to begin settings batch for updating image resolution: {0}", e.PluginError));
return;
}
// Update image resolution.
try
{
// Update the connection's image resolution settings.
zView.SetSetting(connection, ZView.SettingKey.ImageWidth, imageWidth);
zView.SetSetting(connection, ZView.SettingKey.ImageHeight, imageHeight);
// Update the internally cached image resolution in order to check
// if the image resolution has changed in subsequent frames.
_imageWidth = imageWidth;
_imageHeight = imageHeight;
}
catch (PluginException e)
{
Debug.LogError(string.Format("Failed to set image resolution settings: {0}", e.PluginError));
}
// End settings batch.
try
{
zView.EndSettingsBatch(connection);
}
catch (PluginException e)
{
Debug.LogError(string.Format("Failed to end settings batch for updating image resolution: {0}", e.PluginError));
}
}
}
public override void Render(ZView zView, IntPtr connection, IntPtr receivedFrame)
{
// Grab a reference to the Core stereo rig's center eye camera.
if (_currentCamera == null)
{
GameObject currentCameraObject = ZCoreProxy.Instance.CurrentCameraObject;
if (currentCameraObject != null)
{
_currentCamera = currentCameraObject.GetComponent <Camera>();
if (_currentCamera == null)
{
Debug.LogError("Failed to find the stereo rig center eye camera.");
}
}
}
// Check to see if the image width or height changed and update them
// accordingly.
this.UpdateImageResolution(zView, connection);
// Cache the camera's culling mask to be restored after it renders the frame.
int cullingMask = _camera.cullingMask;
// Copy the center eye camera's attributes to the standard mode primary camera.
if (_currentCamera != null)
{
_camera.CopyFrom(_currentCamera);
// Update the camera's transform based on the center eye's view matrix.
Matrix4x4 viewMatrix = this.FlipHandedness(ZCoreProxy.Instance.GetFrustumViewMatrix(ZCoreProxy.Eye.Center));
Matrix4x4 cameraMatrix = _currentCamera.transform.localToWorldMatrix * viewMatrix.inverse;
_camera.transform.position = cameraMatrix.GetColumn(3);
_camera.transform.rotation =
Quaternion.LookRotation(
cameraMatrix.GetColumn(2),
cameraMatrix.GetColumn(1));
// Set the camera's projection matrix based on the center eye's projection matrix.
_camera.projectionMatrix = ZCoreProxy.Instance.GetFrustumProjectionMatrix(ZCoreProxy.Eye.Center);
}
// Render the scene.
_camera.cullingMask = cullingMask & ~(zView.StandardModeIgnoreLayers);
_camera.targetTexture = _renderTexture;
_camera.Render();
// Restore the camera's culling mask.
_camera.cullingMask = cullingMask;
}
//////////////////////////////////////////////////////////////////
// Section Draw Helpers
//////////////////////////////////////////////////////////////////
private void CheckZViewInitialized()
{
ZView zView = (ZView)this.target;
if (!zView.IsInitialized())
{
EditorGUILayout.HelpBox(
"Failed to properly initialize the zSpace zView SDK. As a result, most " +
"zView functionality will be disabled. Please make sure that the zView " +
"SDK runtime has been properly installed on your machine.",
MessageType.Error);
EditorGUILayout.Space();
}
}
private void DrawInfoSection()
{
ZView zView = (ZView)this.target;
_isInfoSectionExpanded = this.DrawSectionHeader("General Info", _infoIconTexture, _isInfoSectionExpanded);
if (_isInfoSectionExpanded)
{
string pluginVersion = zView.GetPluginVersion();
string runtimeVersion = zView.IsInitialized() ? zView.GetRuntimeVersion() : "Unknown";
EditorGUILayout.LabelField("Plugin Version: " + pluginVersion);
EditorGUILayout.LabelField("Runtime Version: " + runtimeVersion);
EditorGUILayout.Space();
}
}
//////////////////////////////////////////////////////////////////
// Virtual Camera Overrides
//////////////////////////////////////////////////////////////////
public override void SetUp(ZView zView, IntPtr connection, ZView.ModeSetupPhase phase)
{
switch (phase)
{
case ZView.ModeSetupPhase.Initialization:
// Do nothing.
break;
case ZView.ModeSetupPhase.Completion:
// Grab the image dimensions from the connection settings.
_imageWidth = zView.GetSettingUInt16(connection, ZView.SettingKey.ImageWidth);
_imageHeight = zView.GetSettingUInt16(connection, ZView.SettingKey.ImageHeight);
// Create the mask depth render texture (mask only).
// NOTE: This is used for both RGB or RGBA overlay.
_maskDepthRenderTexture = new RenderTexture((int)_imageWidth, (int)_imageHeight, 24, RenderTextureFormat.ARGB32);
_maskDepthRenderTexture.filterMode = FilterMode.Point;
_maskDepthRenderTexture.name = "MaskDepthRenderTexture";
_maskDepthRenderTexture.Create();
// Create the non-environment render texture (non-environment objects + mask).
// NOTE: For the RGB overlay, this is used to perform a depth render
// of the non-environment objects (excluding the mask). For the RGBA overlay,
// this is used to render non-environment objects (including the mask depth).
_nonEnvironmentRenderTexture = new RenderTexture((int)_imageWidth, (int)_imageHeight, 24, RenderTextureFormat.ARGB32);
_nonEnvironmentRenderTexture.filterMode = FilterMode.Point;
_nonEnvironmentRenderTexture.name = "NonEnvironmentRenderTexture";
_nonEnvironmentRenderTexture.Create();
// Create the final composite render texture.
// NOTE: This is used for both RGB or RGBA overlay.
_finalRenderTexture = new RenderTexture((int)_imageWidth, (int)_imageHeight, 24, RenderTextureFormat.ARGB32);
_finalRenderTexture.filterMode = FilterMode.Point;
_finalRenderTexture.name = "CompositeRenderTexture";
_finalRenderTexture.Create();
// Cache the composite render texture's native texture pointer. Per Unity documentation,
// calling GetNativeTexturePtr() when using multi-threaded rendering will
// synchronize with the rendering thread (which is a slow operation). So, only
// call and cache once upon initialization.
_nativeTexturePtr = _finalRenderTexture.GetNativeTexturePtr();
break;
default:
break;
}
}
private void OnConnectionModeSwitched(ZView sender, IntPtr connection)
{
IntPtr mode = sender.GetConnectionMode(connection);
string modeName = "None";
if (mode == sender.GetStandardMode())
{
modeName = "Standard";
}
else if (mode == sender.GetAugmentedRealityMode())
{
modeName = "Augmented Reality";
}
Debug.Log(string.Format("<color=green>Connection Mode Switched: {0}</color>", modeName));
}
private void RenderRGBA(ZView zView)
{
if (zView.ARModeEnvironmentLayers != 0)
{
// Update globals for the depth render shader.
Shader.SetGlobalFloat("_Log2FarPlusOne", (float)Math.Log(_secondaryCamera.farClipPlane + 1, 2));
// Perform a depth render of the mask.
_secondaryCamera.clearFlags = CameraClearFlags.Color;
_secondaryCamera.backgroundColor = Color.white;
_secondaryCamera.cullingMask = (1 << zView.ARModeMaskLayer);
_secondaryCamera.targetTexture = _maskDepthRenderTexture;
_secondaryCamera.RenderWithShader(_depthRenderShader, string.Empty);
// Render all non-environment objects including the box mask.
_secondaryCamera.clearFlags = CameraClearFlags.Skybox;
_secondaryCamera.backgroundColor = MASK_COLOR;
_secondaryCamera.cullingMask = _compositorCamera.cullingMask & ~(zView.ARModeEnvironmentLayers);
_secondaryCamera.targetTexture = _nonEnvironmentRenderTexture;
_secondaryCamera.Render();
// Perform the composite render of the entire scene excluding
// the mask.
_compositorCamera.cullingMask = _compositorCamera.cullingMask & ~(1 << zView.ARModeMaskLayer);
_compositorCamera.targetTexture = _finalRenderTexture;
_compositorCamera.Render();
}
else
{
// Perform a render of the entire scene including the box mask.
// NOTE: If no environment layers are set, we can optimize this
// to a single pass.
_secondaryCamera.backgroundColor = MASK_COLOR;
_secondaryCamera.targetTexture = _finalRenderTexture;
_secondaryCamera.Render();
}
}
private void RenderRGB(ZView zView)
{
// Update globals for the depth render shader.
Shader.SetGlobalFloat("_Log2FarPlusOne", (float)Math.Log(_secondaryCamera.farClipPlane + 1, 2));
// Perform a depth render of the mask.
_secondaryCamera.clearFlags = CameraClearFlags.Color;
_secondaryCamera.backgroundColor = Color.white;
_secondaryCamera.cullingMask = (1 << zView.ARModeMaskLayer);
_secondaryCamera.targetTexture = _maskDepthRenderTexture;
_secondaryCamera.RenderWithShader(_depthRenderShader, string.Empty);
// Perform a depth render of the scene excluding the mask
// layer and any environment layers.
_secondaryCamera.cullingMask = _compositorCamera.cullingMask & ~(1 << zView.ARModeMaskLayer) & ~(zView.ARModeEnvironmentLayers);
_secondaryCamera.targetTexture = _nonEnvironmentRenderTexture;
_secondaryCamera.RenderWithShader(_depthRenderShader, string.Empty);
// Perform the composite render of the entire scene excluding
// the mask.
_compositorCamera.cullingMask = _compositorCamera.cullingMask & ~(1 << zView.ARModeMaskLayer);
_compositorCamera.targetTexture = _finalRenderTexture;
_compositorCamera.Render();
}
/// <summary> /// Render a single frame. /// </summary> /// /// <param name="zView"> /// A reference to the zView API Monobehaviour script. /// </param> /// <param name="connection"> /// The connection to perform the VirtualCamera's render for. /// </param> /// <param name="receivedFrame"> /// The received frame from the specified connection. /// </param> public abstract void Render(ZView zView, IntPtr connection, IntPtr receivedFrame);
/// <summary> /// Set up the VirtualCamera. /// </summary> /// /// <remarks> /// Performs any setup related operations specific to the mode the /// VirtualCamera is associated with. This method will be called once /// per ModeSetupPhase when the specified connection has transitioned /// to the ConnectionState.ModeSetup state. /// </remarks> /// /// <param name="zView"> /// A reference to the zView API Monobehaviour script. /// </param> /// <param name="connection"> /// The connection to perform the VirtualCamera's setup phase for. /// </param> /// <param name="phase"> /// The mode setup phase for the specified connection. /// </param> public abstract void SetUp(ZView zView, IntPtr connection, ZView.ModeSetupPhase phase);
public override void Render(ZView zView, IntPtr connection, IntPtr receivedFrame)
{
// Cache whether transparency is enabled.
_isTransparencyEnabled = zView.ARModeEnableTransparency;
// Grab the viewer scale.
float viewerScale = CoreCamera.transform.lossyScale.x;
///////////////////////////////
// Camera Properties Update
///////////////////////////////
// Cache the camera's culling mask and near/far clip planes so that they
// can be restored after it renders the frame.
int originalCullingMask = _compositorCamera.cullingMask;
float originalNearClipPlane = _compositorCamera.nearClipPlane;
float originalFarClipPlane = _compositorCamera.farClipPlane;
// Grab the web cam's display space pose matrix and intrinsic values
// from the frame data.
Matrix4x4 cameraPoseMatrixInDisplaySpace = zView.GetFrameDataMatrix4x4(receivedFrame, ZView.FrameDataKey.CameraPose);
float focalLength = zView.GetFrameDataFloat(receivedFrame, ZView.FrameDataKey.CameraFocalLength);
float principalPointOffsetX = zView.GetFrameDataFloat(receivedFrame, ZView.FrameDataKey.CameraPrincipalPointOffsetX);
float principalPointOffsetY = zView.GetFrameDataFloat(receivedFrame, ZView.FrameDataKey.CameraPrincipalPointOffsetY);
float pixelAspectRatio = zView.GetFrameDataFloat(receivedFrame, ZView.FrameDataKey.CameraPixelAspectRatio);
float axisSkew = zView.GetFrameDataFloat(receivedFrame, ZView.FrameDataKey.CameraAxisSkew);
// Update the near and far clip values to account for viewer scale.
float nearClipPlane = originalNearClipPlane * viewerScale;
float farClipPlane = originalFarClipPlane * viewerScale;
// Calculate the camera's transform by transforming its corresponding
// display space pose matrix to world space.
Matrix4x4 displayToWorld =
CoreCamera.transform.parent?.localToWorldMatrix ??
Matrix4x4.identity;
Matrix4x4 worldPoseMatrix = displayToWorld * cameraPoseMatrixInDisplaySpace;
// Calculate the camera's projection matrix based on the camera intrinsic
// and near/far clip values.
Matrix4x4 projectionMatrix =
this.ComputeProjectionMatrix(
focalLength,
principalPointOffsetX,
principalPointOffsetY,
pixelAspectRatio,
axisSkew,
(float)_imageWidth,
(float)_imageHeight,
nearClipPlane,
farClipPlane);
// Update the primary camera's properties (i.e. transform, projection, etc.).
_compositorCamera.transform.position = worldPoseMatrix.GetColumn(3);
_compositorCamera.transform.rotation = Quaternion.LookRotation(worldPoseMatrix.GetColumn(2), worldPoseMatrix.GetColumn(1));
_compositorCamera.projectionMatrix = projectionMatrix;
_compositorCamera.cullingMask = _compositorCamera.cullingMask & ~(zView.ARModeIgnoreLayers);
_compositorCamera.nearClipPlane = nearClipPlane;
_compositorCamera.farClipPlane = farClipPlane;
// Copy the compositor camera's properties to the secondary camera.
_secondaryCamera.CopyFrom(_compositorCamera);
#if UNITY_5_6_OR_NEWER
//make sure this is not HDR, it's not supported
_secondaryCamera.allowHDR = false;
#endif // UNITY_5_6_OR_NEWER
///////////////////////////////
// Box Mask Update
///////////////////////////////
// Enable the box mask to be rendered by the depth camera.
// Note: The box mask will be disabled immediately after it is rendered
// by the AR depth camera so that it isn't inadvertently rendered by
// other cameras in the scene.
_boxMaskObject.SetActive(true);
// Update the box mask's transform and layer.
_boxMaskObject.transform.SetPositionAndRotation(
CoreCamera.transform.parent?.position ?? Vector3.zero,
CoreCamera.transform.parent?.rotation ?? Quaternion.identity);
_boxMaskObject.transform.localScale = CoreCamera.transform.lossyScale;
_boxMaskObject.layer = zView.ARModeMaskLayer;
// Update the box mask's size.
_boxMask.SetSize(zView.ARModeMaskSize);
// Set the box mask's cutout size to be the size of the viewport
// in viewport space (meters) since its associated transform's
// local scale accounts for viewer scale.
_boxMask.SetCutoutSize(ZProviderProxy.Instance.DisplaySize);
// Update the box mask's render queue priority.
_boxMask.SetRenderQueue(zView.ARModeMaskRenderQueue);
///////////////////////////////
// Scene Render
///////////////////////////////
if (zView.ARModeEnableTransparency)
{
this.RenderRGBA(zView);
}
else
{
this.RenderRGB(zView);
}
// Disable the box mask so that it isn't inadvertently rendered by
// any other cameras in the scene.
_boxMaskObject.SetActive(false);
// Restore the camera's culling mask and near/far clip planes.
_compositorCamera.cullingMask = originalCullingMask;
_compositorCamera.nearClipPlane = originalNearClipPlane;
_compositorCamera.farClipPlane = originalFarClipPlane;
}
private void OnConnectionModePaused(ZView sender, IntPtr connection)
{
Debug.Log("<color=green>Connection Mode Paused</color>");
}
private void OnConnectionError(ZView sender, IntPtr connection)
{
ZView.PluginError error = sender.GetConnectionError(connection);
Debug.LogError(string.Format("<color=green>Connection Error: {0}</color>", error));
}
private GlobalState()
{
// Initialize the zView context.
PluginError error = zvuInitialize(NodeType.Presenter, out _context);
if (error == PluginError.Ok)
{
// Set the context's node name.
error = zvuSetNodeName(_context, ZView.StringToNativeUtf8(this.GetProjectName()));
if (error != PluginError.Ok)
{
Debug.LogError(string.Format("Failed to set node name: ({0})", error));
}
// Set the context's node status.
error = zvuSetNodeStatus(_context, ZView.StringToNativeUtf8(string.Empty));
if (error != PluginError.Ok)
{
Debug.LogError(string.Format("Failed to set node status: ({0})", error));
}
// Get both standard and augmented reality modes.
List <ZVSupportedMode> supportedModes = new List <ZVSupportedMode>();
_modeStandard = this.GetMode(_context, CompositingMode.None, CameraMode.LocalHeadTracked);
if (_modeStandard != IntPtr.Zero)
{
supportedModes.Add(
new ZVSupportedMode
{
mode = _modeStandard,
modeAvailability = ModeAvailability.Available
});
}
_modeAugmentedReality = this.GetMode(_context, CompositingMode.AugmentedRealityCamera, CameraMode.RemoteMovable);
if (_modeAugmentedReality != IntPtr.Zero)
{
supportedModes.Add(
new ZVSupportedMode
{
mode = _modeAugmentedReality,
modeAvailability = ModeAvailability.Available
});
}
// Set the context's supported modes.
error = zvuSetSupportedModes(_context, supportedModes.ToArray(), supportedModes.Count);
if (error != PluginError.Ok)
{
Debug.LogError(string.Format("Failed to set supported modes: ({0})", error));
}
// Set the context's supported capabilities.
error = zvuSetSupportedCapabilities(_context, null, 0);
if (error != PluginError.Ok)
{
Debug.LogError(string.Format("Failed to set supported capabilities: ({0})", error));
}
// Start listening for new connections.
error = zvuStartListeningForConnections(_context, ZView.StringToNativeUtf8(string.Empty));
if (error != PluginError.Ok)
{
Debug.LogError(string.Format("Failed to start listening for connections: ({0})", error));
}
_isInitialized = true;
}
else
{
Debug.LogWarning(string.Format("Failed to initialize zView context: ({0})", error));
_isInitialized = false;
}
}
private void OnVideoRecordingPaused(ZView sender, IntPtr connection)
{
Debug.Log("<color=red>Video Recording Paused</color>");
}
private void DrawConnectionsSection()
{
ZView zView = (ZView)this.target;
_isConnectionsSectionExpanded = this.DrawSectionHeader("Connections", _connectionsIconTexture, _isConnectionsSectionExpanded, true);
if (_isConnectionsSectionExpanded)
{
bool doActiveConnectionsExist = false;
try
{
// For each connection:
int numConnections = zView.GetNumConnections();
for (int i = 0; i < numConnections; ++i)
{
// Get the connection.
IntPtr connection = zView.GetConnection(i);
// Get the connection's state.
ZView.ConnectionState connectionState = zView.GetConnectionState(connection);
if (connectionState != ZView.ConnectionState.Error)
{
// Display the connection's information.
int connectionNumber = i + 1;
string initiationStatusString = zView.WasConnectionLocallyInitiated(connection) ? "locally" : "remotely";
EditorGUILayout.LabelField(string.Format("Connection {0} (initiated {1})", connectionNumber, initiationStatusString));
EditorGUI.indentLevel++;
{
EditorGUILayout.LabelField("Name: " + zView.GetConnectedNodeName(connection));
EditorGUILayout.LabelField("Status: " + zView.GetConnectedNodeStatus(connection));
EditorGUILayout.LabelField("State: " + connectionState);
IntPtr mode = zView.GetConnectionMode(connection);
string modeString = "Unknown";
if (mode == IntPtr.Zero)
{
modeString = "None";
}
else if (mode == zView.GetStandardMode())
{
modeString = "Standard";
}
else if (mode == zView.GetAugmentedRealityMode())
{
modeString = "Augmented Reality";
}
EditorGUILayout.LabelField("Mode: " + modeString);
}
EditorGUI.indentLevel--;
EditorGUILayout.Space();
doActiveConnectionsExist = true;
}
}
}
catch
{
// TODO: Add warning.
}
if (!doActiveConnectionsExist)
{
EditorGUILayout.LabelField("No active connections");
EditorGUILayout.Space();
}
}
}
