public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorFlags = dwFlags;
kinectSensor = KinectSensor.GetDefault();
if(kinectSensor == null)
return null;
coordMapper = kinectSensor.CoordinateMapper;
this.bodyCount = kinectSensor.BodyFrameSource.BodyCount;
sensorData.bodyCount = this.bodyCount;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = -0.05f;
sensorData.faceOverlayOffset = -0.04f;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
if(!bUseMultiSource)
bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();
bodyData = new Body[sensorData.bodyCount];
}
var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
sensorData.colorImageWidth = frameDesc.Width;
sensorData.colorImageHeight = frameDesc.Height;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
if(!bUseMultiSource)
colorFrameReader = kinectSensor.ColorFrameSource.OpenReader();
sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels];
}
sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width;
sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
if(!bUseMultiSource)
depthFrameReader = kinectSensor.DepthFrameSource.OpenReader();
sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
if(!bUseMultiSource)
bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader();
sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if(!bUseMultiSource)
infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader();
sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels];
}
//if(!kinectSensor.IsOpen)
{
//Debug.Log("Opening sensor, available: " + kinectSensor.IsAvailable);
kinectSensor.Open();
}
float fWaitTime = Time.realtimeSinceStartup + 3f;
while(!kinectSensor.IsAvailable && Time.realtimeSinceStartup < fWaitTime)
{
// wait for sensor to open
}
Debug.Log("K2-sensor " + (kinectSensor.IsOpen ? "opened" : "closed") +
", available: " + kinectSensor.IsAvailable);
if(bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen)
{
multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)((int)dwFlags & 0x3F));
}
return sensorData;
}
void Start()
{
kinectManager = KinectManager.Instance;
sensorData = kinectManager ? kinectManager.GetSensorData(sensorIndex) : null;
if (sensorData != null && sensorData.sensorInterface != null)
{
// cache space tables
Debug.Log("Caching space tables of sensor " + sensorIndex + "...");
sensorData.sensorInterface.GetDepthCameraSpaceTable(sensorData);
sensorData.sensorInterface.GetColorCameraSpaceTable(sensorData);
// init servers
Debug.Log("Initing network servers of sensor " + sensorIndex + "...");
KinectInterop.FrameSource dwFlags = ((DepthSensorBase)sensorData.sensorInterface).frameSourceFlags;
InitNetServers(dwFlags);
}
else
{
Debug.LogError("KinectManager not found or not initialized.");
if (serverStatusText)
{
serverStatusText.text = "KinectManager not found or not initialized.";
}
}
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = 6;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
sensorData.colorImageWidth = 1920;
sensorData.colorImageHeight = 1080;
// flip color image vertically
sensorData.colorImageScale = new Vector3(1f, -1f, 1f);
sensorData.depthImageWidth = 512;
sensorData.depthImageHeight = 424;
// look for face-tracking manager
faceManager = GameObject.FindObjectOfType <FacetrackingManager>();
bFaceManagerAvailable = faceManager != null;
bFaceTrackingInited = false;
Debug.Log("DummyK2-sensor opened");
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = 6;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
sensorData.colorImageWidth = 1920;
sensorData.colorImageHeight = 1080;
// flip color image vertically
sensorData.colorImageScale = new Vector3(1f, -1f, 1f);
sensorData.depthImageWidth = 512;
sensorData.depthImageHeight = 424;
Debug.Log("DummyK2-sensor opened");
return(sensorData);
}
public virtual KinectInterop.SensorData OpenSensor(KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save the parameters for later
frameSourceFlags = dwFlags;
isSyncDepthAndColor = bSyncDepthAndColor && ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) && ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0);
//isSyncBodyAndDepth = bSyncBodyAndDepth && ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) && ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0);
return(null);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorFlags = dwFlags;
kinectSensor = KinectSensor.GetDefault();
if (kinectSensor == null)
{
return(null);
}
coordMapper = kinectSensor.CoordinateMapper;
this.bodyCount = kinectSensor.BodyFrameSource.BodyCount;
sensorData.bodyCount = this.bodyCount;
sensorData.jointCount = 25;
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
if (!bUseMultiSource)
{
bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();
}
bodyData = new Body[sensorData.bodyCount];
}
//if(!kinectSensor.IsOpen)
{
//Debug.Log("Opening sensor, available: " + kinectSensor.IsAvailable);
kinectSensor.Open();
}
float fWaitTime = Time.realtimeSinceStartup + 3f;
while (!kinectSensor.IsAvailable && Time.realtimeSinceStartup < fWaitTime)
{
// wait for sensor to open
}
Debug.Log("K2-sensor " + (kinectSensor.IsOpen ? "opened" : "closed") +
", available: " + kinectSensor.IsAvailable);
if (bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen)
{
multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)((int)dwFlags & 0x3F));
}
return(sensorData);
}
public override KinectInterop.SensorData OpenSensor(KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save initial parameters
base.OpenSensor(dwFlags, bSyncDepthAndColor, bSyncBodyAndDepth);
List <KinectInterop.SensorDeviceInfo> alSensors = GetAvailableSensors();
if (deviceIndex < 0 || deviceIndex >= alSensors.Count)
{
return(null);
}
sensorDeviceId = alSensors[deviceIndex].sensorId;
sensorPlatform = KinectInterop.DepthSensorPlatform.DummyK4A;
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.colorImageWidth = 1920; // 1080p
sensorData.colorImageHeight = 1080;
sensorData.depthImageWidth = 640; // NFOV Unbinned
sensorData.depthImageHeight = 576;
sensorData.depthCamIntr = JsonUtility.FromJson <KinectInterop.CameraIntrinsics>(jsonDepthCamIntr);
sensorData.colorCamIntr = JsonUtility.FromJson <KinectInterop.CameraIntrinsics>(jsonColorCamIntr);
sensorData.depth2ColorExtr = JsonUtility.FromJson <KinectInterop.CameraExtrinsics>(jsonDepth2ColorExtr);
sensorData.color2DepthExtr = JsonUtility.FromJson <KinectInterop.CameraExtrinsics>(jsonColor2DepthExtr);
// flip color & depth image vertically
sensorData.colorImageScale = new Vector3(-1f, -1f, 1f);
sensorData.depthImageScale = new Vector3(-1f, -1f, 1f);
sensorData.infraredImageScale = new Vector3(-1f, -1f, 1f);
sensorData.sensorSpaceScale = new Vector3(-1f, -1f, 1f);
// depth camera offset & matrix z-flip
sensorRotOffset = new Vector3(6f, 0f, 0f); // the depth camera is tilted 6 degrees downwards
sensorRotFlipZ = true;
sensorRotIgnoreY = true;
// color camera data & intrinsics
sensorData.colorImageFormat = TextureFormat.BGRA32;
sensorData.colorImageStride = 4; // 4 bytes per pixel
Debug.Log("DummyK4A-sensor opened");
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
if (sensorData == null)
{
sensorData = new KinectInterop.SensorData();
}
sensorFlags = dwFlags;
sensorData.bodyCount = 6;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
// by-default image widths & heights
sensorData.colorImageWidth = 1920;
sensorData.colorImageHeight = 1080;
// flip color image vertically
sensorData.colorImageScale = new Vector3(1f, -1f, 1f);
sensorData.depthImageWidth = 512;
sensorData.depthImageHeight = 424;
_saveLatestFrames = bUseMultiSource;
Task task = null;
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
task = InitializeKinect();
}, true);
while (task != null && !task.IsCompleted)
{
task.Wait(100);
}
return((_kinectSensor != null && _kinectSensor.IsActive) ? sensorData : null);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = 6;
sensorData.jointCount = 20;
sensorData.depthCameraFOV = 46.6f;
sensorData.colorCameraFOV = 48.6f;
sensorData.depthCameraOffset = 0.01f;
sensorData.faceOverlayOffset = 0.01f;
sensorData.colorImageWidth = 640;
sensorData.colorImageHeight = 480;
sensorData.depthImageWidth = 640;
sensorData.depthImageHeight = 480;
Debug.Log("DummyK1-sensor opened");
return(sensorData);
}
// gets the frame-source flags
private KinectInterop.FrameSource GetFrameSourceFlags()
{
KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeNone;
if (getDepthFrames != DepthTextureType.None)
{
dwFlags |= KinectInterop.FrameSource.TypeDepth;
}
if (getColorFrames != ColorTextureType.None)
{
dwFlags |= KinectInterop.FrameSource.TypeColor;
}
if (getInfraredFrames != InfraredTextureType.None)
{
dwFlags |= KinectInterop.FrameSource.TypeInfrared;
}
if (getPoseFrames != PoseUsageType.None)
{
dwFlags |= KinectInterop.FrameSource.TypePose;
}
return(dwFlags);
}
public override KinectInterop.SensorData OpenSensor(KinectManager kinectManager, KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save initial parameters
base.OpenSensor(kinectManager, dwFlags, bSyncDepthAndColor, bSyncBodyAndDepth);
List <KinectInterop.SensorDeviceInfo> alSensors = GetAvailableSensors();
if (deviceIndex < 0 || deviceIndex >= alSensors.Count)
{
return(null);
}
sensorDeviceId = alSensors[deviceIndex].sensorId;
sensorPlatform = KinectInterop.DepthSensorPlatform.DummyK4A;
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.sensorIntPlatform = sensorPlatform;
sensorData.sensorId = alSensors[deviceIndex].sensorId;
sensorData.sensorName = alSensors[deviceIndex].sensorName;
sensorData.sensorCaps = alSensors[deviceIndex].sensorCaps;
sensorData.colorImageWidth = 1920; // 1080p
sensorData.colorImageHeight = 1080;
sensorData.depthImageWidth = 640; // NFOV Unbinned
sensorData.depthImageHeight = 576;
sensorData.depthCamIntr = JsonUtility.FromJson <KinectInterop.CameraIntrinsics>(jsonDepthCamIntr);
sensorData.colorCamIntr = JsonUtility.FromJson <KinectInterop.CameraIntrinsics>(jsonColorCamIntr);
sensorData.depth2ColorExtr = JsonUtility.FromJson <KinectInterop.CameraExtrinsics>(jsonDepth2ColorExtr);
sensorData.color2DepthExtr = JsonUtility.FromJson <KinectInterop.CameraExtrinsics>(jsonColor2DepthExtr);
float[] r = sensorData.depth2ColorExtr.rotation;
float[] t = sensorData.depth2ColorExtr.translation;
depth2colorCamMat = new Matrix4x4(new Vector4(r[0], r[3], r[6], 0), new Vector4(r[1], r[4], r[7], 0), new Vector4(r[2], r[5], r[8], 0), new Vector4(t[0] * 0.001f, t[1] * 0.001f, t[2] * 0.001f, 1));
//Debug.Log("Depth2colorCamMat Pos: " + (Vector3)depth2colorCamMat.GetColumn(3) + ", Rot: " + depth2colorCamMat.rotation.eulerAngles);
// flip color & depth image vertically
sensorData.colorImageScale = new Vector3(-1f, -1f, 1f);
sensorData.depthImageScale = new Vector3(-1f, -1f, 1f);
sensorData.infraredImageScale = new Vector3(-1f, -1f, 1f);
sensorData.sensorSpaceScale = new Vector3(-1f, -1f, 1f);
sensorData.unitToMeterFactor = 0.001f;
// depth camera offset & matrix z-flip
sensorRotOffset = Vector3.zero; // new Vector3(6f, 0f, 0f); // the depth camera is tilted 6 degrees downwards
sensorRotFlipZ = true;
sensorRotIgnoreY = true;
// color camera data & intrinsics
sensorData.colorImageFormat = TextureFormat.BGRA32;
sensorData.colorImageStride = 4; // 4 bytes per pixel
if (consoleLogMessages)
{
Debug.Log("D" + deviceIndex + " DummyK4A-sensor opened");
}
return(sensorData);
}
// initializes the sensor net servers
private void InitNetServers(KinectInterop.FrameSource dwFlags)
{
try
{
lastKeepAliveFrameTime = 0;
lastSensorPoseStr = string.Empty;
int minPort = int.MaxValue, maxPort = int.MinValue;
controlFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Control, "ControlServer", sbConsole);
controlFrameServer.ReceivedMessage += new ReceivedMessageEventHandler(ControlFrameReceived);
minPort = Mathf.Min(minPort, controlFrameServer.serverPort); maxPort = Mathf.Max(maxPort, controlFrameServer.serverPort);
if (compressRawFrames)
{
controlFrameCompressor = LZ4CompressorFactory.CreateNew();
}
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0 && sensorData.colorImageTexture != null)
{
colorFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Color, "ColorServer", sbConsole);
minPort = Mathf.Min(minPort, colorFrameServer.serverPort); maxPort = Mathf.Max(maxPort, colorFrameServer.serverPort);
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
depthFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Depth, "DepthServer", sbConsole);
minPort = Mathf.Min(minPort, depthFrameServer.serverPort); maxPort = Mathf.Max(maxPort, depthFrameServer.serverPort);
if (compressRawFrames)
{
depthFrameCompressor = LZ4CompressorFactory.CreateNew();
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
infraredTex2d = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight, TextureFormat.ARGB32, false);
infraredFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Infrared, "InfraredServer", sbConsole);
minPort = Mathf.Min(minPort, infraredFrameServer.serverPort); maxPort = Mathf.Max(maxPort, infraredFrameServer.serverPort);
}
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
bodyDataFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.BodyData, "BodyDataServer", sbConsole);
minPort = Mathf.Min(minPort, bodyDataFrameServer.serverPort); maxPort = Mathf.Max(maxPort, bodyDataFrameServer.serverPort);
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0 && serveBodyIndexFrames)
{
bodyIndexFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.BodyIndex, "BodyIndexServer", sbConsole);
minPort = Mathf.Min(minPort, bodyIndexFrameServer.serverPort); maxPort = Mathf.Max(maxPort, bodyIndexFrameServer.serverPort);
if (compressRawFrames)
{
bodyIndexFrameCompressor = LZ4CompressorFactory.CreateNew();
}
}
if ((dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
poseFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Pose, "PoseServer", sbConsole);
minPort = Mathf.Min(minPort, poseFrameServer.serverPort); maxPort = Mathf.Max(maxPort, poseFrameServer.serverPort);
}
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0 && (dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
depth2colorFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Depth2Color, "TColorServer", sbConsole);
minPort = Mathf.Min(minPort, depth2colorFrameServer.serverPort); maxPort = Mathf.Max(maxPort, depth2colorFrameServer.serverPort);
}
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0 && (dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
color2depthFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Color2Depth, "TDepthServer", sbConsole);
minPort = Mathf.Min(minPort, color2depthFrameServer.serverPort); maxPort = Mathf.Max(maxPort, color2depthFrameServer.serverPort);
if (compressRawFrames)
{
color2depthFrameCompressor = LZ4CompressorFactory.CreateNew();
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0 && (dwFlags & KinectInterop.FrameSource.TypeDepth) != 0 && (dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
color2bodyIndexFrameServer = new TcpNetServer(baseListenPort + (int)NetMessageType.Color2BodyIndex, "TBodyIndex", sbConsole);
minPort = Mathf.Min(minPort, color2bodyIndexFrameServer.serverPort); maxPort = Mathf.Max(maxPort, color2bodyIndexFrameServer.serverPort);
if (compressRawFrames)
{
color2bodyIndexFrameCompressor = LZ4CompressorFactory.CreateNew();
}
}
if (serverStatusText)
{
string localName = GetLocalNameOrIP();
serverStatusText.text = string.Format("NetServer listens on {0}:{1}-{2}", localName, minPort, maxPort);
}
}
catch (System.Exception ex)
{
Debug.LogError("Error initing NetServer: " + ex.Message);
Debug.Log(ex);
if (serverStatusText)
{
serverStatusText.text = string.Format("Error initing NetServer: {0}", ex.Message);
}
}
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
bool bColor = false, bDepth = false, bInfrared = false;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
bColor = true;
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
bDepth = true;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
bDepth = true;
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
bInfrared = true;
}
int hr = InitNative(bDepth, bColor, bInfrared);
if (hr == 0)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.SkeletonCount;
sensorData.jointCount = Constants.JointCount;
sensorData.depthCameraFOV = 46.6f;
sensorData.colorCameraFOV = 48.6f;
sensorData.depthCameraOffset = 0.01f;
sensorData.faceOverlayOffset = 0.01f;
sensorData.colorImageWidth = GetColorWidth();
sensorData.colorImageHeight = GetColorHeight();
sensorData.depthImageWidth = GetDepthWidth();
sensorData.depthImageHeight = GetDepthHeight();
usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight;
usersColorMap = new byte[usersClrSize * 3];
usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
usersLabelMap = new short[usersMapSize];
usersDepthMap = new short[usersMapSize];
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
StartLookingForUsers(IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
SetSkeletonSmoothing(Constants.SmoothingFactor);
}
return(sensorData);
}
else
{
Debug.LogError("InitKinectSensor failed: " + Marshal.PtrToStringAnsi(GetLastErrorString()));
}
return(null);
}
// locates and starts the available depth-sensors and their interfaces
private void StartDepthSensors()
{
try
{
// try to initialize the available sensors
KinectInterop.FrameSource dwFlags = GetFrameSourceFlags();
// locate the available depth-sensor interfaces in the scene
List <DepthSensorBase> sensorInts = new List <DepthSensorBase>();
sensorInts.AddRange(gameObject.GetComponents <DepthSensorBase>()); // FindObjectsOfType<MonoBehaviour>();
sensorInts.AddRange(gameObject.GetComponentsInChildren <DepthSensorBase>());
if (sensorInts.Count == 0)
{
// by-default add K4A interface
transform.position = new Vector3(0f, 1f, 0f);
transform.rotation = Quaternion.identity;
DepthSensorBase sensorInt = gameObject.AddComponent <Kinect4AzureInterface>();
sensorInts.Add(sensorInt);
}
for (int i = 0; i < sensorInts.Count; i++)
{
if (sensorInts[i] is DepthSensorBase)
{
DepthSensorBase sensorInt = (DepthSensorBase)sensorInts[i];
if (!sensorInt.enabled || sensorInt.deviceStreamingMode == KinectInterop.DeviceStreamingMode.Disabled || sensorInt.deviceIndex < 0)
{
Debug.Log(string.Format("S{0}: {1} disabled.", i, sensorInt.GetType().Name));
continue;
}
try
{
Debug.Log(string.Format("Opening S{0}: {1}, device-index: {2}", i, sensorInt.GetType().Name, sensorInt.deviceIndex));
KinectInterop.SensorData sensorData = sensorInt.OpenSensor(dwFlags, syncDepthAndColor, false);
if (sensorData != null)
{
//Debug.Log("Succeeded opening " + sensorInt.GetType().Name);
sensorData.sensorInterface = sensorInt;
KinectInterop.InitSensorData(sensorData, this);
sensorInterfaces.Add(sensorInt);
sensorDatas.Add(sensorData);
if (pollFramesInThread)
{
sensorData.threadStopEvent = new AutoResetEvent(false);
sensorData.pollFramesThread = new Thread(() => PollFramesThread(sensorData));
sensorData.pollFramesThread.IsBackground = true;
sensorData.pollFramesThread.Start();
}
}
}
catch (Exception ex)
{
Debug.LogException(ex);
Debug.LogError("Failed opening " + sensorInt.GetType().Name + ", device-index: " + sensorInt.deviceIndex);
}
}
}
Debug.Log(string.Format("{0} sensor(s) opened.", sensorDatas.Count));
// set initialization status
if (sensorInterfaces.Count > 0)
{
kinectInitialized = true;
}
else
{
kinectInitialized = false;
string sErrorMessage = "No suitable depth-sensor found. Please check the connected devices and installed SDKs.";
Debug.LogError(sErrorMessage);
if (statusInfoText != null)
{
statusInfoText.text = sErrorMessage;
}
}
}
//catch (DllNotFoundException ex)
//{
// string message = ex.Message + " cannot be loaded. Please check the respective SDK installation.";
// Debug.LogError(message);
// Debug.LogException(ex);
// if (calibrationText != null)
// {
// calibrationText.text = message;
// }
// return;
//}
catch (Exception ex)
{
string message = ex.Message;
Debug.LogError(message);
Debug.LogException(ex);
if (statusInfoText != null)
{
statusInfoText.text = message;
}
return;
}
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
// init interface
int hr = InitAstraInterface();
if (hr != 0)
{
return(null);
}
sensorFlags = dwFlags;
bMultiSource = bUseMultiSource;
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
hr = OpenAstraColorStream();
// try to get a color frame
hr = PollColorFrame(500);
if (hr == 0)
{
ReleaseColorFrame();
Debug.Log("Astra-sensor detected");
}
if (hr != 0)
{
bWebColorStream = true;
isAstraPro = true;
Debug.Log("AstraPro camera detected.");
for (int i = 0; i < WebCamTexture.devices.Length; i++)
{
Debug.Log(WebCamTexture.devices [i].name);
if (WebCamTexture.devices[i].name.IndexOf("astra", StringComparison.CurrentCultureIgnoreCase) >= 0)
{
colorWebCam = new WebCamTexture(WebCamTexture.devices[i].name, 640, 480, 30);
break;
}
}
if (colorWebCam)
{
colorWebCam.Play();
sensorData.colorImageWidth = colorWebCam.width;
sensorData.colorImageHeight = colorWebCam.height;
sensorData.colorImageTexture = colorWebCam;
//Debug.Log("Webcam - vMirrored: " + colorWebCam.videoVerticallyMirrored + ", rotAngle: " + colorWebCam.videoRotationAngle);
}
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
hr = OpenAstraDepthStream();
// try to get a depth frame
hr = PollDepthFrame(500);
if (hr == 0)
{
ReleaseDepthFrame();
}
}
if (((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) ||
((dwFlags & KinectInterop.FrameSource.TypeBody) != 0))
{
hr = OpenAstraBodyStream();
if (hr == 0)
{
obtBody = new ObtBody();
obtBody.joints = new ObtJoint[OBT_MAX_JOINTS];
obtBodyInited = true;
}
}
// if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
// {
// }
sensorData.bodyCount = Constants.SkeletonCount;
sensorData.jointCount = Constants.JointCount;
sensorData.depthCameraFOV = 45.64f;
sensorData.colorCameraFOV = 45.64f;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
if (!bWebColorStream)
{
sensorData.colorImageWidth = GetColorWidth();
sensorData.colorImageHeight = GetColorHeight();
// flip color image vertically
sensorData.colorImageScale = new Vector3(1f, -1f, 1f);
}
else
{
// flip color image horizontally
sensorData.colorImageScale = new Vector3(-1f, 1f, 1f);
}
sensorData.depthImageWidth = GetDepthWidth();
sensorData.depthImageHeight = GetDepthHeight();
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
int colorImageSize = !colorWebCam ? (GetColorDataSize() * 4 / 3) : 0;
sensorData.colorImage = new byte[colorImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
int depthImageSize = GetDepthDataSize() / sizeof(ushort);
sensorData.depthImage = new ushort[depthImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
int bodyIndexImageSize = GetBodyIndexDataSize();
sensorData.bodyIndexImage = new byte[bodyIndexImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
int depthImageSize = GetDepthDataSize() / sizeof(ushort);
sensorData.infraredImage = new ushort[depthImageSize];
}
// setup coordinate mapper
coordMapper = new OrbbecAstraMapper();
coordMapper.SetupSpaceMapping(sensorData.depthImageWidth, sensorData.depthImageHeight, 1.0226f, 0.7966157f); // hfov, vfov in rad
coordMapper.SetupCalibrationData(isAstraPro);
// set lost-user time tolerance equal to KM
if (KinectManager.Instance != null)
{
waitTimeBeforeRemove = KinectManager.Instance.waitTimeBeforeRemove;
}
// enable depth-to-color sync, if needed
if (bMultiSource && ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) && ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0))
{
//hr = EnableDepthColorSync(1);
}
Debug.Log("OrbbecAstra sensor opened");
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
//sensorFlags = dwFlags;
kinectSensor = KinectSensor.GetDefault();
if (kinectSensor == null)
{
return(null);
}
coordMapper = kinectSensor.CoordinateMapper;
sensorData.bodyCount = kinectSensor.BodyFrameSource.BodyCount;
sensorData.jointCount = 25;
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
if (!bUseMultiSource)
{
bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();
}
bodyData = new Body[sensorData.bodyCount];
}
var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
sensorData.colorImageWidth = frameDesc.Width;
sensorData.colorImageHeight = frameDesc.Height;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
if (!bUseMultiSource)
{
colorFrameReader = kinectSensor.ColorFrameSource.OpenReader();
}
sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels];
}
sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width;
sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
if (!bUseMultiSource)
{
depthFrameReader = kinectSensor.DepthFrameSource.OpenReader();
}
sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
if (!bUseMultiSource)
{
bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader();
}
sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (!bUseMultiSource)
{
infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader();
}
sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels];
}
if (!kinectSensor.IsOpen)
{
kinectSensor.Open();
}
if (bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen)
{
multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)dwFlags);
}
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
// init interface
if (!bNuitrackInited)
{
bNuitrackInited = NuitrackInit();
if (!bNuitrackInited)
{
return(null);
}
}
sensorFlags = dwFlags;
bMultiSource = bUseMultiSource;
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
for (int i = 0; i < WebCamTexture.devices.Length; i++)
{
if (WebCamTexture.devices[i].name.IndexOf("astra", StringComparison.CurrentCultureIgnoreCase) >= 0)
{
Debug.Log(" " + WebCamTexture.devices[i].name + "- AstraPro detected.");
colorWebCam = new WebCamTexture(WebCamTexture.devices[i].name, 640, 480, 30);
break;
}
}
if (!colorWebCam)
{
colorSensor = nuitrack.ColorSensor.Create();
colorSensor.OnUpdateEvent += HandleOnColorSensorUpdateEvent;
}
else
{
bWebColorStream = true;
colorWebCam.Play();
sensorData.colorImageTexture = colorWebCam;
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
depthSensor = nuitrack.DepthSensor.Create();
// depthSensor.SetMirror (true);
depthSensor.OnUpdateEvent += HandleOnDepthSensorUpdateEvent;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
userTracker = nuitrack.UserTracker.Create();
userTracker.OnUpdateEvent += HandleOnUserTrackerUpdateEvent;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonTracker = nuitrack.SkeletonTracker.Create();
skeletonTracker.OnSkeletonUpdateEvent += HandleOnSkeletonUpdateEvent;
handTracker = nuitrack.HandTracker.Create();
handTracker.OnUpdateEvent += HandleOnHandsUpdateEvent;
gestureRecognizer = nuitrack.GestureRecognizer.Create();
gestureRecognizer.OnNewGesturesEvent += OnNewGestures;
}
// if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
// {
// }
nuitrack.Nuitrack.onIssueUpdateEvent += OnIssuesUpdate;
nuitrack.Nuitrack.Run();
sensorData.bodyCount = Constants.SkeletonCount;
sensorData.jointCount = Constants.JointCount;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
if (!bWebColorStream)
{
// // wait for color frame
// if (colorSensor != null)
// {
// colorFrame = colorSensor.GetColorFrame();
// float waitTillTime = Time.realtimeSinceStartup + 2.5f;
//
// while (colorFrame == null && Time.realtimeSinceStartup <= waitTillTime)
// {
// nuitrack.Nuitrack.Update();
// System.Threading.Thread.Sleep(50);
// colorFrame = colorSensor.GetColorFrame();
// }
// }
sensorData.colorImageWidth = colorSensor != null?colorSensor.GetOutputMode().XRes : 640;
sensorData.colorImageHeight = colorSensor != null?colorSensor.GetOutputMode().YRes : 480;
// flip color image vertically
sensorData.colorImageScale = new Vector3(1f, -1f, 1f);
}
else
{
sensorData.colorImageWidth = colorWebCam.width;
sensorData.colorImageHeight = colorWebCam.height;
// flip color image horizontally
sensorData.colorImageScale = new Vector3(-1f, 1f, 1f);
}
Debug.Log(" Color sensor: " + (colorSensor != null ? colorSensor.ToString() : "-") +
", width: " + sensorData.colorImageWidth + ", height: " + sensorData.colorImageHeight);
// // wait for depth frame
// if (depthSensor != null)
// {
// depthFrame = depthSensor.GetDepthFrame();
// float waitTillTime = Time.realtimeSinceStartup + 2.5f;
//
// while (depthFrame == null && Time.realtimeSinceStartup <= waitTillTime)
// {
// nuitrack.Nuitrack.Update();
// System.Threading.Thread.Sleep(50);
// depthFrame = depthSensor.GetDepthFrame();
// }
// }
sensorData.depthImageWidth = depthSensor != null?depthSensor.GetOutputMode().XRes : 640;
sensorData.depthImageHeight = depthSensor != null?depthSensor.GetOutputMode().YRes : 480;
Debug.Log(" Depth sensor: " + (depthSensor != null ? depthSensor.ToString() : "-") +
", width: " + sensorData.depthImageWidth + ", height: " + sensorData.depthImageHeight);
// color & depth FOV
float hfovr = colorSensor != null?colorSensor.GetOutputMode().HFOV : 0f;
float vfovr = 2f * Mathf.Atan(Mathf.Tan(hfovr / 2f) * sensorData.depthImageHeight / sensorData.depthImageWidth);
sensorData.colorCameraFOV = vfovr * Mathf.Rad2Deg;
hfovr = depthSensor != null?depthSensor.GetOutputMode().HFOV : 0f;
vfovr = 2f * Mathf.Atan(Mathf.Tan(hfovr / 2f) * sensorData.depthImageHeight / sensorData.depthImageWidth);
sensorData.depthCameraFOV = vfovr * Mathf.Rad2Deg;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
int colorImageSize = !colorWebCam ? (sensorData.colorImageWidth * sensorData.colorImageHeight * 3) : 0;
sensorData.colorImage = new byte[colorImageSize];
if (!colorWebCam)
{
sensorData.colorImageTexture2D = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGB24, false);
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
int depthImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
sensorData.depthImage = new ushort[depthImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
int bodyIndexImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
sensorData.bodyIndexImage = new byte[bodyIndexImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
int depthImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
sensorData.infraredImage = new ushort[depthImageSize];
}
// setup coordinate mapper
coordMapper = new OrbbecAstraMapper();
coordMapper.SetupSpaceMapping(sensorData.depthImageWidth, sensorData.depthImageHeight, hfovr, vfovr);
// d2c-calibration data is valid for Orbbec-Astra only (sensor id & calib.data not provided by Nuitrack)
coordMapper.SetupCalibrationData(bWebColorStream);
// set lost-user time tolerance equal to KM
if (KinectManager.Instance != null)
{
waitTimeBeforeRemove = KinectManager.Instance.waitTimeBeforeRemove;
}
Debug.Log("Nuitrack sensor opened");
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
// init interface
int hr = InitAstraInterface();
if (hr != 0)
{
return(null);
}
bMultiSource = bUseMultiSource;
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
hr = OpenAstraColorStream();
// try to get a color frame
hr = PollColorFrame(500);
if (hr == 0)
{
ReleaseColorFrame();
}
if (hr != 0)
{
bWebColorStream = true;
for (int i = 0; i < WebCamTexture.devices.Length; i++)
{
if (WebCamTexture.devices[i].name.IndexOf("astra", StringComparison.CurrentCultureIgnoreCase) >= 0)
{
colorWebCam = new WebCamTexture(WebCamTexture.devices[i].name);
break;
}
}
if (colorWebCam)
{
colorWebCam.Play();
sensorData.colorImageWidth = colorWebCam.width;
sensorData.colorImageHeight = colorWebCam.height;
sensorData.colorImageTexture = colorWebCam;
Debug.Log("Webcam - vMirrored: " + colorWebCam.videoVerticallyMirrored + ", rotAngle: " + colorWebCam.videoRotationAngle);
}
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
hr = OpenAstraDepthStream();
// try to get a depth frame
hr = PollDepthFrame(500);
if (hr == 0)
{
ReleaseDepthFrame();
}
}
if (((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) ||
((dwFlags & KinectInterop.FrameSource.TypeBody) != 0))
{
hr = OpenAstraBodyStream();
if (hr == 0)
{
obtBody = new ObtBody();
obtBody.joints = new ObtJoint[OBT_MAX_JOINTS];
obtBodyInited = true;
}
}
// if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
// {
// }
sensorData.bodyCount = Constants.SkeletonCount;
sensorData.jointCount = Constants.JointCount;
sensorData.depthCameraFOV = 49.5f;
sensorData.colorCameraFOV = 48.6f;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
if (!bWebColorStream)
{
sensorData.colorImageWidth = GetColorWidth();
sensorData.colorImageHeight = GetColorHeight();
}
// flip color image horizontally
sensorData.colorImageScale = new Vector3(-1f, 1f, 1f);
sensorData.depthImageWidth = GetDepthWidth();
sensorData.depthImageHeight = GetDepthHeight();
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
int colorImageSize = GetColorDataSize();
sensorData.colorImage = new byte[colorImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
int depthImageSize = GetDepthDataSize() / sizeof(ushort);
sensorData.depthImage = new ushort[depthImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
int bodyIndexImageSize = GetBodyIndexDataSize();
sensorData.bodyIndexImage = new byte[bodyIndexImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
int depthImageSize = GetDepthDataSize() / sizeof(ushort);
sensorData.infraredImage = new ushort[depthImageSize];
}
Debug.Log("Orbbec Astra sensor opened");
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
if((dwFlags & KinectInterop.FrameSource.TypeAudio) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesAudio;
}
FacetrackingManager[] faceManagers = GameObject.FindObjectsOfType(typeof(FacetrackingManager)) as FacetrackingManager[];
if(faceManagers != null && faceManagers.Length > 0)
{
for(int i = 0; i < faceManagers.Length; i++)
{
if(faceManagers[i].enabled)
{
//Debug.Log("Found FacetrackingManager => UsesColor");
nuiFlags |= NuiInitializeFlags.UsesColor;
break;
}
}
}
SpeechManager[] speechManagers = GameObject.FindObjectsOfType(typeof(SpeechManager)) as SpeechManager[];
if(speechManagers != null && speechManagers.Length > 0)
{
for(int i = 0; i < speechManagers.Length; i++)
{
if(speechManagers[i].enabled)
{
//Debug.Log("Found SpeechManager => UsesAudio");
nuiFlags |= NuiInitializeFlags.UsesAudio;
break;
}
}
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if(hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if(hr != 0)
{
Debug.LogError(string.Format("Error initializing KinectInteraction: hr=0x{0:X}", hr));
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
sensorData.depthCameraFOV = 46.6f;
sensorData.colorCameraFOV = 48.6f;
sensorData.depthCameraOffset = 0.01f;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return sensorData;
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return null;
}
public override KinectInterop.SensorData OpenSensor(KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save initial parameters
base.OpenSensor(dwFlags, bSyncDepthAndColor, bSyncBodyAndDepth);
// try to open the sensor or play back the recording
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if (deviceStreamingMode == KinectInterop.DeviceStreamingMode.PlayRecording)
{
if (string.IsNullOrEmpty(recordingFile))
{
Debug.LogError("Playback selected, but the path to recording file is missing.");
return(null);
}
if (!System.IO.File.Exists(recordingFile))
{
Debug.LogError("PlayRecording selected, but the recording file cannot be found: " + recordingFile);
return(null);
}
Debug.Log("Playing back: " + recordingFile);
kinectPlayback = new Playback(recordingFile);
sensorPlatform = KinectInterop.DepthSensorPlatform.Kinect4Azure;
sensorDeviceId = KinectInterop.GetFileName(recordingFile, false); // deviceIndex.ToString();
//Debug.Log("D" + deviceIndex + " ID: " + sensorDeviceId);
colorCameraMode = (ColorCameraMode)kinectPlayback.playback_config.color_resolution;
depthCameraMode = (DepthCameraMode)kinectPlayback.playback_config.depth_mode;
deviceSyncMode = kinectPlayback.playback_config.wired_sync_mode;
coordMapperCalib = kinectPlayback.playback_calibration;
playbackStartTime = 0;
Debug.Log(string.Format("color_track_enabled: {0}, depth_track_enabled: {1}, ir_track_enabled: {2}, imu_track_enabled: {3}, depth_delay_off_color_usec: {4}",
kinectPlayback.playback_config.color_track_enabled, kinectPlayback.playback_config.depth_track_enabled,
kinectPlayback.playback_config.ir_track_enabled, kinectPlayback.playback_config.imu_track_enabled,
kinectPlayback.playback_config.depth_delay_off_color_usec));
}
else
{
List <KinectInterop.SensorDeviceInfo> alSensors = GetAvailableSensors();
if (deviceIndex >= alSensors.Count)
{
Debug.Log(" D" + deviceIndex + " is not available. You can set the device index to -1, to disable it.");
return(null);
}
kinectSensor = Device.Open(deviceIndex);
if (kinectSensor == null)
{
Debug.LogError(" D" + deviceIndex + " cannot be opened.");
return(null);
}
// try to open the sensor
sensorPlatform = KinectInterop.DepthSensorPlatform.Kinect4Azure;
sensorDeviceId = kinectSensor.SerialNum;
//Debug.Log("D" + deviceIndex + " ID: " + sensorDeviceId);
if (deviceSyncMode == WiredSyncMode.Master && (dwFlags & KinectInterop.FrameSource.TypeColor) == 0)
{
// fix by Andreas Pedroni - master requires color camera on
dwFlags |= KinectInterop.FrameSource.TypeColor;
}
DeviceConfiguration kinectConfig = new DeviceConfiguration();
kinectConfig.SynchronizedImagesOnly = isSyncDepthAndColor = false;
kinectConfig.WiredSyncMode = deviceSyncMode;
kinectConfig.SuboridinateDelayOffMaster = new TimeSpan(subDeviceDelayUsec * 10);
// color
// kinectConfig.ColorFormat = ImageFormat.ColorBGRA32;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
kinectConfig.ColorResolution = (ColorResolution)colorCameraMode;
}
else
{
kinectConfig.ColorResolution = ColorResolution.Off;
}
// depth
depthCameraMode = DepthCameraMode._512_x_512_30Fps_2_88mWfov;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
kinectConfig.DepthMode = (DepthMode)depthCameraMode;
}
else
{
kinectConfig.DepthMode = DepthMode.Off;
}
Debug.Log(string.Format("DepthMode = {0}", kinectConfig.DepthMode));
Debug.Log(string.Format("ColorResolution = {0}", kinectConfig.ColorResolution));
// fps
if (colorCameraMode != ColorCameraMode._4096_x_3072_15Fps && depthCameraMode != DepthCameraMode._1024x1024_15Fps_2_21mWfov)
{
kinectConfig.CameraFPS = FPS.FPS30;
}
else
{
kinectConfig.CameraFPS = FPS.FPS15;
}
Debug.Log(string.Format("CameraFPS = {0}", kinectConfig.CameraFPS));
// infrared
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (kinectConfig.DepthMode == DepthMode.Off)
{
kinectConfig.DepthMode = DepthMode.PassiveIR;
depthCameraMode = DepthCameraMode.PassiveIR_30Fps;
}
}
// check for at least one frame type
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0 ||
(dwFlags & KinectInterop.FrameSource.TypeDepth) != 0 ||
(dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
// start the cameras
kinectSensor.StartCameras(kinectConfig);
isCamerasStarted = true;
}
else
{
Debug.LogWarning("Sensor cameras are not started, because color, depth & IR frames are all disabled.");
}
if ((dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
// start the IMU
kinectSensor.StartImu();
isImuStarted = true;
}
// get reference to the coordinate mapper
if (isCamerasStarted)
{
coordMapperCalib = kinectSensor.GetCalibration();
}
Debug.Log(string.Format("dwFlags = {0}", dwFlags));
}
// reset the frame number
//currentFrameNumber = 0;
// flip color & depth image vertically
sensorData.colorImageScale = new Vector3(-1f, -1f, 1f);
sensorData.depthImageScale = new Vector3(-1f, -1f, 1f);
sensorData.infraredImageScale = new Vector3(-1f, -1f, 1f);
sensorData.sensorSpaceScale = new Vector3(-1f, -1f, 1f);
// depth camera offset & matrix z-flip
sensorRotOffset = new Vector3(6f, 0f, 0f); // the depth camera is tilted 6 degrees downwards
sensorRotFlipZ = true;
sensorRotIgnoreY = true;
// color camera data & intrinsics
sensorData.colorImageFormat = TextureFormat.BGRA32;
sensorData.colorImageStride = 4; // 4 bytes per pixel
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
CameraCalibration colorCamera = coordMapperCalib.ColorCameraCalibration;
sensorData.colorImageWidth = colorCamera.ResolutionWidth;
sensorData.colorImageHeight = colorCamera.ResolutionHeight;
rawColorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
sensorData.colorImageTexture = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.BGRA32, false);
sensorData.colorImageTexture.wrapMode = TextureWrapMode.Clamp;
sensorData.colorImageTexture.filterMode = FilterMode.Point;
}
// depth camera data & intrinsics
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
CameraCalibration depthCamera = coordMapperCalib.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
rawDepthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
// infrared data
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (sensorData.depthImageWidth == 0 || sensorData.depthImageHeight == 0)
{
CameraCalibration depthCamera = coordMapperCalib.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
}
rawInfraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.infraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
minInfraredValue = 0f;
maxInfraredValue = 1000f;
}
// imu data
if (kinectPlayback != null || (dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
imuAhrs = new AHRS.MahonyAHRS(0.0006f, 5f); // 600us
imuTurnRot1 = Quaternion.Euler(0f, 90f, 90f);
imuTurnRot2 = Quaternion.Euler(90f, !flipImuRotation ? 90f : -90f, 0f);
//lastFlipImuRot = flipImuRotation;
}
// calibration data
if (isCamerasStarted || kinectPlayback != null)
{
GetCameraIntrinsics(CalibrationDeviceType.Color, coordMapperCalib.ColorCameraCalibration, ref sensorData.colorCamIntr);
GetCameraIntrinsics(CalibrationDeviceType.Depth, coordMapperCalib.DepthCameraCalibration, ref sensorData.depthCamIntr);
GetCameraExtrinsics(coordMapperCalib.ColorCameraCalibration.Extrinsics, ref sensorData.depth2ColorExtr);
GetCameraExtrinsics(coordMapperCalib.DepthCameraCalibration.Extrinsics, ref sensorData.color2DepthExtr);
}
// body & body-index data
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
InitBodyTracking(dwFlags, sensorData, coordMapperCalib, true);
if (isSyncBodyAndDepth && bodyTracker == null)
{
// don't sync body and depth if the body tracker can't be created
isSyncBodyAndDepth = false;
}
}
Debug.Log("Kinect4Azure-sensor opened.");
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if (hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if (hr != 0)
{
Debug.LogError("Initialization of KinectInteraction failed.");
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return(sensorData);
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return(null);
}
public KinectInterop.SensorData OpenDefaultSensor (KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if(hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if(hr != 0)
{
Debug.LogError("Initialization of KinectInteraction failed.");
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return sensorData;
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return null;
}
public override KinectInterop.SensorData OpenSensor(KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save initial parameters
base.OpenSensor(dwFlags, bSyncDepthAndColor, bSyncBodyAndDepth);
// ensure resources are in path
//KinectInterop.CopyResourceFile("depthengine_1_0.x64.dll", "depthengine_1_0.dll");
KinectInterop.CopyResourceFile("onnxruntime.dll", "onnxruntime.dll");
KinectInterop.CopyResourceFile("dnn_model.onnx", "dnn_model.onnx");
// try to open the sensor or play back the recording
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if (deviceStreamingMode == KinectInterop.DeviceStreamingMode.PlayRecording)
{
if (string.IsNullOrEmpty(recordingFile))
{
Debug.LogError("Playback selected, but the path to recording file is missing.");
return(null);
}
if (!System.IO.File.Exists(recordingFile))
{
Debug.LogError("PlayRecording selected, but the recording file cannot be found: " + recordingFile);
return(null);
}
Debug.Log("Playing back: " + recordingFile);
kinectPlayback = new Playback(recordingFile);
colorCameraMode = (ColorCameraMode)kinectPlayback.playback_config.color_resolution;
depthCameraMode = (DepthCameraMode)kinectPlayback.playback_config.depth_mode;
deviceSyncMode = kinectPlayback.playback_config.wired_sync_mode;
coordMapper = kinectPlayback.playback_calibration;
playbackStartTime = DateTime.Now.Ticks;
Debug.Log(string.Format("color_track_enabled: {0}, depth_track_enabled: {1}, ir_track_enabled: {2}, imu_track_enabled: {3}, depth_delay_off_color_usec: {4}",
kinectPlayback.playback_config.color_track_enabled, kinectPlayback.playback_config.depth_track_enabled,
kinectPlayback.playback_config.ir_track_enabled, kinectPlayback.playback_config.imu_track_enabled,
kinectPlayback.playback_config.depth_delay_off_color_usec));
}
else
{
List <KinectInterop.SensorDeviceInfo> alSensors = GetAvailableSensors();
if (deviceIndex >= alSensors.Count)
{
Debug.Log(" D" + deviceIndex + " is not available. You can set the device index to -1, to disable it.");
return(null);
}
// try to open the sensor
kinectSensor = Device.Open(deviceIndex);
if (kinectSensor == null)
{
Debug.LogError(" D" + recordingFile + " cannot be opened.");
return(null);
}
DeviceConfiguration kinectConfig = new DeviceConfiguration();
kinectConfig.SynchronizedImagesOnly = isSyncDepthAndColor;
kinectConfig.WiredSyncMode = deviceSyncMode;
kinectConfig.SuboridinateDelayOffMaster = new TimeSpan(subDeviceDelayUsec * 10);
// color
kinectConfig.ColorFormat = ImageFormat.ColorBGRA32;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
kinectConfig.ColorResolution = (ColorResolution)colorCameraMode;
}
else
{
kinectConfig.ColorResolution = ColorResolution.Off;
}
// depth
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
kinectConfig.DepthMode = (DepthMode)depthCameraMode;
}
else
{
kinectConfig.DepthMode = DepthMode.Off;
}
// fps
if (colorCameraMode != ColorCameraMode._4096_x_3072_15Fps && depthCameraMode != DepthCameraMode._1024x1024_15Fps_2_21mWfov)
{
kinectConfig.CameraFPS = FPS.FPS30;
}
else
{
kinectConfig.CameraFPS = FPS.FPS15;
}
// infrared
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
// ??
}
// start the cameras
kinectSensor.StartCameras(kinectConfig);
isCamerasStarted = true;
if ((dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
// start the IMU
kinectSensor.StartImu();
isImuStarted = true;
}
// get reference to the coordinate mapper
coordMapper = kinectSensor.GetCalibration();
}
// reset the frame number
currentFrameNumber = 0;
// flip color & depth image vertically
sensorData.colorImageScale = new Vector3(-1f, -1f, 1f);
sensorData.depthImageScale = new Vector3(-1f, -1f, 1f);
sensorData.infraredImageScale = new Vector3(-1f, -1f, 1f);
sensorData.sensorSpaceScale = new Vector3(-1f, -1f, 1f);
// color camera data & intrinsics
sensorData.colorImageFormat = TextureFormat.BGRA32;
sensorData.colorImageStride = 4; // 4 bytes per pixel
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
CameraCalibration colorCamera = coordMapper.ColorCameraCalibration;
sensorData.colorImageWidth = colorCamera.ResolutionWidth;
sensorData.colorImageHeight = colorCamera.ResolutionHeight;
rawColorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
sensorData.colorImageTexture = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.BGRA32, false);
sensorData.colorImageTexture.wrapMode = TextureWrapMode.Clamp;
sensorData.colorImageTexture.filterMode = FilterMode.Point;
}
// depth camera data & intrinsics
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
CameraCalibration depthCamera = coordMapper.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
rawDepthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
// infrared data
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (sensorData.depthImageWidth == 0 || sensorData.depthImageHeight == 0)
{
CameraCalibration depthCamera = coordMapper.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
}
rawInfraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.infraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
// calibration data
GetCameraIntrinsics(CalibrationDeviceType.Color, coordMapper.ColorCameraCalibration, ref sensorData.colorCamIntr);
GetCameraIntrinsics(CalibrationDeviceType.Depth, coordMapper.DepthCameraCalibration, ref sensorData.depthCamIntr);
GetCameraExtrinsics(coordMapper.ColorCameraCalibration.Extrinsics, ref sensorData.depth2ColorExtr);
Debug.Log("Kinect4Azure-sensor opened.");
return(sensorData);
}
public override KinectInterop.SensorData OpenSensor(KinectManager kinectManager, KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save initial parameters
base.OpenSensor(kinectManager, dwFlags, bSyncDepthAndColor, bSyncBodyAndDepth);
// color settings
int colorWidth = 0, colorHeight = 0, colorFps = 0;
ParseCameraMode(colorCameraMode.ToString(), out colorWidth, out colorHeight, out colorFps);
// depth settings
int depthWidth = 0, depthHeight = 0, depthFps = 0;
ParseCameraMode(depthCameraMode.ToString(), out depthWidth, out depthHeight, out depthFps);
try
{
m_pipeline = new Pipeline();
using (Config config = new Config())
{
if (deviceStreamingMode == KinectInterop.DeviceStreamingMode.PlayRecording)
{
if (string.IsNullOrEmpty(recordingFile))
{
Debug.LogError("PlayRecording selected, but the path to recording file is missing.");
return(null);
}
if (!System.IO.File.Exists(recordingFile))
{
Debug.LogError("PlayRecording selected, but the recording file cannot be found: " + recordingFile);
return(null);
}
sensorPlatform = KinectInterop.DepthSensorPlatform.RealSense;
sensorDeviceId = KinectInterop.GetFileName(recordingFile, false);
// playback from file
if (consoleLogMessages)
{
Debug.Log("Playing back: " + recordingFile);
}
config.EnableDeviceFromFile(recordingFile, false);
}
else
{
// get the list of available sensors
List <KinectInterop.SensorDeviceInfo> alSensors = GetAvailableSensors();
if (deviceIndex >= alSensors.Count)
{
Debug.LogError(" D" + deviceIndex + " is not available. You can set the device index to -1, to disable it.");
return(null);
}
// sensor serial number
sensorPlatform = KinectInterop.DepthSensorPlatform.RealSense;
sensorDeviceId = alSensors[deviceIndex].sensorId;
config.EnableDevice(sensorDeviceId);
// color
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//Debug.Log(string.Format("Color camera mode: {0} x {1} @ {2} FPS", colorWidth, colorHeight, colorFps));
config.EnableStream(Stream.Color, -1, colorWidth, colorHeight, Format.Rgb8, colorFps);
}
// depth
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//Debug.Log(string.Format("Depth camera mode: {0} x {1} @ {2} FPS", depthWidth, depthHeight, depthFps));
config.EnableStream(Stream.Depth, -1, depthWidth, depthHeight, Format.Z16, depthFps);
}
// infrared
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0 /**|| (dwFlags & KinectInterop.FrameSource.TypeBody) != 0*/)
{
//Debug.Log(string.Format("Infrared camera mode: {0} x {1} @ {2} FPS", depthWidth, depthHeight, depthFps));
config.EnableStream(Stream.Infrared, 1, depthWidth, depthHeight, Format.Y8, depthFps);
//config.EnableStream(Stream.Infrared, 2, depthWidth, depthHeight, Format.Y8, depthFps);
}
// pose
if ((dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
config.EnableStream(Stream.Pose, Format.SixDOF);
}
//// record to file
//if(deviceMode == KinectInterop.DepthSensorMode.CreateRecording && !string.IsNullOrEmpty(deviceFilePath))
//{
// if (!string.IsNullOrEmpty(deviceFilePath))
// {
// config.EnableRecordToFile(deviceFilePath);
// }
// else
// {
// Debug.LogError("Record selected, but the path to recording file is missing.");
// }
//}
}
activeProfile = m_pipeline.Start(config);
}
}
catch (Exception ex)
{
Debug.LogError("RealSenseInterface: " + ex.ToString());
}
// check if the profile was successfully created
if (activeProfile == null)
{
return(null);
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.sensorIntPlatform = sensorPlatform;
// flip color & depth images vertically
sensorData.colorImageScale = new Vector3(-1f, -1f, 1f);
sensorData.depthImageScale = new Vector3(-1f, -1f, 1f);
sensorData.infraredImageScale = new Vector3(-1f, -1f, 1f);
sensorData.sensorSpaceScale = new Vector3(-1f, -1f, 1f);
// depth camera offset & matrix z-flip
sensorRotOffset = Vector3.zero; // if for instance the depth camera is tilted downwards
sensorRotFlipZ = true;
sensorRotIgnoreY = false;
// color
sensorData.colorImageWidth = colorWidth;
sensorData.colorImageHeight = colorHeight;
sensorData.colorImageFormat = TextureFormat.RGB24;
sensorData.colorImageStride = 3; // 3 bytes per pixel
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
rawColorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 3];
sensorData.colorImageTexture = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGB24, false);
sensorData.colorImageTexture.wrapMode = TextureWrapMode.Clamp;
sensorData.colorImageTexture.filterMode = FilterMode.Point;
}
// depth
sensorData.depthImageWidth = depthWidth;
sensorData.depthImageHeight = depthHeight;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
rawDepthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
// infrared
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0 || (dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
rawInfraredImage1 = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
rawInfraredImage2 = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
rawInfraredImageBT = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
rawInfraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.infraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
minInfraredValue = 0f;
maxInfraredValue = 1000f;
}
if (consoleLogMessages)
{
Debug.Log("RealSense-sensor opened: " + sensorDeviceId);
}
return(sensorData);
}
