public override void SetSensorSettings(BaseSensorSettings settings)
{
if (settings == null)
{
return;
}
base.SetSensorSettings(settings);
RSSensorSettings extSettings = (RSSensorSettings)settings;
colorCameraMode = (ColorCameraMode)extSettings.colorCameraMode;
depthCameraMode = (DepthCameraMode)extSettings.depthCameraMode;
}
public override void SetSensorSettings(BaseSensorSettings settings)
{
if (settings == null)
{
return;
}
base.SetSensorSettings(settings);
K4ASensorSettings extSettings = (K4ASensorSettings)settings;
colorCameraMode = (ColorCameraMode)extSettings.colorCameraMode;
depthCameraMode = (DepthCameraMode)extSettings.depthCameraMode;
deviceSyncMode = (WiredSyncMode)extSettings.deviceSyncMode;
subDeviceDelayUsec = extSettings.subDeviceDelayUsec;
flipImuRotation = extSettings.flipImuRotation;
playbackPosSeconds = extSettings.playbackPosSeconds;
}
public override KinectInterop.SensorData OpenSensor(KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save initial parameters
base.OpenSensor(dwFlags, bSyncDepthAndColor, bSyncBodyAndDepth);
// ensure resources are in path
//KinectInterop.CopyResourceFile("depthengine_1_0.x64.dll", "depthengine_1_0.dll");
KinectInterop.CopyResourceFile("onnxruntime.dll", "onnxruntime.dll");
KinectInterop.CopyResourceFile("dnn_model.onnx", "dnn_model.onnx");
// try to open the sensor or play back the recording
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if (deviceStreamingMode == KinectInterop.DeviceStreamingMode.PlayRecording)
{
if (string.IsNullOrEmpty(recordingFile))
{
Debug.LogError("Playback selected, but the path to recording file is missing.");
return(null);
}
if (!System.IO.File.Exists(recordingFile))
{
Debug.LogError("PlayRecording selected, but the recording file cannot be found: " + recordingFile);
return(null);
}
Debug.Log("Playing back: " + recordingFile);
kinectPlayback = new Playback(recordingFile);
colorCameraMode = (ColorCameraMode)kinectPlayback.playback_config.color_resolution;
depthCameraMode = (DepthCameraMode)kinectPlayback.playback_config.depth_mode;
deviceSyncMode = kinectPlayback.playback_config.wired_sync_mode;
coordMapper = kinectPlayback.playback_calibration;
playbackStartTime = DateTime.Now.Ticks;
Debug.Log(string.Format("color_track_enabled: {0}, depth_track_enabled: {1}, ir_track_enabled: {2}, imu_track_enabled: {3}, depth_delay_off_color_usec: {4}",
kinectPlayback.playback_config.color_track_enabled, kinectPlayback.playback_config.depth_track_enabled,
kinectPlayback.playback_config.ir_track_enabled, kinectPlayback.playback_config.imu_track_enabled,
kinectPlayback.playback_config.depth_delay_off_color_usec));
}
else
{
List <KinectInterop.SensorDeviceInfo> alSensors = GetAvailableSensors();
if (deviceIndex >= alSensors.Count)
{
Debug.Log(" D" + deviceIndex + " is not available. You can set the device index to -1, to disable it.");
return(null);
}
// try to open the sensor
kinectSensor = Device.Open(deviceIndex);
if (kinectSensor == null)
{
Debug.LogError(" D" + recordingFile + " cannot be opened.");
return(null);
}
DeviceConfiguration kinectConfig = new DeviceConfiguration();
kinectConfig.SynchronizedImagesOnly = isSyncDepthAndColor;
kinectConfig.WiredSyncMode = deviceSyncMode;
kinectConfig.SuboridinateDelayOffMaster = new TimeSpan(subDeviceDelayUsec * 10);
// color
kinectConfig.ColorFormat = ImageFormat.ColorBGRA32;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
kinectConfig.ColorResolution = (ColorResolution)colorCameraMode;
}
else
{
kinectConfig.ColorResolution = ColorResolution.Off;
}
// depth
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
kinectConfig.DepthMode = (DepthMode)depthCameraMode;
}
else
{
kinectConfig.DepthMode = DepthMode.Off;
}
// fps
if (colorCameraMode != ColorCameraMode._4096_x_3072_15Fps && depthCameraMode != DepthCameraMode._1024x1024_15Fps_2_21mWfov)
{
kinectConfig.CameraFPS = FPS.FPS30;
}
else
{
kinectConfig.CameraFPS = FPS.FPS15;
}
// infrared
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
// ??
}
// start the cameras
kinectSensor.StartCameras(kinectConfig);
isCamerasStarted = true;
if ((dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
// start the IMU
kinectSensor.StartImu();
isImuStarted = true;
}
// get reference to the coordinate mapper
coordMapper = kinectSensor.GetCalibration();
}
// reset the frame number
currentFrameNumber = 0;
// flip color & depth image vertically
sensorData.colorImageScale = new Vector3(-1f, -1f, 1f);
sensorData.depthImageScale = new Vector3(-1f, -1f, 1f);
sensorData.infraredImageScale = new Vector3(-1f, -1f, 1f);
sensorData.sensorSpaceScale = new Vector3(-1f, -1f, 1f);
// color camera data & intrinsics
sensorData.colorImageFormat = TextureFormat.BGRA32;
sensorData.colorImageStride = 4; // 4 bytes per pixel
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
CameraCalibration colorCamera = coordMapper.ColorCameraCalibration;
sensorData.colorImageWidth = colorCamera.ResolutionWidth;
sensorData.colorImageHeight = colorCamera.ResolutionHeight;
rawColorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
sensorData.colorImageTexture = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.BGRA32, false);
sensorData.colorImageTexture.wrapMode = TextureWrapMode.Clamp;
sensorData.colorImageTexture.filterMode = FilterMode.Point;
}
// depth camera data & intrinsics
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
CameraCalibration depthCamera = coordMapper.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
rawDepthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
// infrared data
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (sensorData.depthImageWidth == 0 || sensorData.depthImageHeight == 0)
{
CameraCalibration depthCamera = coordMapper.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
}
rawInfraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.infraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
// calibration data
GetCameraIntrinsics(CalibrationDeviceType.Color, coordMapper.ColorCameraCalibration, ref sensorData.colorCamIntr);
GetCameraIntrinsics(CalibrationDeviceType.Depth, coordMapper.DepthCameraCalibration, ref sensorData.depthCamIntr);
GetCameraExtrinsics(coordMapper.ColorCameraCalibration.Extrinsics, ref sensorData.depth2ColorExtr);
Debug.Log("Kinect4Azure-sensor opened.");
return(sensorData);
}
public AzureKinectSensor(ImageFormat colorImageFormat, ColorCameraMode colorCameraMode, DepthCameraMode depthCameraMode)
{
this._ColorImageFormat = colorImageFormat;
this._ColorCameraMode = colorCameraMode;
this._DepthCameraMode = depthCameraMode;
}
public override KinectInterop.SensorData OpenSensor(KinectInterop.FrameSource dwFlags, bool bSyncDepthAndColor, bool bSyncBodyAndDepth)
{
// save initial parameters
base.OpenSensor(dwFlags, bSyncDepthAndColor, bSyncBodyAndDepth);
// try to open the sensor or play back the recording
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if (deviceStreamingMode == KinectInterop.DeviceStreamingMode.PlayRecording)
{
if (string.IsNullOrEmpty(recordingFile))
{
Debug.LogError("Playback selected, but the path to recording file is missing.");
return(null);
}
if (!System.IO.File.Exists(recordingFile))
{
Debug.LogError("PlayRecording selected, but the recording file cannot be found: " + recordingFile);
return(null);
}
Debug.Log("Playing back: " + recordingFile);
kinectPlayback = new Playback(recordingFile);
sensorPlatform = KinectInterop.DepthSensorPlatform.Kinect4Azure;
sensorDeviceId = KinectInterop.GetFileName(recordingFile, false); // deviceIndex.ToString();
//Debug.Log("D" + deviceIndex + " ID: " + sensorDeviceId);
colorCameraMode = (ColorCameraMode)kinectPlayback.playback_config.color_resolution;
depthCameraMode = (DepthCameraMode)kinectPlayback.playback_config.depth_mode;
deviceSyncMode = kinectPlayback.playback_config.wired_sync_mode;
coordMapperCalib = kinectPlayback.playback_calibration;
playbackStartTime = 0;
Debug.Log(string.Format("color_track_enabled: {0}, depth_track_enabled: {1}, ir_track_enabled: {2}, imu_track_enabled: {3}, depth_delay_off_color_usec: {4}",
kinectPlayback.playback_config.color_track_enabled, kinectPlayback.playback_config.depth_track_enabled,
kinectPlayback.playback_config.ir_track_enabled, kinectPlayback.playback_config.imu_track_enabled,
kinectPlayback.playback_config.depth_delay_off_color_usec));
}
else
{
List <KinectInterop.SensorDeviceInfo> alSensors = GetAvailableSensors();
if (deviceIndex >= alSensors.Count)
{
Debug.Log(" D" + deviceIndex + " is not available. You can set the device index to -1, to disable it.");
return(null);
}
kinectSensor = Device.Open(deviceIndex);
if (kinectSensor == null)
{
Debug.LogError(" D" + deviceIndex + " cannot be opened.");
return(null);
}
// try to open the sensor
sensorPlatform = KinectInterop.DepthSensorPlatform.Kinect4Azure;
sensorDeviceId = kinectSensor.SerialNum;
//Debug.Log("D" + deviceIndex + " ID: " + sensorDeviceId);
if (deviceSyncMode == WiredSyncMode.Master && (dwFlags & KinectInterop.FrameSource.TypeColor) == 0)
{
// fix by Andreas Pedroni - master requires color camera on
dwFlags |= KinectInterop.FrameSource.TypeColor;
}
DeviceConfiguration kinectConfig = new DeviceConfiguration();
kinectConfig.SynchronizedImagesOnly = isSyncDepthAndColor = false;
kinectConfig.WiredSyncMode = deviceSyncMode;
kinectConfig.SuboridinateDelayOffMaster = new TimeSpan(subDeviceDelayUsec * 10);
// color
// kinectConfig.ColorFormat = ImageFormat.ColorBGRA32;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
kinectConfig.ColorResolution = (ColorResolution)colorCameraMode;
}
else
{
kinectConfig.ColorResolution = ColorResolution.Off;
}
// depth
depthCameraMode = DepthCameraMode._512_x_512_30Fps_2_88mWfov;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
kinectConfig.DepthMode = (DepthMode)depthCameraMode;
}
else
{
kinectConfig.DepthMode = DepthMode.Off;
}
Debug.Log(string.Format("DepthMode = {0}", kinectConfig.DepthMode));
Debug.Log(string.Format("ColorResolution = {0}", kinectConfig.ColorResolution));
// fps
if (colorCameraMode != ColorCameraMode._4096_x_3072_15Fps && depthCameraMode != DepthCameraMode._1024x1024_15Fps_2_21mWfov)
{
kinectConfig.CameraFPS = FPS.FPS30;
}
else
{
kinectConfig.CameraFPS = FPS.FPS15;
}
Debug.Log(string.Format("CameraFPS = {0}", kinectConfig.CameraFPS));
// infrared
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (kinectConfig.DepthMode == DepthMode.Off)
{
kinectConfig.DepthMode = DepthMode.PassiveIR;
depthCameraMode = DepthCameraMode.PassiveIR_30Fps;
}
}
// check for at least one frame type
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0 ||
(dwFlags & KinectInterop.FrameSource.TypeDepth) != 0 ||
(dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
// start the cameras
kinectSensor.StartCameras(kinectConfig);
isCamerasStarted = true;
}
else
{
Debug.LogWarning("Sensor cameras are not started, because color, depth & IR frames are all disabled.");
}
if ((dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
// start the IMU
kinectSensor.StartImu();
isImuStarted = true;
}
// get reference to the coordinate mapper
if (isCamerasStarted)
{
coordMapperCalib = kinectSensor.GetCalibration();
}
Debug.Log(string.Format("dwFlags = {0}", dwFlags));
}
// reset the frame number
//currentFrameNumber = 0;
// flip color & depth image vertically
sensorData.colorImageScale = new Vector3(-1f, -1f, 1f);
sensorData.depthImageScale = new Vector3(-1f, -1f, 1f);
sensorData.infraredImageScale = new Vector3(-1f, -1f, 1f);
sensorData.sensorSpaceScale = new Vector3(-1f, -1f, 1f);
// depth camera offset & matrix z-flip
sensorRotOffset = new Vector3(6f, 0f, 0f); // the depth camera is tilted 6 degrees downwards
sensorRotFlipZ = true;
sensorRotIgnoreY = true;
// color camera data & intrinsics
sensorData.colorImageFormat = TextureFormat.BGRA32;
sensorData.colorImageStride = 4; // 4 bytes per pixel
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
CameraCalibration colorCamera = coordMapperCalib.ColorCameraCalibration;
sensorData.colorImageWidth = colorCamera.ResolutionWidth;
sensorData.colorImageHeight = colorCamera.ResolutionHeight;
rawColorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
sensorData.colorImageTexture = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.BGRA32, false);
sensorData.colorImageTexture.wrapMode = TextureWrapMode.Clamp;
sensorData.colorImageTexture.filterMode = FilterMode.Point;
}
// depth camera data & intrinsics
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
CameraCalibration depthCamera = coordMapperCalib.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
rawDepthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
// infrared data
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (sensorData.depthImageWidth == 0 || sensorData.depthImageHeight == 0)
{
CameraCalibration depthCamera = coordMapperCalib.DepthCameraCalibration;
sensorData.depthImageWidth = depthCamera.ResolutionWidth;
sensorData.depthImageHeight = depthCamera.ResolutionHeight;
}
rawInfraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
sensorData.infraredImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
minInfraredValue = 0f;
maxInfraredValue = 1000f;
}
// imu data
if (kinectPlayback != null || (dwFlags & KinectInterop.FrameSource.TypePose) != 0)
{
imuAhrs = new AHRS.MahonyAHRS(0.0006f, 5f); // 600us
imuTurnRot1 = Quaternion.Euler(0f, 90f, 90f);
imuTurnRot2 = Quaternion.Euler(90f, !flipImuRotation ? 90f : -90f, 0f);
//lastFlipImuRot = flipImuRotation;
}
// calibration data
if (isCamerasStarted || kinectPlayback != null)
{
GetCameraIntrinsics(CalibrationDeviceType.Color, coordMapperCalib.ColorCameraCalibration, ref sensorData.colorCamIntr);
GetCameraIntrinsics(CalibrationDeviceType.Depth, coordMapperCalib.DepthCameraCalibration, ref sensorData.depthCamIntr);
GetCameraExtrinsics(coordMapperCalib.ColorCameraCalibration.Extrinsics, ref sensorData.depth2ColorExtr);
GetCameraExtrinsics(coordMapperCalib.DepthCameraCalibration.Extrinsics, ref sensorData.color2DepthExtr);
}
// body & body-index data
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
InitBodyTracking(dwFlags, sensorData, coordMapperCalib, true);
if (isSyncBodyAndDepth && bodyTracker == null)
{
// don't sync body and depth if the body tracker can't be created
isSyncBodyAndDepth = false;
}
}
Debug.Log("Kinect4Azure-sensor opened.");
return(sensorData);
}
