/// <summary>
/// Initializes a new instance of the GestureDetector class along with the gesture frame source and reader
/// </summary>
/// <param name="kinectSensor">Active sensor to initialize the VisualGestureBuilderFrameSource object with</param>
/// <param name="gestureResultView">GestureResultView object to store gesture results of a single body to</param>
public GestureDetector(KinectSensor kinectSensor, KinectManager kinectManager)
{
if (kinectSensor == null)
{
throw new ArgumentNullException("kinectSensor");
}
// create the vgb source. The associated body tracking ID will be set when a valid body frame arrives from the sensor.
this.vgbFrameSource = VisualGestureBuilderFrameSource.Create(kinectSensor, 0);
this.vgbFrameSource.TrackingIdLost += this.Source_TrackingIdLost;
// open the reader for the vgb frames
this.vgbFrameReader = this.vgbFrameSource.OpenReader();
if (this.vgbFrameReader != null)
{
this.vgbFrameReader.IsPaused = true;
this.vgbFrameReader.FrameArrived += this.Reader_GestureFrameArrived;
}
// load the 'Seated' gesture from the gesture database
using (VisualGestureBuilderDatabase database = VisualGestureBuilderDatabase.Create(this.gestureDatabase))
{
// we could load all available gestures in the database with a call to vgbFrameSource.AddGestures(database.AvailableGestures),
// but for this program, we only want to track one discrete gesture from the database, so we'll load it by name
foreach (Gesture gesture in database.AvailableGestures)
{
if (gesture.Name.Equals(this.shootingGestureName))
{
this.vgbFrameSource.AddGesture(gesture);
}
}
}
}
void Update ()
{
if(manager == null)
{
manager = KinectManager.Instance;
}
if(manager && manager.IsInitialized() && foregroundCamera)
{
//backgroundImage.renderer.material.mainTexture = manager.GetUsersClrTex();
if(backgroundImage && (backgroundImage.texture == null))
{
backgroundImage.texture = manager.GetUsersClrTex();
}
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if(portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
// overlay the joints
if(manager.IsUserDetected())
{
long userId = manager.GetUserIdByIndex(playerIndex);
OverlayJoint(userId, (int)KinectInterop.JointType.HandLeft, leftHandOverlay, backgroundRect);
OverlayJoint(userId, (int)KinectInterop.JointType.HandRight, rightHandOverlay, backgroundRect);
}
}
}
// Update is called once per frame
void Update()
{
// Find the kinectManager object
GameObject KM = GameObject.FindGameObjectWithTag("KinectManager");
_testObject = KM.GetComponent<KinectManager>();
//-----------------------
// Get all positions from the Kinect Manager
Result result1 = _testObject.getBodyPos();
if(result1.Success)
{
//Debug.Log(result1.AccessToken);
}
else
{
//If u like to do something when we do not have a position from the kinect
}
//-----------------------
// Get a sertain position from the Kinect Manager
Result result2 = _testObject.getBodyPartPos(KinectJT.HandRight);
if (result2.Success)
{
Debug.Log(result2.AccessToken[0]);
}
else
{
//If u like to do something when we do not have a position from the kinect
}
}
// Use this for initialization
public void SencerInitialize()
{
// キネクトマネージャーの取得
manager = KinectManager.Instance;
if (manager && manager.IsInitialized ()) {
KinectInterop.SensorData sensorData = manager.GetSensorData ();
if (sensorData != null && sensorData.sensorInterface != null) {
// get depth image size
depthImageWidth = sensorData.depthImageWidth;
depthImageHeight = sensorData.depthImageHeight;
// calculate the foreground rectangles
Rect cameraRect = Camera.main.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if (rectWidth > rectHeight)
rectWidth = rectHeight * depthImageWidth / depthImageHeight;
else
rectHeight = rectWidth * depthImageHeight / depthImageWidth;
float foregroundOfsX = (cameraRect.width - rectWidth) / 2;
float foregroundOfsY = (cameraRect.height - rectHeight) / 2;
foregroundImgRect = new Rect (foregroundOfsX, foregroundOfsY, rectWidth, rectHeight);
foregroundGuiRect = new Rect (foregroundOfsX, cameraRect.height - foregroundOfsY, rectWidth, -rectHeight);
}
}
}
/*******************************/
/* Fonction d'initialisation */
/*******************************/
/** \brief Fonction d'initialisation de la Bullet
* \param d : Mouvement à associer au Bullet
*/
public void init(KinectManager.Direction d)
{
if (KinectManager.Instance == null) { // Vérification de l'existence du KinectManager
Debug.LogError("MovementManager wasn't initialized"); // Si absent, erreur
return;
}
else { // Si présent
switch (d) { // Suivant le mouvement demandé
case KinectManager.Direction.Up:
KinectManager.Instance.onPlayerMovementUpEvent += onDirection; // On s'inscrit à l'évènement correspondant
GetComponent<Renderer>().material.color = new Color(1, 0.4f, 0.4f); // Rouge clair
break;
case KinectManager.Direction.Left:
KinectManager.Instance.onPlayerMovementLeftEvent += onDirection;
GetComponent<Renderer>().material.color = new Color(1, 0.4f, 0.4f);
break;
case KinectManager.Direction.Right:
KinectManager.Instance.onPlayerMovementRightEvent += onDirection;
GetComponent<Renderer>().material.color = new Color(1, 0.4f, 0.4f);
break;
case KinectManager.Direction.BonusUp:
KinectManager.Instance.onPlayerMovementBonusUpEvent += onDirection;
GetComponent<Renderer>().material.color = new Color(0.4f, 0.4f, 1); // bleu clair
break;
case KinectManager.Direction.BonusDown:
KinectManager.Instance.onPlayerMovementBonusDownEvent += onDirection;
GetComponent<Renderer>().material.color = new Color(0.4f, 0.4f, 1); ;
break;
}
}
direction = d; // On mémorise le mouvement
initialized = true; // On indique que l'initialisation a eu lieu
}
private bool HandIsGrabbing(KinectManager km)
{
if (kinectInterpreter.useGamepad) {
if (isLeftHand) {
return Input.GetAxis("Horizontal" + kinectInterpreter.gamepad) != 0f || Input.GetAxis("Vertical" + kinectInterpreter.gamepad) != 0f;
} else {
return false;
}
} else {
if (isLeftHand) {
if (km.GetLeftHandState (userId) != KinectInterop.HandState.Closed || km.GetRightHandState (userId) == KinectInterop.HandState.Closed) {
dragBuffer = 0;
return false;
}
} else {
if (km.GetRightHandState (userId) != KinectInterop.HandState.Closed || km.GetLeftHandState (userId) == KinectInterop.HandState.Closed) {
dragBuffer = 0;
return false;
}
}
if (dragBuffer < dragBufferTriggerSize)
dragBuffer++;
return dragBuffer == dragBufferTriggerSize;
}
}
/****************************/
/* Fonctions pour Unity3D */
/****************************/
public void Awake()
{
if (Instance != null)
{
Debug.LogError("There is multiple instance of singleton MovementManager");
return;
}
Instance = this;
}
void Start ()
{
// Make the rigid body not change rotation
if (rigidbody)
rigidbody.freezeRotation = true;
rotationX = transform.localEulerAngles.y;
rotationY = transform.localEulerAngles.x;
kinectManager = KinectManager.instance;
}
// Update is called once per frame
void FixedUpdate()
{
// show pause screen and return if isPaused
if (isPaused)
{
overlay.waitingForPlayer.SetActive(false);
overlay.pauseScreen.SetActive(true);
begin = false;
Cursor.visible = true;
return;
}
else
Cursor.visible = false;
overlay.pauseScreen.SetActive(false);
// setup kinect & return if no user found
manager = KinectManager.Instance;
long userID = manager ? manager.GetUserIdByIndex (0) : 0;
if (userID == 0)
{
overlay.waitingForPlayer.SetActive(true);
begin = false;
return;
}
begin = true;
overlay.waitingForPlayer.SetActive(false);
// set joints
bottomSpine = manager.GetJointPosition (userID, 0);
bottomHead = manager.GetJointPosition (userID, 2);
leftShoulder = manager.GetJointPosition (userID, 4);
leftHand = manager.GetJointPosition (userID, 6);
rightShoulder = manager.GetJointPosition (userID, 8);
rightHand = manager.GetJointPosition (userID, 10);
leftHip = manager.GetJointPosition (userID, 12);
leftKnee = manager.GetJointPosition (userID, 13);
leftFoot = manager.GetJointPosition (userID, 15);
rightHip = manager.GetJointPosition (userID, 16);
rightKnee = manager.GetJointPosition (userID, 17);
rightFoot = manager.GetJointPosition (userID, 19);
xBottom = bottomSpine.x;
// Horizontal movement
HorizontalMovement();
// Forward movement
MoveForward();
// Calculate leg angles
CalcAngles();
// Calculate lowest foot
LowestFoot();
}
private float speed; //vitesse du vaisseau
#endregion Fields
#region Methods
void Awake()
{
j = 0;
speed = Vaisseau.speedVaisseau;
fast = false;
// get needed objects´ references
manager = GameObject.Find("Vaisseau/Main Camera").GetComponent<KinectManager>();
infoGUI = GameObject.Find("HandGuiText");
}
private Program()
{
Logger.Info("Start Application");
var manager = new KinectManager();
var left = new Kinect.Gestures.GrabAndThrow(HandToWatch.HandLeft);
left.Detected += LeftOnDetected;
manager.RegisterGesture(left);
var right = new Kinect.Gestures.GrabAndThrow(HandToWatch.HandRight);
right.Detected += RightOnDetected;
manager.RegisterGesture(right);
}
public VolumeTracker(float ulnaLength)
{
kinect = GameObject.Find("KinectManager").GetComponent<KinectManager>();
transformMatrix = GameControl.Instance.TransformMatrix;
volumes = new Volumes();
offset = new Vector3(0.0f, -0.01f, 0.07f);
if (ulnaLength > 0)
{
upperBound = ulnaLength / 100.0f;
lowerBound = upperBound / 3.0f;
}
}
// Use this for initialization
IEnumerator Start()
{
FireballParticles.enableEmission = false;
FireballCoreParticles.enableEmission = false;
while (KManager == null && IManager == null)
{
KManager = KinectManager.Instance;
IManager = InteractionManager.Instance;
}
yield return null;
}
public override void OnInitialize()
{
manager = GetManager<KinectManager>();
manager.AddGestureEvent(this);
Stream gestureStream =
typeof(GestureEvent).Assembly.GetManifestResourceStream(folder + GestureName + extention);
//TODO null == gesture is if the file is not there.
GestureDetector = new TemplatedGestureDetector(GestureName, gestureStream);
GestureDetector.MinimalPeriodBetweenGestures = 0;
GestureDetector.OnGestureDetected += detectedGesture;
}
public void slidersMinMaxChanged()
{
if ( kinectManager == null )
{
GameObject go = GameObject.Find("goKinectManager");
kinectManager = go.GetComponent<KinectManager>();
}
depthMinText.text = "MIN " + depthZMinSlider.value;
depthMaxText.text = "MAX " + depthZMaxSlider.value;
kinectManager.minDepth = depthZMinSlider.value;
kinectManager.maxDepth = depthZMaxSlider.value;
}
void Start ()
{
kinectManager = KinectManager.Instance;
faceTex = new Texture2D(100, 100, TextureFormat.ARGB32, false);
if(targetObject)
{
targetRenderer = targetObject.GetComponent<Renderer>();
if(targetRenderer && targetRenderer.material)
{
targetRenderer.material.SetTextureScale("_MainTex", new Vector2(1, -1));
}
}
}
// Update is called once per frame
void Update()
{
// Get the KinectManager instance
if(kinectManager == null)
{
kinectManager = KinectManager.Instance;
}
if( kinectManager.IsUserDetected() ) {
UpdateArms();
flagDisconection = false;
} else {
flagDisconection = true;
}
}
void Start()
{
manager = KinectManager.Instance;
if (manager != null)
{
depthWidth = manager.GetDepthImageWidth();
depthHeight = manager.GetDepthImageHeight();
colorWidth = manager.GetColorImageWidth();
colorHeight = manager.GetColorImageHeight();
CreateMesh(depthWidth / SampleSize, depthHeight / SampleSize);
}
}
void Update()
{
if(manager == null)
{
manager = KinectManager.Instance;
}
// get the users texture
if(manager && manager.IsInitialized())
{
foregroundTex = manager.GetUsersLblTex();
}
if(manager.IsUserDetected())
{
uint userId = manager.GetPlayer1ID();
// update colliders
int numColliders = (int)KinectWrapper.NuiSkeletonPositionIndex.Count;
for(int i = 0; i < numColliders; i++)
{
if(manager.IsJointTracked(userId, i))
{
Vector3 posJoint = manager.GetRawSkeletonJointPos(userId, i);
if(posJoint != Vector3.zero)
{
// convert the joint 3d position to depth 2d coordinates
Vector2 posDepth = manager.GetDepthMapPosForJointPos(posJoint);
float scaledX = posDepth.x * foregroundRect.width / KinectWrapper.Constants.DepthImageWidth;
float scaledY = posDepth.y * -foregroundRect.height / KinectWrapper.Constants.DepthImageHeight;
float screenX = foregroundOfs.x + scaledX;
float screenY = Camera.main.pixelHeight - (foregroundOfs.y + scaledY);
float zDistance = posJoint.z - Camera.main.transform.position.z;
Vector3 posScreen = new Vector3(screenX, screenY, zDistance);
Vector3 posCollider = Camera.main.ScreenToWorldPoint(posScreen);
jointColliders[i].transform.position = posCollider;
}
}
}
}
}
void Start ()
{
manager = KinectManager.Instance;
if(manager && manager.IsInitialized())
{
KinectInterop.SensorData sensorData = manager.GetSensorData();
if(sensorData != null && sensorData.sensorInterface != null && foregroundCamera != null)
{
// get depth image size
depthImageWidth = sensorData.depthImageWidth;
depthImageHeight = sensorData.depthImageHeight;
// calculate the foreground rectangles
Rect cameraRect = foregroundCamera.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if(rectWidth > rectHeight)
rectWidth = rectHeight * depthImageWidth / depthImageHeight;
else
rectHeight = rectWidth * depthImageHeight / depthImageWidth;
float foregroundOfsX = (cameraRect.width - rectWidth) / 2;
float foregroundOfsY = (cameraRect.height - rectHeight) / 2;
foregroundImgRect = new Rect(foregroundOfsX, foregroundOfsY, rectWidth, rectHeight);
foregroundGuiRect = new Rect(foregroundOfsX, cameraRect.height - foregroundOfsY, rectWidth, -rectHeight);
// create joint colliders
numColliders = sensorData.jointCount;
jointColliders = new GameObject[numColliders];
for(int i = 0; i < numColliders; i++)
{
string sColObjectName = ((KinectInterop.JointType)i).ToString() + "Collider";
jointColliders[i] = new GameObject(sColObjectName);
jointColliders[i].transform.parent = transform;
SphereCollider collider = jointColliders[i].AddComponent<SphereCollider>();
collider.radius = 0.2f;
}
}
}
}
protected virtual void Awake()
{
if (Instance != null)
{
Debug.Log("Error. There can be only one KinectManager.");
Debug.Log("Kinect Manager #1", Instance.gameObject);
Debug.Log("Kinect manager #2", this.gameObject);
}
else
{
Instance = this;
}
/*Transform KinectRepresentation = this.transform.Find("Kinect");
if (KinectRepresentation != null)
{
foreach (Renderer renderer in this.transform.Find("Kinect").GetComponentsInChildren<Renderer>())
renderer.enabled = false;
}*/
}
// Use this for initialization
void Start()
{
// キネクトマネージャーの取得
manager = KinectManager.Instance;
// リズムカウントの配列を初期化
rhythmTimes = new Queue<float>(rhythmCalcNum);
float initRhythm = 60.0f / initBPM;
for (int i = 0; i < rhythmTimes.Count; i++) {
rhythmTimes.Enqueue(initRhythm);
}
checkObjectPosY = new Queue<float>(checkObjectPosYNum);
// チェックオブジェクトの高さ検出に関する初期化
// 初期位置で配列を埋める
long userId = manager.GetUserIdByIndex (0);
Vector3 pos = manager.GetJointPosition (userId, (int)checkBodyType);
}
void Update ()
{
// get the face-tracking manager instance
if(faceManager == null)
{
kinectManager = KinectManager.Instance;
faceManager = FacetrackingManager.Instance;
}
if(kinectManager && faceManager && faceManager.IsTrackingFace())
{
// get user-id by user-index
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if(userId == 0)
return;
// head rotation
Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true);
if(smoothFactor != 0f)
transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime);
else
transform.rotation = newRotation;
// head position
Vector3 newPosition = faceManager.GetHeadPosition(userId, true);
if(verticalOffset != 0f)
{
Vector3 dirHead = new Vector3(0, verticalOffset, 0);
dirHead = transform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// if(smoothFactor != 0f)
// transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
// else
transform.position = newPosition;
}
}
private void InitializeKinect()
{
try
{
Logger.InfoFormat("Initialize kinect");
_kinectManager = new KinectManager();
_gestureFactory = new GestureFactory(_kinectManager);
//_gestureFactory.GrabAndThrowLeftHanded += GrabAndThrowLeft;
//_gestureFactory.GrabAndThrowRightHanded += GrabAndThrowRight;
_gestureFactory.RightHandClosed += RightHandClosed;
_gestureFactory.LeftHandClosed += LeftHandClosed;
_gestureFactory.RightHandOpened += RightHandOpened;
_gestureFactory.LeftHandOpened += LeftHandOpened;
_kinectManager.TrackedBodyCountChanged += TrackedBodyCountChanged;
Logger.InfoFormat("Kinect initialized with status: {0}", _kinectManager.Status);
}
catch (Exception eX)
{
Logger.ErrorFormat("Failed to start kinect: {0}", eX.Message);
}
}
/**
* Main method. Initializes all input managers and starts them.
* Then, executes the start-up script (right now it just goes
* to Earth. To make a better one, look at the Extensibility section
* in TravelCommand.cs)
*
* Author: Ross Kahn
**/
public static void Main()
{
KeyboardMouseManager keymanager = new KeyboardMouseManager();
keymanager.start();
KinectManager kinectManager = new KinectManager();
SpeechManager speechManager = new SpeechManager();
kinectManager.Initialize();
if (KinectManager.isConnected())
{
// Start the speech manager with the Kinect microphone array
speechManager.Initialize(kinectManager.getSensor());
}
else
{
// Start the speech manager with the default system microphone because
// no kinect was detected
speechManager.Initialize();
}
// Start-up script. Works just like a regular travel command
TravelCommand command = new TravelCommand(Constants.PLANET.EARTH);
doCommand(command);
// Keeps the application running. Since this program is event-driven,
// there needs to be a non-blocking main loop as the program waits
// for events to come in. That's what Application.Run() does
try
{
Application.Run();
}
catch (Exception ex)
{
MessageBox.Show(ex.StackTrace);
}
}
// Update is called once per frame
void Update()
{
km = KinectManager.Instance;
string trackedJointsString = "Tracked Joints:";
if (km != null) {
if (km.IsUserDetected()) {
uint userId = km.GetPlayer1ID();
/*trackedJointsString += "\nchecking joints";
if(km.IsJointTracked(userId, (int)rightHand))
trackedJointsString += "\nRight hand";
if(km.IsJointTracked(userId, (int)rightWrist))
trackedJointsString += "\nRight wrist";
if(km.IsJointTracked(userId, (int)centreHip))
trackedJointsString += "\nCentre Hip";
if(km.IsJointTracked(userId, (int)leftShoulder))
trackedJointsString += "\nLeft shoulder";
if(km.IsJointTracked(userId, (int)centreShoulder))
trackedJointsString += "\nCentre shoulder";
if(km.IsJointTracked(userId, (int)rightShoulder))
trackedJointsString += "\nRight shoulder";
trackedJointsString += "\ndone checking joints";*/
if (km.IsJointTracked(userId, (int)rightHand) && km.IsJointTracked(userId, (int)rightShoulder))
wristVector = km.GetJointPosition(userId, (int)rightHand) - km.GetJointPosition(userId, (int)rightShoulder);
if (km.IsJointTracked(userId, (int)rightShoulder) && km.IsJointTracked(userId, (int)leftShoulder))
shouldersVector = km.GetJointPosition(userId, (int)rightShoulder) - km.GetJointPosition(userId, (int)leftShoulder);
if (km.IsJointTracked(userId, (int)centreShoulder) && km.IsJointTracked(userId, (int)centreHip))
backVector = km.GetJointPosition(userId, (int)centreShoulder) - km.GetJointPosition(userId, (int)centreHip);
//GramSchmidt Orthonormal Space
Vector3 e2 = backVector.normalized;
Vector3 e1 = (shouldersVector - Vector3.Dot (shouldersVector,e2) * e2).normalized;
wristVectorInPlane = new Vector2(Vector3.Dot(e1, wristVector), Vector3.Dot(e2, wristVector));
}
}
trackedJointsText.text = trackedJointsString;
CheckPointingTopRight();
}
// Update is called once per frame
void Update()
{
if ((manager != null && manager.IsInteractionInited()) && (Kmanager != null) && (!manager.IsSceneReady()))
{
if (!manager.isPlayerDetected())
{
Idle_Timer += Time.deltaTime;
//DebugGUI.guiText.text = "Idle : " + Idle_Timer.ToString();
if (Idle_Timer > 5.0f)
{
Idle_Timer = 0;
PlayerPrefs.SetInt("prodPlayAgain", 0); // Enables the splash screen on load
Application.LoadLevel(Application.loadedLevelName);
}
}
else
Idle_Timer = 0;
if (Input.GetKeyDown(KeyCode.UpArrow))
{
Kmanager.SensorAngle += 2;
KinectWrapper.NuiCameraElevationSetAngle(Kmanager.SensorAngle);
}
else if (Input.GetKeyDown(KeyCode.DownArrow))
{
Kmanager.SensorAngle -= 2;
KinectWrapper.NuiCameraElevationSetAngle(Kmanager.SensorAngle);
}
}
else
{
if (manager == null)
manager = InteractionManager.Instance;
if (Kmanager == null)
Kmanager = KinectManager.Instance;
}
}
void Start ()
{
manager = KinectManager.Instance;
if(manager && manager.IsInitialized())
{
Rect cameraRect = Camera.main.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
KinectInterop.SensorData sensorData = manager.GetSensorData();
if(sensorData != null && sensorData.sensorInterface != null)
{
if(rectWidth > rectHeight)
rectWidth = rectHeight * sensorData.depthImageWidth / sensorData.depthImageHeight;
else
rectHeight = rectWidth * sensorData.depthImageHeight / sensorData.depthImageWidth;
foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight);
}
}
}
// Use this for initialization
void Start()
{
KinectManager[] avatars = FindObjectsOfType(typeof(KinectManager)) as KinectManager[];
kinectManager = avatars[0];
if(networkView.isMine)
kinectManager.Player1Controllers.Add(GetComponent<AvatarController>());
//enable script
BotControlScript script = GetComponent<BotControlScript>();
if(networkView.isMine)
{
script.enabled = true;
}
else
{
script.enabled = true;
gameObject.rigidbody.useGravity = false;//.useGravity = false;
}
}
void Update ()
{
// get reference to the Kinect2Interface
if(k2interface == null)
{
manager = KinectManager.Instance;
if(manager && manager.IsInitialized())
{
KinectInterop.SensorData sensorData = manager.GetSensorData();
if(sensorData != null && sensorData.sensorInterface != null)
{
k2interface = (Kinect2Interface)sensorData.sensorInterface;
}
}
}
// get the face points
if(k2interface != null && k2interface.faceFrameResults != null)
{
if(manager != null && manager.IsUserDetected())
{
ulong userId = (ulong)manager.GetPrimaryUserID();
for(int i = 0; i < k2interface.faceFrameResults.Length; i++)
{
if(k2interface.faceFrameResults[i] != null && k2interface.faceFrameResults[i].TrackingId == userId)
{
facePoints = k2interface.faceFrameResults[i].FacePointsInColorSpace;
break;
}
}
}
}
}
public void Update()
{
KinectManager manager = KinectManager.Instance;
if (!manager || !manager.IsInitialized() || !manager.IsUserDetected())
{
return;
}
int iJointIndex = (int)TrackedJoint;
if (manager.IsUserDetected())
{
uint userId = manager.GetPlayer1ID();
if (manager.IsJointTracked(userId, iJointIndex))
{
Vector3 posJoint = manager.GetRawSkeletonJointPos(userId, iJointIndex);
if (posJoint != Vector3.zero)
{
joint = GameObject.Find("HipCenterCollider");
Debug.Log(joint.name);
//PROVA1
/*
* for (i=0; i<joints.Length; i++)
* {
* if (joints[i].name == "HipCenter")
* {
* if (collision.gameObject == joints[i])
* {
* Debug.Log("collision" + collision.gameObject.name);
* }
*
* }
*
* } */
/* //PROVA2
* foreach(GameObject joints in GameObject[])
* { if (joints.name == "HipCenter"){
*
* }
* }
*
*
* if (collision.gameObject == joints[0])
* {
* Debug.Log("Mateixa posició");
* riu.enabled = true;
* activ = true;
* desactiv = false;
* PujarAlpha();
* }
* else
* {
* riu.enabled = true;
* desactiv = true;
* activ = false;
* PujarAlpha();
* } */
}
}
}
}
// Update is called once per frame
void FixedUpdate()
{
if (standby)
{
// waiting for input (allow user input)
ResetTimer();
// print ("in standby");
if (!usingKinect && GameObject.Find("TUIOHand") != null)
{
// start game
standby = false;
isPlaying = true; // user is allowed to move items
print("game started");
}
else if (usingKinect)
{
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
// print("kinnect manager is initialized");
if (kinectManager.IsUserDetected())
{
// print ("user was detected");
uint userId = kinectManager.GetPlayer1ID();
trackJoint(kinectManager, userId, iTrackedJoint, rightHandObject);
trackJoint(kinectManager, userId, iLeftHandJoint, leftHandObject);
standby = false;
isPlaying = true; // user is allowed to move items
print("game started");
}
}
}
else
{
// print ("waiting for user");
}
}
else
{
// user is interacting with kinect or tuio
if (isPlaying)
{
if (Input.GetKeyDown(KeyCode.A))
{
UnityEngine.Debug.Log("nr items above each heart:");
for (int i = 0; i < nrHearts; i++)
{
HeartController heartController = hearts [i].GetComponent <HeartController> ();
UnityEngine.Debug.Log(heartController.nrOverlayedItems);
}
print("nr corazones descubiertos" + nrVisibleHearts);
}
// check game over
if (timeLeft <= 0)
{
gameOver = true;
}
else
{
// check if player won
// print("mainController: antes de contar corazones");
CountNrVisibleHearts();
MarkHeartsForCheck();
// print("mainController: despues de setear corazones a true");
if (nrVisibleHearts == nrHearts)
{
won = true;
}
UpdateTimer();
}
isPlaying = !gameOver && !won;
// print ("is playing: " + isPlaying);
if (!isPlaying)
{
// take picture with gopro
if (takePicture)
{
// Process process = Process.Start (processInfo);
}
UnityEngine.Debug.Log("retorno de la toma de foto");
// do not allow collision between hands and items anymore
Physics2D.IgnoreLayerCollision(playerLayer, itemsLayer);
// playAgainBtn.SetActive(true);
if (won)
{
winMessage.SetActive(true);
// print ("gano");
}
else
{
loseMessage.SetActive(true);
// print ("perdio");
}
StartCoroutine(RestartGameCoroutine());
}
}
else
{
// user is not allowed to move items anymore but can still click on playAgainBtn
// print ("waiting for user to press play again button");
}
// update player's hands
if (usingKinect)
{
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
// print("kinnect manager is initialized");
if (kinectManager.IsUserDetected())
{
// print ("user was detected");
uint userId = kinectManager.GetPlayer1ID();
trackJoint(kinectManager, userId, iTrackedJoint, rightHandObject);
trackJoint(kinectManager, userId, iLeftHandJoint, leftHandObject);
}
}
}
}
}
private void LoadClothes(string clothesName)
{
app.model.modelClothes.verticalOffset = 0;
app.model.modelClothes.forwardOffset = 0;
app.model.modelClothes.modelCategory = "Clothing";
string subPath = ChangeSubPath();
int playerIndex = app.model.modelData.playerIndex;
string modelPath = app.model.modelClothes.modelCategory + "/" + subPath + "/" + clothesName;
print(modelPath);
UnityEngine.Object modelPrefab = Resources.Load(modelPath, typeof(GameObject));
if (modelPrefab == null)
{
return;
}
print(clothesName);
Debug.Log("Model: " + modelPath);
if (app.model.modelClothes.listClothes.Length >= app.model.modelData.kinectManager.GetUsersCount())
{
if (app.model.modelClothes.listClothes[app.model.modelData.playerIndex])
{
GameObject.Destroy(app.model.modelClothes.listClothes[app.model.modelData.playerIndex]);
//app.model.modelClothes.selModels.RemoveAt(app.model.modelData.playerIndex);
}
}
app.model.modelClothes.listClothes[app.model.modelData.playerIndex] = (GameObject)GameObject.Instantiate(modelPrefab, Vector3.zero, Quaternion.Euler(0, 180f, 0));
app.model.modelClothes.listClothes[app.model.modelData.playerIndex].name = "Model000" + playerIndex;
//app.model.modelClothes.selModels.Add(app.model.modelClothes.selModel);
//app.model.modelClothes.selModels[playerIndex] = (GameObject)GameObject.Instantiate(modelPrefab, Vector3.zero, Quaternion.Euler(0, 180f, 0));
//app.model.modelClothes.selModels[playerIndex].name = "Model000" + playerIndex;
//app.model.modelClothes.selModel = (GameObject)GameObject.Instantiate(modelPrefab, Vector3.zero, Quaternion.Euler(0, 180f, 0));
////app.model.modelClothes.selModel.name = "Model" + "0000";
AvatarController ac = app.model.modelClothes.listClothes[playerIndex].GetComponent <AvatarController>();
if (ac == null)
{
ac = app.model.modelClothes.listClothes[playerIndex].AddComponent <AvatarController>();
ac.playerIndex = playerIndex;
ac.mirroredMovement = true;
ac.verticalMovement = true;
ac.verticalOffset = app.model.modelClothes.verticalOffset;
ac.forwardOffset = app.model.modelClothes.forwardOffset;
ac.smoothFactor = 0f;
}
ac.posRelativeToCamera = app.model.modelData.backgroundCamera;
ac.posRelOverlayColor = (app.model.modelData.foregroundCamera != null);
KinectManager km = KinectManager.Instance;
//ac.Awake();
if (km && km.IsInitialized())
{
long userId = km.GetUserIdByIndex(playerIndex);
if (userId != 0)
{
ac.SuccessfulCalibration(userId, false);
}
// locate the available avatar controllers
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
km.avatarControllers.Clear();
foreach (MonoBehaviour monoScript in monoScripts)
{
if ((monoScript is AvatarController) && monoScript.enabled)
{
AvatarController avatar = (AvatarController)monoScript;
km.avatarControllers.Add(avatar);
}
}
}
AvatarScaler scaler = app.model.modelClothes.listClothes[app.model.modelData.playerIndex].GetComponent <AvatarScaler>();
if (scaler == null)
{
scaler = app.model.modelClothes.listClothes[app.model.modelData.playerIndex].AddComponent <AvatarScaler>();
scaler.playerIndex = playerIndex;
scaler.mirroredAvatar = true;
scaler.continuousScaling = true;
scaler.bodyScaleFactor = 1.1f;
scaler.bodyWidthFactor = 1.1f;
scaler.armScaleFactor = 1.1f;
scaler.legScaleFactor = 1.1f;
}
scaler.foregroundCamera = app.model.modelData.foregroundCamera;
}
public void GestureInProgress(KinectManager manager, long userId, int userIndex, KinectGestures.Gestures gesture, float progress, KinectInterop.JointType joint, Vector3 screenPos)
{
throw new NotImplementedException();
}
public void UserLost(KinectManager manager, long userId, int userIndex)
{
throw new NotImplementedException();
}
// Use this for initialization
void Start()
{
kinman = KinectManager.Instance;
}
//---------------------------------- END OF PUBLIC FUNCTIONS -----------------------------------------------------------//
void Awake()
{
KinectCoordinatesAdjustment = new KinectWrapper.NuiImageViewArea
{
eDigitalZoom = 0,
lCenterX = 0,
lCenterY = 0
};
int hr = 0;
try
{
hr = KinectWrapper.NuiInitialize(KinectWrapper.NuiInitializeFlags.UsesSkeleton |
KinectWrapper.NuiInitializeFlags.UsesDepthAndPlayerIndex |
KinectWrapper.NuiInitializeFlags.UsesColor);
if (hr != 0)
{
throw new Exception("NuiInitialize Failed");
}
depthStreamHandle = IntPtr.Zero;
hr = KinectWrapper.NuiImageStreamOpen(KinectWrapper.NuiImageType.DepthAndPlayerIndex,
KinectWrapper.Constants.DepthImageResolution, 0, 2, IntPtr.Zero, ref depthStreamHandle);
if (hr != 0)
{
throw new Exception("Cannot open depth stream");
}
colorStreamHandle = IntPtr.Zero;
hr = KinectWrapper.NuiImageStreamOpen(KinectWrapper.NuiImageType.Color,
KinectWrapper.Constants.ColorImageResolution, 0, 2, IntPtr.Zero, ref colorStreamHandle);
if (hr != 0)
{
throw new Exception("Cannot open color stream");
}
// set kinect elevation angle
KinectWrapper.NuiCameraElevationSetAngle(kinectMotorAngle);
//create the transform matrix that converts from kinect-space to world-space
Quaternion quatTiltAngle = new Quaternion();
quatTiltAngle.eulerAngles = new Vector3(-kinectMotorAngle, 0.0f, 0.0f);
// transform matrix - kinect to world
kinectToWorld.SetTRS(new Vector3(0.0f, kinectPlacementHeight, 0.0f), quatTiltAngle, Vector3.one);
flipMatrix = Matrix4x4.identity;
flipMatrix[2, 2] = -1;
DontDestroyOnLoad(gameObject);
}
catch (DllNotFoundException e)
{
string message = "Please check the Kinect SDK installation.";
Debug.LogError(message);
Debug.LogError(e.ToString());
if (UILog != null)
{
UILog.text = message;
}
return;
}
catch (Exception e)
{
string message = e.Message + " - " + KinectWrapper.GetNuiErrorString(hr);
Debug.LogError(message);
Debug.LogError(e.ToString());
if (UILog != null)
{
UILog.text = message;
}
return;
}
InitializeFeeds();
InitializeGradedDepthStreamColors();
instance = this;
if (UILog != null)
{
UILog.text = "Kinect is initialized";
}
Debug.Log("Kinect is initialized");
kinectInitialized = true;
}
private void Start()
{
manager = KinectManager.Instance;
}
// Apply the orientation constraints
public void Constrain(ref KinectInterop.BodyData bodyData)
{
KinectManager manager = KinectManager.Instance;
frameNum++;
for (int i = 0; i < jointConstraints.Count; i++)
{
BoneOrientationConstraint jc = this.jointConstraints[i];
if (jc.thisJoint == (int)KinectInterop.JointType.SpineBase || bodyData.joint[jc.thisJoint].normalRotation == Quaternion.identity)
{
continue;
}
if (bodyData.joint[jc.thisJoint].trackingState == KinectInterop.TrackingState.NotTracked)
{
continue;
}
int prevJoint = (int)manager.GetParentJoint((KinectInterop.JointType)jc.thisJoint);
if (bodyData.joint[prevJoint].trackingState == KinectInterop.TrackingState.NotTracked)
{
continue;
}
Quaternion rotParentN = bodyData.joint[prevJoint].normalRotation;
Quaternion rotDefaultN = Quaternion.FromToRotation(KinectInterop.JointBaseDir[prevJoint], KinectInterop.JointBaseDir[jc.thisJoint]);
rotParentN = rotParentN * rotDefaultN;
Quaternion rotJointN = bodyData.joint[jc.thisJoint].normalRotation;
Quaternion rotLocalN = Quaternion.Inverse(rotParentN) * rotJointN;
Vector3 eulerAnglesN = rotLocalN.eulerAngles;
// if(jc.thisJoint == (int)KinectInterop.JointType.KneeLeft)
// {
// float angle1X = eulerAnglesN.x <= 180f ? eulerAnglesN.x : eulerAnglesN.x - 360f;
// float angle1Y = eulerAnglesN.y <= 180f ? eulerAnglesN.y : eulerAnglesN.y - 360f;
// float angle1Z = eulerAnglesN.z <= 180f ? eulerAnglesN.z : eulerAnglesN.z - 360f;
//
// string sDebugText = string.Format("{0}. {1} - ({2:000}, {3:000}, {4:000})",
// frameNum, ((KinectInterop.JointType)jc.thisJoint).ToString(), angle1X, angle1Y, angle1Z);
//
// if(debugText != null && (Time.time - currentTime) >= 0.5f)
// {
// currentTime = Time.time;
// //debugText.text = sDebugText;
// }
//
// //Debug.Log(sDebugText);
// }
bool isConstrained = false;
string sDebug = string.Empty;
for (int a = 0; a < jc.axisConstrainrs.Count; a++)
{
AxisOrientationConstraint ac = jc.axisConstrainrs[a];
Quaternion rotLimited = rotLocalN;
switch (ac.consType)
{
case 0:
break;
case CT.LimA:
eulerAnglesN = LimitAngles(eulerAnglesN, ac.axis, ac.angleMin, ac.angleMax);
rotLimited = Quaternion.Euler(eulerAnglesN);
break;
case CT.LimST:
rotLimited = LimitSwing(rotLocalN, ac.axis, ac.angleMin);
rotLimited = LimitTwist(rotLimited, ac.axis, ac.angleMax);
break;
case CT.LimH:
float lastAngle = bodyData.joint[jc.thisJoint].lastAngle;
rotLimited = LimitHinge(rotLocalN, ac.axis, ac.angleMin, ac.angleMax, ref lastAngle);
bodyData.joint[jc.thisJoint].lastAngle = lastAngle;
break;
default:
throw new Exception("Undefined constraint type found: " + (int)ac.consType);
}
if (rotLimited != rotLocalN)
{
rotLocalN = rotLimited;
isConstrained = true;
}
}
if (sDebug.Length > 0)
{
if (debugText != null && jc.thisJoint == (int)KinectInterop.JointType.ElbowLeft)
{
// debugText.text = sDebug;
}
Debug.Log(sDebug);
}
if (isConstrained)
{
rotJointN = rotParentN * rotLocalN;
Vector3 eulerJoint = rotJointN.eulerAngles;
Vector3 eulerJointM = new Vector3(eulerJoint.x, -eulerJoint.y, -eulerJoint.z);
Quaternion rotJointM = Quaternion.Euler(eulerJointM);
// put it back into the bone orientations
bodyData.joint[jc.thisJoint].normalRotation = rotJointN;
bodyData.joint[jc.thisJoint].mirroredRotation = rotJointM;
}
}
}
void Update()
{
// get the face-tracking manager instance
if (manager == null)
{
manager = FacetrackingManager.Instance;
}
if (manager && manager.IsTrackingFace())
{
// set head position & rotation
if (HeadTransform != null)
{
// head position
Vector3 newPosition = manager.GetHeadPosition(mirroredHeadMovement);
// head rotation
Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement);
// rotational fix, provided by Richard Borys:
// The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect
Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z),
Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero;
addedRotation.x = newRotation.eulerAngles.x + addedRotation.x;
addedRotation.y = newRotation.eulerAngles.y + addedRotation.y;
addedRotation.z = newRotation.eulerAngles.z + addedRotation.z;
newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z);
// end of rotational fix
if (smoothFactor != 0f)
{
HeadTransform.rotation = Quaternion.Slerp(HeadTransform.rotation, newRotation, smoothFactor * Time.deltaTime);
}
else
{
HeadTransform.rotation = newRotation;
}
// check for head pos overlay
if (foregroundCamera)
{
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if (portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager)
{
long userId = kinectManager.GetUserIdByIndex(manager.playerIndex);
Vector3 posColorOverlay = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);
if (posColorOverlay != Vector3.zero)
{
newPosition = posColorOverlay;
// if(overlayObj)
// {
// overlayObj.position = newPosition;
// }
}
}
}
else
{
// move around the initial position
newPosition += HeadInitialPosition;
}
// vertical offet
if (verticalOffset != 0f)
{
// add the vertical offset
Vector3 dirHead = new Vector3(0, verticalOffset, 0);
dirHead = HeadTransform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// set the position
if (smoothFactor != 0f)
{
HeadTransform.position = Vector3.Lerp(HeadTransform.position, newPosition, smoothFactor * Time.deltaTime);
}
else
{
HeadTransform.position = newPosition;
}
// scale factor
if (HeadTransform.localScale.x != modelScaleFactor)
{
HeadTransform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor);
}
}
// apply animation units
// AU0 - Upper Lip Raiser
// 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip
float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker);
SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp);
SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp);
// AU1 - Jaw Lowerer
// 0=closed; 1=fully open; -1= closed, like 0
float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen);
SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown);
// AU2 – Lip Stretcher
// 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth)
float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft);
fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left;
SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched);
float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight);
fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right;
SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched);
// AU3 – Brow Lowerer
// 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes)
float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer);
fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left;
SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered);
float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer);
fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right;
SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered);
// AU4 – Lip Corner Depressor
// 0=neutral; -1=very happy smile; +1=very sad frown
float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft);
fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left;
SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed);
float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight);
fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right;
SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed);
// AU6, AU7 – Eyelid closed
// 0=neutral; -1=raised; +1=fully lowered
float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed);
fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left;
SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered);
SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised);
float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed);
fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right;
SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered);
SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised);
}
else
{
// hide the model behind the camera
if (HeadTransform && HeadTransform.position.z >= 0f)
{
HeadTransform.position = new Vector3(0f, 0f, -10f);
}
}
}
public void UpdateFaceModelMesh(long userId, GameObject faceModelMesh, Vector3 headPos, Rect faceRect, ref Vector3[] avModelVertices, ref Vector2[] avModelUV, ref bool bGotModelVertices)
{
// init the vertices array if needed
if (avModelVertices == null)
{
int iNumVertices = sensorData.sensorInterface.GetFaceModelVerticesCount(userId);
avModelVertices = new Vector3[iNumVertices];
}
// get face model vertices
bGotModelVertices = sensorData.sensorInterface.GetFaceModelVertices(userId, ref avModelVertices);
if (bGotModelVertices && faceModelMesh != null)
{
//Quaternion faceModelRot = faceModelMesh.transform.rotation;
//faceModelMesh.transform.rotation = Quaternion.identity;
KinectManager kinectManager = KinectManager.Instance;
if (texturedModelMesh != TextureType.None)
{
float colorWidth = (float)kinectManager.GetColorImageWidth();
float colorHeight = (float)kinectManager.GetColorImageHeight();
//bool bGotFaceRect = sensorData.sensorInterface.GetFaceRect(userId, ref faceRect);
bool faceRectValid = /**bGotFaceRect &&*/ faceRect.width > 0 && faceRect.height > 0;
if (texturedModelMesh == TextureType.ColorMap &&
faceModelMesh.GetComponent <MeshRenderer>().material.mainTexture == null)
{
faceModelMesh.GetComponent <MeshRenderer>().material.mainTexture = kinectManager.GetUsersClrTex();
}
for (int i = 0; i < avModelVertices.Length; i++)
{
Vector2 posDepth = kinectManager.MapSpacePointToDepthCoords(avModelVertices[i]);
bool bUvSet = false;
if (posDepth != Vector2.zero)
{
ushort depth = kinectManager.GetDepthForPixel((int)posDepth.x, (int)posDepth.y);
Vector2 posColor = kinectManager.MapDepthPointToColorCoords(posDepth, depth);
if (posColor != Vector2.zero && !float.IsInfinity(posColor.x) && !float.IsInfinity(posColor.y))
{
if (texturedModelMesh == TextureType.ColorMap)
{
avModelUV[i] = new Vector2(posColor.x / colorWidth, posColor.y / colorHeight);
bUvSet = true;
}
else if (texturedModelMesh == TextureType.FaceRectangle && faceRectValid)
{
avModelUV[i] = new Vector2((posColor.x - faceRect.x) / faceRect.width,
-(posColor.y - faceRect.y) / faceRect.height);
bUvSet = true;
}
}
}
if (!bUvSet)
{
avModelUV[i] = Vector2.zero;
}
}
}
else
{
if (faceModelMesh.GetComponent <MeshRenderer>().material.mainTexture != null)
{
faceModelMesh.GetComponent <MeshRenderer>().material.mainTexture = null;
}
}
// make vertices relative to the head pos
Matrix4x4 kinectToWorld = kinectManager ? kinectManager.GetKinectToWorldMatrix() : Matrix4x4.identity;
Vector3 headPosWorld = kinectToWorld.MultiplyPoint3x4(headPos);
for (int i = 0; i < avModelVertices.Length; i++)
{
avModelVertices[i] = kinectToWorld.MultiplyPoint3x4(avModelVertices[i]) - headPosWorld;
}
Mesh mesh = faceModelMesh.GetComponent <MeshFilter>().mesh;
mesh.vertices = avModelVertices;
if (texturedModelMesh != TextureType.None)
{
mesh.uv = avModelUV;
}
mesh.RecalculateNormals();
mesh.RecalculateBounds();
// check for head pos overlay
Vector3 newHeadPos = headPos;
if (foregroundCamera)
{
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if (portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
if (kinectManager)
{
Vector3 posColorOverlay = kinectManager.GetJointPosColorOverlay(primaryUserID, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);
if (posColorOverlay != Vector3.zero)
{
newHeadPos = posColorOverlay;
}
}
}
faceModelMesh.transform.position = newHeadPos;
//faceModelMesh.transform.rotation = faceModelRot;
// apply scale factor
if (faceModelMesh.transform.localScale.x != modelMeshScale)
{
faceModelMesh.transform.localScale = new Vector3(modelMeshScale, modelMeshScale, modelMeshScale);
}
}
}
void Update()
{
if (isFacetrackingInitialized)
{
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
primaryUserID = kinectManager.GetUserIdByIndex(playerIndex);
}
// update the face tracker
isTrackingFace = false;
if (sensorData.sensorInterface.UpdateFaceTracking())
{
// estimate the tracking state
isTrackingFace = sensorData.sensorInterface.IsFaceTracked(primaryUserID);
// get the facetracking parameters
if (isTrackingFace)
{
lastFaceTrackedTime = Time.realtimeSinceStartup;
// get face rectangle
/**bGotFaceRect =*/ sensorData.sensorInterface.GetFaceRect(primaryUserID, ref faceRect);
// get head position
bGotHeadPos = sensorData.sensorInterface.GetHeadPosition(primaryUserID, ref headPos);
// get head rotation
bGotHeadRot = sensorData.sensorInterface.GetHeadRotation(primaryUserID, ref headRot);
// get the animation units
bGotAU = sensorData.sensorInterface.GetAnimUnits(primaryUserID, ref dictAU);
// get the shape units
bGotSU = sensorData.sensorInterface.GetShapeUnits(primaryUserID, ref dictSU);
if (faceModelMesh != null && faceModelMesh.activeInHierarchy)
{
// apply model vertices to the mesh
if (!bFaceModelMeshInited)
{
bFaceModelMeshInited = CreateFaceModelMesh(faceModelMesh, headPos, ref avModelVertices, ref avModelUV, ref bGotModelVertices);
}
}
if (getFaceModelData)
{
UpdateFaceModelMesh(primaryUserID, faceModelMesh, headPos, faceRect, ref avModelVertices, ref avModelUV, ref bGotModelVertices);
}
}
else if ((Time.realtimeSinceStartup - lastFaceTrackedTime) <= faceTrackingTolerance)
{
// allow tolerance in tracking
isTrackingFace = true;
}
}
if (faceModelMesh != null && bFaceModelMeshInited)
{
faceModelMesh.SetActive(isTrackingFace);
}
}
}
private void LoadModel(string modelDir)
{
string modelPath = modelCategory + "/" + modelDir + "/model";
UnityEngine.Object modelPrefab = Resources.Load(modelPath, typeof(GameObject));
if (modelPrefab == null)
{
return;
}
if (selModel != null)
{
GameObject.Destroy(selModel);
}
selModel = (GameObject)GameObject.Instantiate(modelPrefab, Vector3.zero, Quaternion.Euler(0, 180f, 0));
selModel.name = "Model" + modelDir;
AvatarController ac = selModel.GetComponent <AvatarController>();
if (ac == null)
{
ac = selModel.AddComponent <AvatarController>();
ac.playerIndex = playerIndex;
ac.mirroredMovement = true;
ac.verticalMovement = true;
ac.verticalOffset = verticalOffset;
ac.forwardOffset = forwardOffset;
ac.smoothFactor = 0f;
}
ac.posRelativeToCamera = modelRelativeToCamera;
ac.posRelOverlayColor = (foregroundCamera != null);
KinectManager km = KinectManager.Instance;
//ac.Awake();
long userId = km.GetUserIdByIndex(playerIndex);
if (userId != 0)
{
ac.SuccessfulCalibration(userId);
}
// locate the available avatar controllers
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
km.avatarControllers.Clear();
foreach (MonoBehaviour monoScript in monoScripts)
{
if ((monoScript is AvatarController) && monoScript.enabled)
{
AvatarController avatar = (AvatarController)monoScript;
km.avatarControllers.Add(avatar);
}
}
AvatarScaler scaler = selModel.GetComponent <AvatarScaler>();
if (scaler == null)
{
scaler = selModel.AddComponent <AvatarScaler>();
scaler.playerIndex = playerIndex;
scaler.mirroredAvatar = true;
scaler.continuousScaling = continuousScaling;
scaler.bodyScaleFactor = bodyScaleFactor;
scaler.bodyWidthFactor = bodyWidthFactor;
scaler.armScaleFactor = armScaleFactor;
scaler.legScaleFactor = legScaleFactor;
}
scaler.foregroundCamera = foregroundCamera;
//scaler.debugText = debugText;
//scaler.Start();
}
void Update()
{
KinectManager kinectManager = KinectManager.Instance;
// update Kinect interaction
if (kinectManager && kinectManager.IsInitialized())
{
playerUserID = kinectManager.GetUserIdByIndex(playerIndex);
if (playerUserID != 0)
{
lastUserID = playerUserID;
HandEventType handEvent = HandEventType.None;
// get the left hand state
leftHandState = kinectManager.GetLeftHandState(playerUserID);
// check if the left hand is interacting
isleftIboxValid = kinectManager.GetLeftHandInteractionBox(playerUserID, ref leftIboxLeftBotBack, ref leftIboxRightTopFront, isleftIboxValid);
//bool bLeftHandPrimaryNow = false;
// was the left hand interacting till now
bool wasLeftHandInteracting = isLeftHandInteracting;
if (isleftIboxValid && //bLeftHandPrimaryNow &&
kinectManager.GetJointTrackingState(playerUserID, (int)KinectInterop.JointType.HandLeft) != KinectInterop.TrackingState.NotTracked)
{
leftHandPos = kinectManager.GetJointPosition(playerUserID, (int)KinectInterop.JointType.HandLeft);
leftHandScreenPos.x = Mathf.Clamp01((leftHandPos.x - leftIboxLeftBotBack.x) / (leftIboxRightTopFront.x - leftIboxLeftBotBack.x));
leftHandScreenPos.y = Mathf.Clamp01((leftHandPos.y - leftIboxLeftBotBack.y) / (leftIboxRightTopFront.y - leftIboxLeftBotBack.y));
leftHandScreenPos.z = Mathf.Clamp01((leftIboxLeftBotBack.z - leftHandPos.z) / (leftIboxLeftBotBack.z - leftIboxRightTopFront.z));
isLeftHandInteracting = (leftHandPos.x >= (leftIboxLeftBotBack.x - 1.0f)) && (leftHandPos.x <= (leftIboxRightTopFront.x + 0.5f)) &&
(leftHandPos.y >= (leftIboxLeftBotBack.y - 0.1f)) && (leftHandPos.y <= (leftIboxRightTopFront.y + 0.7f)) &&
(leftIboxLeftBotBack.z >= leftHandPos.z) && (leftIboxRightTopFront.z * 0.8f <= leftHandPos.z);
//bLeftHandPrimaryNow = isLeftHandInteracting;
// start interacting?
if (!wasLeftHandInteracting && isLeftHandInteracting)
{
leftHandInteractingSince = Time.realtimeSinceStartup;
}
// check for left press
isLeftHandPress = ((leftIboxRightTopFront.z - 0.1f) >= leftHandPos.z);
// check for left hand click
if (allowHandClicks && !dragInProgress && isLeftHandInteracting &&
((leftHandPos - lastLeftHandPos).magnitude < KinectInterop.Constants.ClickMaxDistance))
{
if ((Time.realtimeSinceStartup - lastLeftHandTime) >= KinectInterop.Constants.ClickStayDuration)
{
if (!isLeftHandClick)
{
isLeftHandClick = true;
leftHandClickProgress = 1f;
foreach (InteractionListenerInterface listener in interactionListeners)
{
if (listener.HandClickDetected(playerUserID, playerIndex, false, leftHandScreenPos))
{
isLeftHandClick = false;
leftHandClickProgress = 0f;
lastLeftHandPos = Vector3.zero;
lastLeftHandTime = Time.realtimeSinceStartup;
}
}
if (controlMouseCursor)
{
MouseControl.MouseClick();
isLeftHandClick = false;
leftHandClickProgress = 0f;
lastLeftHandPos = Vector3.zero;
lastLeftHandTime = Time.realtimeSinceStartup;
}
}
}
else
{
leftHandClickProgress = (Time.realtimeSinceStartup - lastLeftHandTime) / KinectInterop.Constants.ClickStayDuration;
}
}
else
{
isLeftHandClick = false;
leftHandClickProgress = 0f;
lastLeftHandPos = leftHandPos;
lastLeftHandTime = Time.realtimeSinceStartup;
}
}
else
{
isLeftHandInteracting = false;
isLeftHandPress = false;
}
// get the right hand state
rightHandState = kinectManager.GetRightHandState(playerUserID);
// check if the right hand is interacting
isRightIboxValid = kinectManager.GetRightHandInteractionBox(playerUserID, ref rightIboxLeftBotBack, ref rightIboxRightTopFront, isRightIboxValid);
//bool bRightHandPrimaryNow = false;
// was the right hand interacting till now
bool wasRightHandInteracting = isRightHandInteracting;
if (isRightIboxValid && //bRightHandPrimaryNow &&
kinectManager.GetJointTrackingState(playerUserID, (int)KinectInterop.JointType.HandRight) != KinectInterop.TrackingState.NotTracked)
{
rightHandPos = kinectManager.GetJointPosition(playerUserID, (int)KinectInterop.JointType.HandRight);
rightHandScreenPos.x = Mathf.Clamp01((rightHandPos.x - rightIboxLeftBotBack.x) / (rightIboxRightTopFront.x - rightIboxLeftBotBack.x));
rightHandScreenPos.y = Mathf.Clamp01((rightHandPos.y - rightIboxLeftBotBack.y) / (rightIboxRightTopFront.y - rightIboxLeftBotBack.y));
rightHandScreenPos.z = Mathf.Clamp01((rightIboxLeftBotBack.z - rightHandPos.z) / (rightIboxLeftBotBack.z - rightIboxRightTopFront.z));
isRightHandInteracting = (rightHandPos.x >= (rightIboxLeftBotBack.x - 0.5f)) && (rightHandPos.x <= (rightIboxRightTopFront.x + 1.0f)) &&
(rightHandPos.y >= (rightIboxLeftBotBack.y - 0.1f)) && (rightHandPos.y <= (rightIboxRightTopFront.y + 0.7f)) &&
(rightIboxLeftBotBack.z >= rightHandPos.z) && (rightIboxRightTopFront.z * 0.8f <= rightHandPos.z);
//bRightHandPrimaryNow = isRightHandInteracting;
if (!wasRightHandInteracting && isRightHandInteracting)
{
rightHandInteractingSince = Time.realtimeSinceStartup;
}
// check for right press
isRightHandPress = ((rightIboxRightTopFront.z - 0.1f) >= rightHandPos.z);
// check for right hand click
if (allowHandClicks && !dragInProgress && isRightHandInteracting &&
((rightHandPos - lastRightHandPos).magnitude < KinectInterop.Constants.ClickMaxDistance))
{
if ((Time.realtimeSinceStartup - lastRightHandTime) >= KinectInterop.Constants.ClickStayDuration)
{
if (!isRightHandClick)
{
isRightHandClick = true;
rightHandClickProgress = 1f;
foreach (InteractionListenerInterface listener in interactionListeners)
{
if (listener.HandClickDetected(playerUserID, playerIndex, true, rightHandScreenPos))
{
isRightHandClick = false;
rightHandClickProgress = 0f;
lastRightHandPos = Vector3.zero;
lastRightHandTime = Time.realtimeSinceStartup;
}
}
if (controlMouseCursor)
{
MouseControl.MouseClick();
isRightHandClick = false;
rightHandClickProgress = 0f;
lastRightHandPos = Vector3.zero;
lastRightHandTime = Time.realtimeSinceStartup;
}
}
}
else
{
rightHandClickProgress = (Time.realtimeSinceStartup - lastRightHandTime) / KinectInterop.Constants.ClickStayDuration;
}
}
else
{
isRightHandClick = false;
rightHandClickProgress = 0f;
lastRightHandPos = rightHandPos;
lastRightHandTime = Time.realtimeSinceStartup;
}
}
else
{
isRightHandInteracting = false;
isRightHandPress = false;
}
// if both hands are interacting, check which one interacts longer than the other
if (isLeftHandInteracting && isRightHandInteracting)
{
if (rightHandInteractingSince <= leftHandInteractingSince)
{
isLeftHandInteracting = false;
}
else
{
isRightHandInteracting = false;
}
}
// if left hand just stopped interacting, send extra non-interaction event
if (wasLeftHandInteracting && !isLeftHandInteracting)
{
foreach (InteractionListenerInterface listener in interactionListeners)
{
// if(lastLeftHandEvent == HandEventType.Grip)
// listener.HandGripDetected (playerUserID, playerIndex, false, isLeftHandInteracting, leftHandScreenPos);
// else //if(lastLeftHandEvent == HandEventType.Release)
// listener.HandReleaseDetected (playerUserID, playerIndex, false, isLeftHandInteracting, leftHandScreenPos);
if (lastLeftHandEvent == HandEventType.Grip)
{
listener.HandReleaseDetected(playerUserID, playerIndex, false, true, leftHandScreenPos);
}
}
}
// if right hand just stopped interacting, send extra non-interaction event
if (wasRightHandInteracting && !isRightHandInteracting)
{
foreach (InteractionListenerInterface listener in interactionListeners)
{
// if(lastRightHandEvent == HandEventType.Grip)
// listener.HandGripDetected (playerUserID, playerIndex, true, isRightHandInteracting, rightHandScreenPos);
// else //if(lastRightHandEvent == HandEventType.Release)
// listener.HandReleaseDetected (playerUserID, playerIndex, true, isRightHandInteracting, rightHandScreenPos);
if (lastRightHandEvent == HandEventType.Grip)
{
listener.HandReleaseDetected(playerUserID, playerIndex, true, true, rightHandScreenPos);
}
}
}
// process left hand
handEvent = HandStateToEvent(leftHandState, lastLeftHandEvent);
if ((isLeftHandInteracting != isLeftHandPrimary) || (isRightHandInteracting != isRightHandPrimary))
{
if (controlMouseCursor && dragInProgress)
{
MouseControl.MouseRelease();
dragInProgress = false;
}
lastLeftHandEvent = HandEventType.Release;
lastRightHandEvent = HandEventType.Release;
}
if (controlMouseCursor && (handEvent != lastLeftHandEvent))
{
if (controlMouseDrag && !dragInProgress && (handEvent == HandEventType.Grip))
{
dragInProgress = true;
MouseControl.MouseDrag();
}
else if (dragInProgress && (handEvent == HandEventType.Release))
{
MouseControl.MouseRelease();
dragInProgress = false;
}
}
leftHandEvent = handEvent;
if (handEvent != HandEventType.None)
{
if (leftHandEvent != lastLeftHandEvent)
{
foreach (InteractionListenerInterface listener in interactionListeners)
{
if (leftHandEvent == HandEventType.Grip)
{
listener.HandGripDetected(playerUserID, playerIndex, false, isLeftHandInteracting, leftHandScreenPos);
}
else if (leftHandEvent == HandEventType.Release)
{
listener.HandReleaseDetected(playerUserID, playerIndex, false, isLeftHandInteracting, leftHandScreenPos);
}
}
}
lastLeftHandEvent = handEvent;
}
// if the hand is primary, set the cursor position
if (isLeftHandInteracting)
{
isLeftHandPrimary = true;
if ((leftHandClickProgress < 0.8f) /**&& !isLeftHandPress*/)
{
float smooth = smoothFactor * Time.deltaTime;
if (smooth == 0f)
{
smooth = 1f;
}
cursorScreenPos = Vector3.Lerp(cursorScreenPos, leftHandScreenPos, smooth);
}
// move mouse-only if there is no cursor texture
if (controlMouseCursor &&
(!guiHandCursor || (!gripHandTexture && !releaseHandTexture && !normalHandTexture)))
{
MouseControl.MouseMove(cursorScreenPos, debugText);
}
}
else
{
isLeftHandPrimary = false;
}
// process right hand
handEvent = HandStateToEvent(rightHandState, lastRightHandEvent);
if (controlMouseCursor && (handEvent != lastRightHandEvent))
{
if (controlMouseDrag && !dragInProgress && (handEvent == HandEventType.Grip))
{
dragInProgress = true;
MouseControl.MouseDrag();
}
else if (dragInProgress && (handEvent == HandEventType.Release))
{
MouseControl.MouseRelease();
dragInProgress = false;
}
}
rightHandEvent = handEvent;
if (handEvent != HandEventType.None)
{
if (rightHandEvent != lastRightHandEvent)
{
foreach (InteractionListenerInterface listener in interactionListeners)
{
if (rightHandEvent == HandEventType.Grip)
{
listener.HandGripDetected(playerUserID, playerIndex, true, isRightHandInteracting, rightHandScreenPos);
}
else if (rightHandEvent == HandEventType.Release)
{
listener.HandReleaseDetected(playerUserID, playerIndex, true, isRightHandInteracting, rightHandScreenPos);
}
}
}
lastRightHandEvent = handEvent;
}
// if the hand is primary, set the cursor position
if (isRightHandInteracting)
{
isRightHandPrimary = true;
if ((rightHandClickProgress < 0.8f) /**&& !isRightHandPress*/)
{
float smooth = smoothFactor * Time.deltaTime;
if (smooth == 0f)
{
smooth = 1f;
}
cursorScreenPos = Vector3.Lerp(cursorScreenPos, rightHandScreenPos, smooth);
}
// move mouse-only if there is no cursor texture
if (controlMouseCursor &&
(!guiHandCursor || (!gripHandTexture && !releaseHandTexture && !normalHandTexture)))
{
MouseControl.MouseMove(cursorScreenPos, debugText);
}
}
else
{
isRightHandPrimary = false;
}
}
else
{
// send release events
if (lastLeftHandEvent == HandEventType.Grip || lastRightHandEvent == HandEventType.Grip)
{
foreach (InteractionListenerInterface listener in interactionListeners)
{
if (lastLeftHandEvent == HandEventType.Grip)
{
listener.HandReleaseDetected(lastUserID, playerIndex, false, true, leftHandScreenPos);
}
if (lastRightHandEvent == HandEventType.Grip)
{
listener.HandReleaseDetected(lastUserID, playerIndex, true, true, leftHandScreenPos);
}
}
}
leftHandState = KinectInterop.HandState.NotTracked;
rightHandState = KinectInterop.HandState.NotTracked;
isLeftHandPrimary = isRightHandPrimary = false;
isLeftHandInteracting = isRightHandInteracting = false;
leftHandInteractingSince = rightHandInteractingSince = 0f;
isLeftHandClick = isRightHandClick = false;
leftHandClickProgress = rightHandClickProgress = 0f;
lastLeftHandTime = lastRightHandTime = Time.realtimeSinceStartup;
isLeftHandPress = false;
isRightHandPress = false;
leftHandEvent = HandEventType.None;
rightHandEvent = HandEventType.None;
lastLeftHandEvent = HandEventType.Release;
lastRightHandEvent = HandEventType.Release;
if (controlMouseCursor && dragInProgress)
{
MouseControl.MouseRelease();
dragInProgress = false;
}
}
}
}
// ApplyBoneOrientationConstraints and constrain rotations.
public void Constrain(ref KinectInterop.BodyData bodyData)
{
KinectManager manager = KinectManager.Instance;
for (int i = 0; i < this.jointConstraints.Count; i++)
{
BoneOrientationConstraint jc = this.jointConstraints[i];
if (jc.thisJoint == (int)JointType.SpineBase || bodyData.joint[jc.thisJoint].normalRotation == Quaternion.identity)
{
continue;
}
if (bodyData.joint[jc.thisJoint].trackingState == TrackingState.NotTracked)
{
continue;
}
int prevJoint = (int)KinectInterop.GetParentJoint((JointType)jc.thisJoint);
if (bodyData.joint[prevJoint].trackingState == TrackingState.NotTracked)
{
continue;
}
Quaternion rotJointN = bodyData.joint[jc.thisJoint].normalRotation;
Quaternion rotParentN = bodyData.joint[prevJoint].normalRotation;
Quaternion rotLocalN = Quaternion.Inverse(rotParentN) * rotJointN;
Vector3 eulerAnglesN = rotLocalN.eulerAngles;
Quaternion rotJointM = bodyData.joint[jc.thisJoint].mirroredRotation;
Quaternion rotParentM = bodyData.joint[prevJoint].mirroredRotation;
Quaternion rotLocalM = Quaternion.Inverse(rotParentM) * rotJointM;
Vector3 eulerAnglesM = rotLocalM.eulerAngles;
bool isConstrained = false;
for (int a = 0; a < jc.axisConstrainrs.Count; a++)
{
AxisOrientationConstraint ac = jc.axisConstrainrs[a];
Quaternion axisRotation = Quaternion.AngleAxis(eulerAnglesN[ac.axis], ac.rotateAround);
float angleFromMin = Quaternion.Angle(axisRotation, ac.minQuaternion);
float angleFromMax = Quaternion.Angle(axisRotation, ac.maxQuaternion);
if (!(angleFromMin <= ac.angleRange && angleFromMax <= ac.angleRange))
{
// correct the axis that has fallen out of range.
if (angleFromMin > angleFromMax)
{
eulerAnglesN[ac.axis] = ac.angleMax;
}
else
{
eulerAnglesN[ac.axis] = ac.angleMin;
}
// fix mirrored rotation as well
if (ac.axis == 0)
{
eulerAnglesM[ac.axis] = eulerAnglesN[ac.axis];
}
else
{
eulerAnglesM[ac.axis] = -eulerAnglesN[ac.axis];
}
isConstrained = true;
}
}
if (isConstrained)
{
rotLocalN = Quaternion.Euler(eulerAnglesN);
rotJointN = rotParentN * rotLocalN;
rotLocalM = Quaternion.Euler(eulerAnglesM);
rotJointM = rotParentM * rotLocalM;
// Put it back into the bone directions
bodyData.joint[jc.thisJoint].normalRotation = rotJointN;
bodyData.joint[jc.thisJoint].mirroredRotation = rotJointM;
// dirJoint = constrainedRotation * dirParent;
// bodyData.joint[jc.thisJoint].direction = dirJoint;
}
}
}
// Update is called once per frame
void Update()
{
KinectManager manager = KinectManager.Instance;
// get 1st player
uint playerID = manager != null?manager.GetPlayer1ID() : 0;
if (playerID <= 0)
{
// reset the pointman position and rotation
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (SkeletonLine)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the user position in space
Vector3 posPointMan = manager.GetUserPosition(playerID);
posPointMan.z = !MirroredMovement ? -posPointMan.z : posPointMan.z;
// store the initial position
if (initialPosUserID != playerID)
{
initialPosUserID = playerID;
initialPosOffset = transform.position - (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
}
transform.position = initialPosOffset + (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = MirroredMovement ? KinectWrapper.GetSkeletonMirroredJoint(i): i;
if (manager.IsJointTracked(playerID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(playerID, joint);
posJoint.z = !MirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(playerID, joint, !MirroredMovement);
rotJoint = initialRotation * rotJoint;
posJoint -= posPointMan;
if (MirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (i == 3)
{
UnityEngine.Debug.Log("head : " + posJoint);
}
}
else
{
bones[i].gameObject.SetActive(false);
}
}
}
if (SkeletonLine)
{
for (int i = 0; i < bones.Length; i++)
{
bool bLineDrawn = false;
if (bones[i] != null)
{
if (bones[i].gameObject.activeSelf)
{
Vector3 posJoint = bones[i].transform.position;
int parI = parIdxs[i];
Vector3 posParent = bones[parI].transform.position;
if (bones[parI].gameObject.activeSelf)
{
lines[i].gameObject.SetActive(true);
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint);
bLineDrawn = true;
}
}
}
if (!bLineDrawn)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
// Use this for initialization
void Start()
{
kinectManager = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <KinectManager>();
interactionManager = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <InteractionManager>();
leftPlayerThumbTextPos = GameObject.FindGameObjectWithTag("leftthumb").GetComponent <Text>();
}
private void Update()
{
KinectManager manager = KinectManager.Instance;
if (!manager || !manager.IsInitialized() || !manager.IsUserDetected())
{
return;
}
int iJointIndex = (int)TrackedJoint;
if (manager.IsUserDetected())
{
uint userId = manager.GetPlayer1ID();
if (manager.IsJointTracked(userId, iJointIndex))
{
Vector3 posJoint = manager.GetRawSkeletonJointPos(userId, iJointIndex);
if (posJoint != Vector3.zero)
{
if (!flag)
{
aurora.SetActive(true);
time = 0;
flag3 = true;
flag = true;
}
if (posJoint.x <= rangMax && posJoint.x >= rangMin)
{
aurora.GetComponent <MovimentAurores>().enabled = true;
auroraAnim.SetFloat("revert", 1f);
if (flag4)
{
auroraSound.Play();
flag4 = false;
}
}
else
{
if (flag)
{
flag3 = false;
if (time > 0)
{
auroraAnim.SetFloat("revert", -1f);
flag = false;
flag4 = true;
}
}
}
}
}
}
if (flag3)
{
if (time >= 0)
{
time++;
}
}
if (!flag)
{
if (time >= 0)
{
time--;
}
if (time <= 0)
{
auroraAnim.SetFloat("revert", 0f);
}
}
// Debug.Log(time);
}
public bool GestureCancelled(KinectManager manager, long userId, int userIndex, KinectGestures.Gestures gesture, KinectInterop.JointType joint)
{
throw new NotImplementedException();
}
void Update()
{
KinectManager manager = KinectManager.Instance;
if (manager && manager.IsInitialized())
{
if (backgroundImage && (backgroundImage.texture == null))
{
backgroundImage.texture = manager.GetUsersClrTex();
}
}
if (currentIndex != prevIndex)
{
prevIndex = currentIndex;
if (headOverlayer && headMasks != null)
{
if (headOverlayer.overlayObject)
{
headOverlayer.overlayObject.gameObject.SetActive(false);
}
headOverlayer.overlayObject = currentIndex >= 0 && currentIndex < headMasks.Length ? headMasks [currentIndex] : null;
headOverlayer.playerIndex = playerIndex;
if (headOverlayer.overlayObject)
{
headOverlayer.overlayObject.gameObject.SetActive(true);
}
}
if (leftHandOverlayer && leftHandMasks != null)
{
if (leftHandOverlayer.overlayObject)
{
leftHandOverlayer.overlayObject.gameObject.SetActive(false);
}
leftHandOverlayer.overlayObject = currentIndex >= 0 && currentIndex < leftHandMasks.Length ? leftHandMasks [currentIndex] : null;
leftHandOverlayer.playerIndex = playerIndex;
if (leftHandOverlayer.overlayObject)
{
leftHandOverlayer.overlayObject.gameObject.SetActive(true);
}
}
if (chestOverlayer && chestMasks != null)
{
if (chestOverlayer.overlayObject)
{
chestOverlayer.overlayObject.gameObject.SetActive(false);
}
chestOverlayer.overlayObject = currentIndex >= 0 && currentIndex < chestMasks.Length ? chestMasks [currentIndex] : null;
chestOverlayer.playerIndex = playerIndex;
if (chestOverlayer.overlayObject)
{
chestOverlayer.overlayObject.gameObject.SetActive(true);
}
}
}
}
public bool GestureCompleted(KinectManager manager, long userId, int userIndex, KinectGestures.Gestures gesture, KinectInterop.JointType joint, Vector3 screenPos)
{
throw new NotImplementedException();
}
void Update()
{
KinectManager manager = KinectManager.Instance;
if (manager && manager.IsInitialized() && foregroundCamera)
{
//backgroundImage.renderer.material.mainTexture = manager.GetUsersClrTex();
if (backgroundImage && (backgroundImage.texture == null))
{
backgroundImage.texture = manager.GetUsersClrTex();
}
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if (portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
// overlay all joints in the skeleton
if (manager.IsUserDetected())
{
long userId = manager.GetUserIdByIndex(playerIndex);
int jointsCount = manager.GetJointCount();
for (int i = 0; i < jointsCount; i++)
{
int joint = i;
if (manager.IsJointTracked(userId, joint))
{
Vector3 posJoint = manager.GetJointPosColorOverlay(userId, joint, foregroundCamera, backgroundRect);
//Vector3 posJoint = manager.GetJointPosition(userId, joint);
if (joints != null)
{
// overlay the joint
if (posJoint != Vector3.zero)
{
// if(debugText && joint == 0)
// {
// debugText.GetComponent<GUIText>().text = string.Format("{0} - {1}\nRealPos: {2}",
// (KinectInterop.JointType)joint, posJoint,
// manager.GetJointPosition(userId, joint));
// }
joints[i].SetActive(true);
joints[i].transform.position = posJoint;
Quaternion rotJoint = manager.GetJointOrientation(userId, joint, false);
rotJoint = initialRotation * rotJoint;
joints[i].transform.rotation = rotJoint;
}
else
{
joints[i].SetActive(false);
}
}
if (lines[i] == null && linePrefab != null)
{
lines[i] = Instantiate(linePrefab) as LineRenderer;
lines[i].transform.parent = transform;
lines[i].gameObject.SetActive(false);
}
if (lines[i] != null)
{
// overlay the line to the parent joint
int jointParent = (int)manager.GetParentJoint((KinectInterop.JointType)joint);
Vector3 posParent = manager.GetJointPosColorOverlay(userId, jointParent, foregroundCamera, backgroundRect);
if (posJoint != Vector3.zero && posParent != Vector3.zero)
{
lines[i].gameObject.SetActive(true);
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint);
}
else
{
lines[i].gameObject.SetActive(false);
}
}
}
else
{
if (joints != null)
{
joints[i].SetActive(false);
}
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
}
void Update()
{
debugText.GetComponent <GUIText>().text = "MARICO";
KinectManager manager = KinectManager.Instance;
if (manager && manager.IsInitialized())
{
//backgroundImage.GetComponent<Renderer>().material.mainTexture = manager.GetUsersClrTex();
/* if(backgroundImage && (backgroundImage.texture == null))
* {
* backgroundImage.texture = manager.GetUsersClrTex();
* }*/
// Vector3 vRight = BottomRight - BottomLeft;
// Vector3 vUp = TopLeft - BottomLeft;
int iJointIndex = (int)TrackedJoint;
if (manager.IsUserDetected() && gestureListener)
{
uint userId = manager.GetPlayer1ID();
if (manager.IsJointTracked(userId, iJointIndex))
{
Vector3 posJoint = manager.GetRawSkeletonJointPos(userId, iJointIndex);
if (gestureListener.IsPush())
{
debugText.GetComponent <GUIText>().text = "PUSHH ";
Debug.Log("PUSHHHHHH");
i++;
}
if (posJoint != Vector3.zero)
{
// 3d position to depth
Vector2 posDepth = manager.GetDepthMapPosForJointPos(posJoint);
// depth pos to color pos
Vector2 posColor = manager.GetColorMapPosForDepthPos(posDepth);
float scaleX = (float)posColor.x / KinectWrapper.Constants.ColorImageWidth;
float scaleY = 1.0f - (float)posColor.y / KinectWrapper.Constants.ColorImageHeight;
// Vector3 localPos = new Vector3(scaleX * 10f - 5f, 0f, scaleY * 10f - 5f); // 5f is 1/2 of 10f - size of the plane
// Vector3 vPosOverlay = backgroundImage.transform.TransformPoint(localPos);
//Vector3 vPosOverlay = BottomLeft + ((vRight * scaleX) + (vUp * scaleY));
if (debugText)
{
debugText.GetComponent <GUIText>().text = "Tracked user ID: " + userId; // new Vector2(scaleX, scaleY).ToString();
}
if (OverlayObject)
{
Vector3 vPosOverlay = Camera.main.ViewportToWorldPoint(new Vector3(scaleX, scaleY, distanceToCamera));
OverlayObject.transform.position = Vector3.Lerp(OverlayObject.transform.position, vPosOverlay, smoothFactor * Time.deltaTime);
}
}
}
}
}
}
void Update()
{
KinectManager manager = KinectManager.Instance;
// get 1st player
Int64 userID = manager ? manager.GetPrimaryUserID() : 0;
if (userID <= 0)
{
// reset the pointman position and rotation
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (SkeletonLine)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the position in space
Vector3 posPointMan = manager.GetUserPosition(userID);
posPointMan.z = !MirroredMovement ? -posPointMan.z : posPointMan.z;
// store the initial position
if (initialPosUserID != userID)
{
initialPosUserID = userID;
initialPosOffset = transform.position - (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
}
transform.position = initialPosOffset + (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = (int)manager.GetJointAtIndex(
!MirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i));
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !MirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !MirroredMovement);
rotJoint = initialRotation * rotJoint;
posJoint -= posPointMan;
if (MirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (SkeletonLine)
{
lines[i].gameObject.SetActive(true);
Vector3 posJoint2 = bones[i].transform.position;
Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
dirFromParent.z = !MirroredMovement ? -dirFromParent.z : dirFromParent.z;
Vector3 posParent = posJoint2 - dirFromParent;
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint2);
}
}
else
{
bones[i].gameObject.SetActive(false);
if (SkeletonLine)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
void Awake()
{
mgr = GetComponent <KinectManager>();
}
void Update()
{
KinectManager manager = KinectManager.Instance;
if (manager && manager.IsInitialized())
{
// //backgroundImage.renderer.material.mainTexture = manager.GetUsersClrTex();
// if(backgroundImage && (backgroundImage.texture == null))
// {
// backgroundImage.texture = manager.GetUsersClrTex();
// }
// overlay all joints in the skeleton
if (manager.IsUserDetected(playerIndex))
{
long userId = manager.GetUserIdByIndex(playerIndex);
int jointsCount = manager.GetJointCount();
for (int i = 0; i < jointsCount; i++)
{
int joint = i;
if (manager.IsJointTracked(userId, joint))
{
Vector3 posJoint = !kinectCamera?manager.GetJointPosition(userId, joint) : manager.GetJointKinectPosition(userId, joint);
posJoint = new Vector3(posJoint.x * scaleFactors.x, posJoint.y * scaleFactors.y, posJoint.z * scaleFactors.z);
if (kinectCamera)
{
posJoint = kinectCamera.transform.TransformPoint(posJoint);
}
if (joints != null)
{
// overlay the joint
if (posJoint != Vector3.zero)
{
joints[i].SetActive(true);
joints[i].transform.position = posJoint;
// Quaternion rotJoint = manager.GetJointOrientation(userId, joint, false);
// rotJoint = initialRotation * rotJoint;
// joints[i].transform.rotation = rotJoint;
}
else
{
joints[i].SetActive(false);
}
}
if (lines[i] == null && linePrefab != null)
{
lines[i] = Instantiate(linePrefab); // as LineRenderer;
lines[i].transform.parent = transform;
lines[i].name = ((KinectInterop.JointType)i).ToString() + "_Line";
lines[i].gameObject.SetActive(false);
}
if (lines[i] != null)
{
// overlay the line to the parent joint
int jointParent = (int)manager.GetParentJoint((KinectInterop.JointType)joint);
Vector3 posParent = Vector3.zero;
if (manager.IsJointTracked(userId, jointParent))
{
posParent = !kinectCamera?manager.GetJointPosition(userId, jointParent) : manager.GetJointKinectPosition(userId, jointParent);
posJoint = new Vector3(posJoint.x * scaleFactors.x, posJoint.y * scaleFactors.y, posJoint.z * scaleFactors.z);
if (kinectCamera)
{
posParent = kinectCamera.transform.TransformPoint(posParent);
}
}
if (posJoint != Vector3.zero && posParent != Vector3.zero)
{
lines[i].gameObject.SetActive(true);
// //lines[i].SetVertexCount(2);
// lines[i].SetPosition(0, posParent);
// lines[i].SetPosition(1, posJoint);
Vector3 dirFromParent = posJoint - posParent;
lines [i].transform.localPosition = posParent + dirFromParent / 2f;
lines [i].transform.up = transform.rotation * dirFromParent.normalized;
Vector3 lineScale = lines [i].transform.localScale;
lines [i].transform.localScale = new Vector3(lineScale.x, dirFromParent.magnitude / 2f, lineScale.z);
}
else
{
lines[i].gameObject.SetActive(false);
}
}
}
else
{
if (joints != null)
{
joints[i].SetActive(false);
}
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
}
// gets the difference between the User pose and the list of saved poses
private void GetPoseDifference(bool isMirrored)
{
// by-default values
bPoseMatched = false;
fMatchPercent = 0f;
fMatchPoseTime = 0f;
KinectManager kinectManager = KinectManager.Instance;
if (poseJoints == null || poseUser.avBoneDirs == null)
{
return;
}
if (sbDebug != null)
{
sbDebug.Clear();
sbDebug.AppendLine();
}
// check the difference with saved poses, starting from the last one
for (int p = 0; p < 2; p++)
{
float fAngleDiff = 0f;
float fMaxDiff = 0f;
PoseModelDataClass modelPose = PoseArchive.AllSavedPoses[p + exerNum];
for (int i = 0; i < poseJoints.Count; i++)
{
Vector3 vPoseBone = modelPose.BonePos[i];
Vector3 vUserBone = poseUser.avBoneDirs[i];
if (vPoseBone == Vector3.zero || vUserBone == Vector3.zero)
{
continue;
}
float fDiff = Vector3.Angle(vPoseBone, vUserBone);
if (fDiff > 90f)
{
fDiff = 90f;
}
fAngleDiff += fDiff;
fMaxDiff += 90f; // we assume the max diff could be 90 degrees
if (sbDebug != null)
{
sbDebug.AppendFormat("SP: {0}, {1} - angle: {2:F0}, match: {3:F0}%", p, poseJoints[i], fDiff, (1f - fDiff / 90f) * 100f);
sbDebug.AppendLine();
}
}
float fPoseMatch = fMaxDiff > 0f ? (1f - fAngleDiff / fMaxDiff) : 0f;
if (fPoseMatch > fMatchPercent)
{
fMatchPercent = fPoseMatch;
//fMatchPoseTime = poseModel.fTime;
bPoseMatched = (fMatchPercent >= matchThreshold);
if (bPoseMatched)
{
matchPercentIndex[p] = fMatchPercent * 100f;
}
}
}
Debug.Log(new Vector2(matchPercentIndex[0], matchPercentIndex[0]));
if (matchPercentIndex[0] != 0 && matchPercentIndex[1] != 0)
{
Debug.Log(new Vector2(matchPercentIndex[0], matchPercentIndex[1]));
if (matchPercentIndex[0] / matchPercentIndex[1] > 1)
{
identifiedPose = 1;
}
else
{
identifiedPose = 2;
}
}
else
{
identifiedPose = 0;
}
}
// gets the current User pose
private void GetUserPose(float fCurrentTime, bool isMirrored)
{
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager == null || User == null || poseJoints == null)
{
return;
}
//poseUser.fTime = fCurrentTime;
if (poseUser.avBoneDirs == null)
{
poseUser.avBoneDirs = new Vector3[poseJoints.Count];
}
//Use User not poseModel
/*
* for (int i = 0; i < poseJoints.Count; i++)
* {
* poseUser.avBoneDirs[i] = User.GetBoneTransform(User.GetBoneIndexByJoint(poseJoints[i], false)).position;
* }*/
/*
* Transform headTransform = User.GetBoneTransform(User.GetBoneIndexByJoint(Head, isMirrored));
* for (int i = 0; i < poseJoints.Count; i++)
* {
* KinectInterop.JointType joint = poseJoints[i];
* //KinectInterop.JointType nextJoint = kinectManager.GetNextJoint(joint);
*
* int jointCount = kinectManager.GetJointCount();
*
* if(Head != joint)
* {
* Transform userJointTransform = User.GetBoneTransform(User.GetBoneIndexByJoint(joint, isMirrored));
*
* if (userJointTransform != null && headTransform != null)
* {
* poseUser.avBoneDirs[i] = (headTransform.position - userJointTransform.position).normalized;
* }
* }
* }*/
for (int i = 0; i < poseJoints.Count; i++)
{
KinectInterop.JointType joint = poseJoints[i];
KinectInterop.JointType nextJoint = kinectManager.GetNextJoint(joint);
int jointCount = kinectManager.GetJointCount();
if (nextJoint != joint && (int)nextJoint >= 0 && (int)nextJoint < jointCount)
{
Transform avatarTransform1 = User.GetBoneTransform(User.GetBoneIndexByJoint(joint, isMirrored));
Transform avatarTransform2 = User.GetBoneTransform(User.GetBoneIndexByJoint(nextJoint, isMirrored));
if (avatarTransform1 != null && avatarTransform2 != null)
{
poseUser.avBoneDirs[i] = (avatarTransform2.position - avatarTransform1.position).normalized;
//Debug.Log(poseUser.avBoneDirs[i]);
}
}
}
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "FM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Face tracking is not supported!");
}
// Initialize the face tracker
if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
{
throw new Exception("Face tracking could not be initialized.");
}
instance = this;
isFacetrackingInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
void Update()
{
KinectManager kinectManager = KinectManager.Instance;
// update Kinect interaction
if (kinectManager && kinectManager.IsInitialized())
{
primaryUserID = kinectManager.GetPrimaryUserID();
if (primaryUserID != 0)
{
HandEventType handEvent = HandEventType.None;
// get the left hand state
leftHandState = kinectManager.GetLeftHandState(primaryUserID);
// check if the left hand is interacting
isleftIboxValid = kinectManager.GetLeftHandInteractionBox(primaryUserID, ref leftIboxLeftBotBack, ref leftIboxRightTopFront, isleftIboxValid);
//bool bLeftHandPrimaryNow = false;
if (isleftIboxValid && //bLeftHandPrimaryNow &&
kinectManager.GetJointTrackingState(primaryUserID, (int)JointType.HandLeft) != TrackingState.NotTracked)
{
Vector3 leftHandPos = kinectManager.GetJointPosition(primaryUserID, (int)JointType.HandLeft);
leftHandScreenPos.x = Mathf.Clamp01((leftHandPos.x - leftIboxLeftBotBack.x) / (leftIboxRightTopFront.x - leftIboxLeftBotBack.x));
leftHandScreenPos.y = Mathf.Clamp01((leftHandPos.y - leftIboxLeftBotBack.y) / (leftIboxRightTopFront.y - leftIboxLeftBotBack.y));
leftHandScreenPos.z = Mathf.Clamp01((leftIboxLeftBotBack.z - leftHandPos.z) / (leftIboxLeftBotBack.z - leftIboxRightTopFront.z));
bool wasLeftHandInteracting = isLeftHandInteracting;
isLeftHandInteracting = (leftHandPos.x >= (leftIboxLeftBotBack.x - 1.0f)) && (leftHandPos.x <= (leftIboxRightTopFront.x + 1.0f)) &&
(leftHandPos.y >= (leftIboxLeftBotBack.y - 0.1f)) && (leftHandPos.y <= (leftIboxRightTopFront.y + 0.7f)) &&
(leftIboxLeftBotBack.z >= leftHandPos.z) && (leftIboxRightTopFront.z <= leftHandPos.z);
//bLeftHandPrimaryNow = isLeftHandInteracting;
if (!wasLeftHandInteracting && isLeftHandInteracting)
{
leftHandInteractingSince = Time.realtimeSinceStartup;
}
// check for left hand click
float fClickDist = (leftHandPos - lastLeftHandPos).magnitude;
if (fClickDist < KinectInterop.Constants.ClickMaxDistance)
{
if ((Time.realtimeSinceStartup - lastLeftHandTime) >= KinectInterop.Constants.ClickStayDuration)
{
isLeftHandClick = true;
leftHandClickProgress = 1f;
}
else
{
leftHandClickProgress = (Time.realtimeSinceStartup - lastLeftHandTime) / KinectInterop.Constants.ClickStayDuration;
}
}
else
{
isLeftHandClick = false;
leftHandClickProgress = 0f;
lastLeftHandPos = leftHandPos;
lastLeftHandTime = Time.realtimeSinceStartup;
}
}
else
{
isLeftHandInteracting = false;
}
// get the right hand state
rightHandState = kinectManager.GetRightHandState(primaryUserID);
// check if the right hand is interacting
isRightIboxValid = kinectManager.GetRightHandInteractionBox(primaryUserID, ref rightIboxLeftBotBack, ref rightIboxRightTopFront, isRightIboxValid);
//bool bRightHandPrimaryNow = false;
if (isRightIboxValid && //bRightHandPrimaryNow &&
kinectManager.GetJointTrackingState(primaryUserID, (int)JointType.HandRight) != TrackingState.NotTracked)
{
Vector3 rightHandPos = kinectManager.GetJointPosition(primaryUserID, (int)JointType.HandRight);
rightHandScreenPos.x = Mathf.Clamp01((rightHandPos.x - rightIboxLeftBotBack.x) / (rightIboxRightTopFront.x - rightIboxLeftBotBack.x));
rightHandScreenPos.y = Mathf.Clamp01((rightHandPos.y - rightIboxLeftBotBack.y) / (rightIboxRightTopFront.y - rightIboxLeftBotBack.y));
rightHandScreenPos.z = Mathf.Clamp01((rightIboxLeftBotBack.z - rightHandPos.z) / (rightIboxLeftBotBack.z - rightIboxRightTopFront.z));
bool wasRightHandInteracting = isRightHandInteracting;
isRightHandInteracting = (rightHandPos.x >= (rightIboxLeftBotBack.x - 0.5f)) && (rightHandPos.x <= (rightIboxRightTopFront.x + 1.0f)) &&
(rightHandPos.y >= (rightIboxLeftBotBack.y - 0.1f)) && (rightHandPos.y <= (rightIboxRightTopFront.y + 0.7f)) &&
(rightIboxLeftBotBack.z >= rightHandPos.z) && (rightIboxRightTopFront.z * 0.8f <= rightHandPos.z);
//bRightHandPrimaryNow = isRightHandInteracting;
if (!wasRightHandInteracting && isRightHandInteracting)
{
rightHandInteractingSince = Time.realtimeSinceStartup;
}
if (isLeftHandInteracting && isRightHandInteracting)
{
if (rightHandInteractingSince <= leftHandInteractingSince)
{
isLeftHandInteracting = false;
}
else
{
isRightHandInteracting = false;
}
}
// check for right hand click
float fClickDist = (rightHandPos - lastRightHandPos).magnitude;
if (fClickDist < KinectInterop.Constants.ClickMaxDistance)
{
if ((Time.realtimeSinceStartup - lastRightHandTime) >= KinectInterop.Constants.ClickStayDuration)
{
isRightHandClick = true;
rightHandClickProgress = 1f;
}
else
{
rightHandClickProgress = (Time.realtimeSinceStartup - lastRightHandTime) / KinectInterop.Constants.ClickStayDuration;
}
}
else
{
isRightHandClick = false;
rightHandClickProgress = 0f;
lastRightHandPos = rightHandPos;
lastRightHandTime = Time.realtimeSinceStartup;
}
}
else
{
isRightHandInteracting = false;
}
// process left hand
handEvent = HandStateToEvent(leftHandState, lastLeftHandEvent);
if ((isLeftHandInteracting != isLeftHandPrimary) || (isRightHandInteracting != isRightHandPrimary))
{
if (controlMouseCursor && dragInProgress)
{
MouseControl.MouseRelease();
dragInProgress = false;
}
lastLeftHandEvent = HandEventType.Release;
lastRightHandEvent = HandEventType.Release;
}
if (controlMouseCursor && (handEvent != lastLeftHandEvent))
{
if (!dragInProgress && (handEvent == HandEventType.Grip))
{
dragInProgress = true;
MouseControl.MouseDrag();
}
else if (dragInProgress && (handEvent == HandEventType.Release))
{
MouseControl.MouseRelease();
dragInProgress = false;
}
}
leftHandEvent = handEvent;
if (handEvent != HandEventType.None)
{
lastLeftHandEvent = handEvent;
}
// if the hand is primary, set the cursor position
if (isLeftHandInteracting)
{
isLeftHandPrimary = true;
cursorScreenPos = leftHandScreenPos;
if (controlMouseCursor && !handCursor)
{
MouseControl.MouseMove(cursorScreenPos);
}
}
else
{
isLeftHandPrimary = false;
}
// process right hand
handEvent = HandStateToEvent(rightHandState, lastRightHandEvent);
if (controlMouseCursor && (handEvent != lastRightHandEvent))
{
if (!dragInProgress && (handEvent == HandEventType.Grip))
{
dragInProgress = true;
MouseControl.MouseDrag();
}
else if (dragInProgress && (handEvent == HandEventType.Release))
{
MouseControl.MouseRelease();
dragInProgress = false;
}
}
rightHandEvent = handEvent;
if (handEvent != HandEventType.None)
{
lastRightHandEvent = handEvent;
}
// if the hand is primary, set the cursor position
if (isRightHandInteracting)
{
isRightHandPrimary = true;
cursorScreenPos = rightHandScreenPos;
if (controlMouseCursor && !handCursor)
{
MouseControl.MouseMove(cursorScreenPos);
}
}
else
{
isRightHandPrimary = false;
}
}
else
{
leftHandState = HandState.NotTracked;
rightHandState = HandState.NotTracked;
isLeftHandPrimary = false;
isRightHandPrimary = false;
leftHandEvent = HandEventType.None;
rightHandEvent = HandEventType.None;
lastLeftHandEvent = HandEventType.Release;
lastRightHandEvent = HandEventType.Release;
if (controlMouseCursor && dragInProgress)
{
MouseControl.MouseRelease();
dragInProgress = false;
}
}
}
}
