// This function sends both how many skeletons are in the frame, and where they are
private void sendOSCSkeletonPositions(object sender, SkeletonFrameReadyEventArgs e)
{
sendOSCSkeletonCount();
List <Skeleton> skeletonList = kinectGroup.getMasterSkeletons();
OscBundle oscBundle = new OscBundle();
int skeletonNum = 0;
float xScale = sanitizeTextToFloat(XScaleTextBox.Text);
float xOffset = sanitizeTextToFloat(XOffsetTextBox.Text);
float yScale = sanitizeTextToFloat(YScaleTextBox.Text);
float yOffset = sanitizeTextToFloat(YOffsetTextBox.Text);
foreach (Skeleton s in skeletonList)
{
var oscMessage = new OscElement("/skeletonPosition",
skeletonNum,
(float)(s.Position.X * SCREEN_WIDTH_PX_M_RATIO * xScale + xOffset),
(float)(s.Position.Y * SCREEN_HEIGHT_PX_M_RATIO * yScale + yOffset),
(float)(s.Position.Z * SCREEN_WIDTH_PX_M_RATIO * xScale + xOffset),
(int)(s.TrackingId));
oscBundle.AddElement(oscMessage);
skeletonNum++;
}
oscWriter.Send(oscBundle);
}
/// <summary>
/// Sends Osc on the global port
/// </summary>
/// <param name="x"></param>
/// <param name="y"></param>
private void SendBlobOsc(float x, float y, int blobId)
{
// send osc data
var elements = new List <OscElement>();
var address = "/blob";
elements.Add(new OscElement(address + "/id", blobId));
elements.Add(new OscElement(address + "/x", x));
elements.Add(new OscElement(address + "/y", y));
oscWriter.Send(new OscBundle(DateTime.Now, elements.ToArray()));
}
public override void Send(UdpWriter osc, StreamWriter fileWriter, int pointScale)
{
if (osc != null)
{
osc.Send(new OscElement(
"/face_animation",
sensorId, user,
c[AnimationUnit.LipRaiser],
c[AnimationUnit.LipStretcher],
c[AnimationUnit.LipCornerDepressor],
c[AnimationUnit.JawLower],
c[AnimationUnit.BrowLower],
c[AnimationUnit.BrowRaiser],
time));
}
if (fileWriter != null)
{
fileWriter.WriteLine("FaceAnimation," +
sensorId + "," + user + "," +
c[AnimationUnit.LipRaiser].ToString().Replace(",", ".") + "," +
c[AnimationUnit.LipStretcher].ToString().Replace(",", ".") + "," +
c[AnimationUnit.LipCornerDepressor].ToString().Replace(",", ".") + "," +
c[AnimationUnit.JawLower].ToString().Replace(",", ".") + "," +
c[AnimationUnit.BrowLower].ToString().Replace(",", ".") + "," +
c[AnimationUnit.BrowRaiser].ToString().Replace(",", ".") + "," +
time.ToString().Replace(",", "."));
}
}
/// <summary>
/// Sends a skeleton's position and depth via OSC
/// </summary>
/// <param name="skeleton">skeleton to send</param>
private void SendSkeletonOsc(Skeleton skeleton)
{
OscBundle b = new OscBundle();
b.AddElement(GetOscJointMessage(skeleton, JointType.AnkleLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.AnkleRight));
b.AddElement(GetOscJointMessage(skeleton, JointType.ElbowLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.ElbowRight));
b.AddElement(GetOscJointMessage(skeleton, JointType.FootLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.FootRight));
b.AddElement(GetOscJointMessage(skeleton, JointType.HandLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.HandRight));
b.AddElement(GetOscJointMessage(skeleton, JointType.Head));
b.AddElement(GetOscJointMessage(skeleton, JointType.HipCenter));
b.AddElement(GetOscJointMessage(skeleton, JointType.HipLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.HipRight));
b.AddElement(GetOscJointMessage(skeleton, JointType.KneeLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.KneeRight));
b.AddElement(GetOscJointMessage(skeleton, JointType.ShoulderCenter));
b.AddElement(GetOscJointMessage(skeleton, JointType.ShoulderLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.ShoulderRight));
b.AddElement(GetOscJointMessage(skeleton, JointType.Spine));
b.AddElement(GetOscJointMessage(skeleton, JointType.WristLeft));
b.AddElement(GetOscJointMessage(skeleton, JointType.WristRight));
OSCsend.Send(b);
}
public override void Send(int pointScale, UdpWriter osc, StreamWriter fileWriter)
{
if (osc != null)
{
osc.Send(new OscElement(
"/osceleton2/face_property",
sensorId, user,
happy, engaged, wearingGlasses, leftEyeClosed, rightEyeClosed, mouthOpen, mouthMoved, lookingAway,
time));
}
if (fileWriter != null)
{
fileWriter.WriteLine("FaceProperty," +
sensorId + "," + user + "," +
happy.ToString().Replace(",", ".") + "," +
engaged.ToString().Replace(",", ".") + "," +
wearingGlasses.ToString().Replace(",", ".") + "," +
leftEyeClosed.ToString().Replace(",", ".") + "," +
rightEyeClosed.ToString().Replace(",", ".") + "," +
mouthOpen.ToString().Replace(",", ".") + "," +
mouthMoved.ToString().Replace(",", ".") + "," +
lookingAway.ToString().Replace(",", ".") + "," +
time.ToString().Replace(",", "."));
}
}
/*
* private void sendOSCForearms(object sender, SkeletonFrameReadyEventArgs e) {
* // If there isn't a skeleton we want to send, we don't send anything
* if (prunedSkeletonList.Count == 0)
* return;
* int i = 0;
* foreach (Skeleton skel in prunedSkeletonList) {
*
* sendOneOSCForearm(skel.Joints[JointType.HandLeft], skel.Joints[JointType.ElbowLeft],i);
* i++;
* sendOneOSCForearm(skel.Joints[JointType.HandRight], skel.Joints[JointType.ElbowRight], i);
* i++;
* }
* }
*/
private void sendOneOSCForearm(Joint startJoint, Joint endJoint, int i)
{
var jointElement = new List <OscElement>();
jointElement.Add(new OscElement(
"/forearm",
i,
(float)Math.Round(startJoint.Position.X, 4),
(float)Math.Round(startJoint.Position.Y, 4),
(float)Math.Round(startJoint.Position.Z, 4),
(float)Math.Round(endJoint.Position.X, 4),
(float)Math.Round(endJoint.Position.Y, 4),
(float)Math.Round(endJoint.Position.Z, 4)
));
deltaToscWriter.Send(new OscBundle(DateTime.Now, jointElement.ToArray()));
if (showOscData)
{
string oscText = "\n\n/hand " + i + " " +
(float)Math.Round(startJoint.Position.X, 2) + " " +
(float)Math.Round(startJoint.Position.Y, 2) + " " +
(float)Math.Round(startJoint.Position.Z, 2) + "\n" +
(float)Math.Round(endJoint.Position.X, 2) + " " +
(float)Math.Round(endJoint.Position.Y, 2) + " " +
(float)Math.Round(endJoint.Position.Z, 2);
oscViewer.Text = oscText;
}
}
// Send out one skeleton data via OSC in Animata-friendly format
private void sendOneOSCAnimataSkeleton(Skeleton skel, int counter)
{
Console.WriteLine("Triggered");
double playerHeight = skeletonHeight(skel);
// joints bundled individually as 2 floats (x, y)
string oscText = "";
foreach (Joint joint in skel.Joints)
{
// string jointName = "s" + counter + joint.JointType.ToString();
string jointName = joint.JointType.ToString();
var jointElement = new List <OscElement>();
// Joint positions are returned in meters, so we'll assume a 2 meter tall person
// and scale that range to pixels for the animation
Point origin = new Point(-1, -1); // Offset in meters to map our origin to the characters
double playerHeightRatio = 2 / (playerHeight + .2);
// Translate Kinect joint positions to pixel coordinates
float xScale = sanitizeTextToFloat(XScaleTextBox.Text);
float xOffset = sanitizeTextToFloat(XOffsetTextBox.Text);
float yScale = sanitizeTextToFloat(YScaleTextBox.Text);
float yOffset = sanitizeTextToFloat(YOffsetTextBox.Text);
float jointX = joint.Position.X * SCREEN_WIDTH_PX_M_RATIO * xScale + xOffset;
float jointY = -1 * joint.Position.Y * SCREEN_HEIGHT_PX_M_RATIO * yScale + yOffset;
jointX = (float)Math.Round(jointX, 4);
jointY = (float)Math.Round(jointY, 4);
jointElement.Add(new OscElement(
"/joint", jointName,
(float)Math.Round(jointX, 4), (float)Math.Round(jointY, 4)));
oscWriter.Send(new OscBundle(DateTime.Now, jointElement.ToArray()));
oscWriter2.Send(new OscBundle(DateTime.Now, jointElement.ToArray()));
if (showOscData)
{
oscText += jointName + " " + jointX + " " + jointY + "\n"; //GenerateOscDataDump(counter, jointName, joint.Position
}
}
if (showOscData)
{
oscViewer.Text = oscText;
}
return;
}
public void record()
{
if (!isRecd)
{
Console.WriteLine("Recording");
OscElement o = new OscElement("/Looper/0/State", "Record");
oscLoop.Send(o);
isRecd = true;
isOvdb = false;
isPlay = false;
isStop = false;
}
else
{
return;
}
//Console.WriteLine("msg sent maybe??");
}
public void Send(int channel, float peakToPeak)
{
OscElement msg = new OscElement(
"/PeakToPeak",
(int)(DateTime.Now - new DateTime(1970, 1, 1)).TotalMilliseconds,
channel,
peakToPeak
);
writer.Send(msg);
}
void SendJointMessage(int joint, double x, double y, double z, double confidence, double time, int pointScale, UdpWriter osc, StreamWriter fileWriter)
{
if (osc != null)
{
osc.Send(new OscElement("/osceleton2/joint", oscMapping[joint], sensorId, user, (float)(x * pointScale), (float)(-y * pointScale), (float)(z * pointScale), (float)confidence, time));
}
if (fileWriter != null)
{
// Joint, user, joint, x, y, z, confidence, time
fileWriter.WriteLine("Joint," + sensorId + "," + user + "," + joint + "," +
(x * pointScale).ToString().Replace(",", ".") + "," +
(-y * pointScale).ToString().Replace(",", ".") + "," +
(z * pointScale).ToString().Replace(",", ".") + "," +
confidence.ToString().Replace(",", ".") + "," +
time.ToString().Replace(",", "."));
}
}
public override void Send(int pointScale, UdpWriter osc, StreamWriter fileWriter)
{
if (osc != null)
{
osc.Send(new OscElement(
"/osceleton2/face_rotation",
sensorId, user,
pitch, yaw, roll,
time));
}
if (fileWriter != null)
{
fileWriter.WriteLine("FaceRotation," +
sensorId + "," + user + "," +
pitch + "," + yaw + "," + roll + "," +
time.ToString().Replace(",", "."));
}
}
void sendToOsc(string line, String channel, UdpWriter oscWriter)
{
String[] splitted = line.Split(' ');
OscBundle b = new OscBundle(0);
int i = 0;
//Remove the Label
i++;
b.AddElement(new OscElement("/" + channel + "/timestamp", splitted[i++]));
// splitted.Length - 1 because the linebreak creates one more blank string when splitting
for (i = 2; i < splitted.Length - 1; i += 3)
{
b.AddElement(new OscElement("/kinect" + ((i + 1) / 3).ToString(),
splitted[i],
splitted[i + 1],
splitted[i + 2])
);
}
oscWriter.Send(b);
}
void SendHandStateMessage(int hand, double x, double y, double z, double confidence, int state, double stateConfidence, double time, int pointScale, UdpWriter osc, StreamWriter fileWriter)
{
if (osc != null)
{
osc.Send(new OscElement("/osceleton2/hand", sensorId, user, hand,
(float)(x * pointScale), (float)(-y * pointScale), (float)(z * pointScale), (float)confidence,
state, (float)stateConfidence, time));
}
if (fileWriter != null)
{
// Hand, user, joint, x, y, z, confidence, state, stateConfidence, time
fileWriter.WriteLine("Hand," + sensorId + "," + user + "," + hand + "," +
(x * pointScale).ToString().Replace(",", ".") + "," +
(-y * pointScale).ToString().Replace(",", ".") + "," +
(z * pointScale).ToString().Replace(",", ".") + "," +
confidence.ToString().Replace(",", ".") + "," +
state + "," +
stateConfidence.ToString().Replace(",", ".") + "," +
time.ToString().Replace(",", "."));
}
}
private void SendHandLocation(Skeleton skeleton)
{
Joint right = skeleton.Joints[JointType.HandRight];
Joint left = skeleton.Joints[JointType.HandLeft];
OscBundle msg = new OscBundle(0,
new OscElement("/right/x", right.Position.X),
new OscElement("/right/y", right.Position.Y),
new OscElement("/right/z", right.Position.Z),
new OscElement("/left/x", left.Position.X),
new OscElement("/left/y", left.Position.Y),
new OscElement("/left/z", left.Position.Z));
if (this.euclidDist(right.Position, left.Position) < 10 && right.Position.Y > 0.3)
{
msg.AddElement(new OscElement("/gestures/swell", 1.0f));
}
//OscBundle msg = new OscBundle(0, new OscElement("/gestures/swell", 1.0f));
udpwriter.Send(msg);
uw2.Send(msg);
}
public void sendOsc(string channel, FaceTrackFrame faceFrame, UdpWriter oscWriter)
{
TimeSpan t = DateTime.UtcNow - new DateTime(1970, 1, 1);
String time = ((long)t.TotalMilliseconds).ToString();
var shape = faceFrame.Get3DShape();
float[][] arr = shape
.Select(x => new float[3] {
x.X, x.Y, x.Z
})
.ToArray();
OscBundle b = new OscBundle(0);
int i = 0;
foreach (var v in shape)
{
var el = new OscElement("/kinect" + i++, v.X, v.Y, v.Z);
b.AddElement(el);
}
oscWriter.Send(b);
}
void kinect_AllFrameReady(KinectUpdateFrameData e)
{
try {
TextMessage.Text = "";
if (!e.IsAllUpdated)
{
return;
}
ImageColor.Source = e.ColorFrame.ToBitmapSource();
var skeletons = e.SkeletonFrame.GetTrackedSkeleton();
if (skeletons.Count() != 2)
{
return;
}
var player1 = skeletons.First().Joints[JointType.Head];
var player2 = skeletons.Last().Joints[JointType.Head];
if (!player1.IsTrackingOrInferred() || !player2.IsTrackingOrInferred())
{
return;
}
var distance = (int)(Math.Sqrt(Math.Pow(player2.Position.X - player1.Position.X, 2) +
Math.Pow(player2.Position.Y - player1.Position.Y, 2) +
Math.Pow(player2.Position.Z - player1.Position.Z, 2)) * 1000);
TextMessage.Text = distance.ToString();
udpWriter.Send(new OscBundle(0, new OscElement("/distance", distance)));
}
catch (Exception ex) {
Trace.WriteLine(ex.Message);
}
}
public override void Send(UdpWriter osc, StreamWriter fileWriter, int pointScale)
{
if (osc != null)
{
osc.Send(new OscElement(
"/face",
sensorId, user,
(float)(x * pointScale), (float)(-y * pointScale), (float)(z * pointScale),
rotationX, rotationY, rotationZ,
time));
}
if (fileWriter != null)
{
fileWriter.WriteLine("Face," +
sensorId + "," + user + "," +
(x * pointScale).ToString().Replace(",", ".") + "," +
(-y * pointScale).ToString().Replace(",", ".") + "," +
(z * pointScale).ToString().Replace(",", ".") + "," +
rotationX.ToString().Replace(",", ".") + "," +
rotationY.ToString().Replace(",", ".") + "," +
rotationZ.ToString().Replace(",", ".") + "," +
time.ToString().Replace(",", "."));
}
}
internal void SendSignalLeftToRight()
{
udpWriter.Send(new OscElement("/left"));
}
static void Main(string[] args)
{//UDP writer for OSC
UdpWriter udpWrite = new UdpWriter("127.0.0.1", 7000);
try
{
Nuitrack.Init("");
}
catch (Exception exception)
{
Console.WriteLine("Cannot initialize Nuitrack.");
throw exception;
}
try
{
// Create and setup all required modules
_depthSensor = DepthSensor.Create();
_userTracker = UserTracker.Create();
_skeletonTracker = SkeletonTracker.Create();
}
catch (Exception exception)
{
Console.WriteLine("Cannot create Nuitrack module.");
Console.WriteLine(exception.Message);
throw exception;
}
// Add event handlers for all modules
_depthSensor.OnUpdateEvent += OnDepthSensorUpdate;
_userTracker.OnNewUserEvent += OnUserTrackerNewUser;
_userTracker.OnLostUserEvent += OnUserTrackerLostUser;
_skeletonTracker.OnSkeletonUpdateEvent += OnSkeletonUpdate;
// Add an event handler for the IssueUpdate event
Nuitrack.onIssueUpdateEvent += OnIssueDataUpdate;
try
{
Nuitrack.Run();
Console.WriteLine(DateTime.Now.ToString());
}
catch (Exception exception)
{
Console.WriteLine("Cannot start Nuitrack.");
throw exception;
}
bool a = true;
while (a)
{
int start = (int)DateTime.Now.TimeOfDay.TotalMilliseconds;
// Update Nuitrack data. Data will be synchronized with skeleton time stamps.
try
{
Nuitrack.Update(_skeletonTracker);
}
catch (LicenseNotAcquiredException exception)
{
Console.WriteLine(DateTime.Now.ToString());
Console.WriteLine("LicenseNotAcquired exception. Exception: {0}", exception);
throw exception;
}
catch (Exception exception)
{
Console.WriteLine("Nuitrack update failed. Exception: ", exception);
}
if (_skeletonData != null)
{
//Create new bundle for each time skeleton data is refreshed
OscBundle bundle = new OscBundle();
//const int jointSize = 10;
foreach (var skeleton in _skeletonData.Skeletons)
{
foreach (var joint in skeleton.Joints)
{
float[] rotationMatrix = joint.Orient.Matrix;
//Ignore joints that are not currently used by Nuitrack
if (joint.Type == JointType.None || joint.Type == JointType.LeftFingertip || joint.Type == JointType.RightFingertip || joint.Type == JointType.LeftFoot || joint.Type == JointType.RightFoot)
{
continue;
}
//Create new message element for joint containing joint type and rotation matrix
OscElement jointMessage = new OscElement("/" + joint.Type, joint.Real.X, joint.Real.Y, joint.Real.Z, rotationMatrix[0], rotationMatrix[1], -1 * rotationMatrix[2], rotationMatrix[3], rotationMatrix[4], -1 * rotationMatrix[5], -1 * rotationMatrix[6], -1 * rotationMatrix[7], rotationMatrix[8]);
Console.WriteLine(joint.Real.X + " " + joint.Real.Y + " " + joint.Real.Z);
bundle.AddElement(jointMessage);
}
//Send the message bundle with the data
udpWrite.Send(bundle);
int difference = delay - start - (int)DateTime.Now.TimeOfDay.TotalMilliseconds;
System.Threading.Thread.Sleep(delay);
}
}
}
Nuitrack.Release();
Console.ReadLine();
}
private void sendOSCHeadOnly(object sender, SkeletonFrameReadyEventArgs e)
{
// If there isn't a skeleton we want to send, we don't send anything
if (leadSkeletonIDs.Count == 0)
{
return;
}
Skeleton headTrackedSkeleton = new Skeleton();
headTrackedSkeleton = null;
// Get the skeleton we want to send
foreach (Skeleton s in masterSkeletonList)
{
if (s.TrackingId == leadSkeletonIDs[0])
{
headTrackedSkeleton = s;
}
}
// Just in case, if we didn't find a skeleton, get out of here
if (headTrackedSkeleton == null)
{
Console.WriteLine("Had skeletons in our leadSkeletonIDs list, but couldn't find the appropriate skeleton... How odd.");
return;
}
double playerHeight = skeletonHeight(headTrackedSkeleton);
// joints bundled individually as 2 floats (x, y)
Joint headJoint = headTrackedSkeleton.Joints[JointType.Head];
var jointElement = new List <OscElement>();
// Joint positions are returned in meters, so we'll assume a 2 meter tall person
// and scale that range to pixels for the animation
var jointX = headJoint.Position.X;
var jointY = headJoint.Position.Y;
var jointZ = headJoint.Position.Z;
jointElement.Add(new OscElement(
"/head",
(float)Math.Round(jointX, 4), (float)Math.Round(jointY, 4),
(float)Math.Round(jointZ, 4)));
oscWriter.Send(new OscBundle(DateTime.Now, jointElement.ToArray()));
oscWriter2.Send(new OscBundle(DateTime.Now, jointElement.ToArray()));
deltaToscWriter.Send(new OscBundle(DateTime.Now, jointElement.ToArray()));
if (showOscData)
{
string oscText = "\n\n/head " +
(float)Math.Round(jointX, 2) + " " +
(float)Math.Round(jointY, 2) + " " +
(float)Math.Round(jointZ, 2);
oscViewer.Text = oscText;
}
// If there isn't a skeleton we want to send, we don't send anything
if (prunedSkeletonList.Count == 0)
{
return;
}
int i = 0;
foreach (Skeleton skel in prunedSkeletonList)
{
if (skel.TrackingId != headTrackedSkeleton.TrackingId)
{
sendOneOSCHand(skel.Joints[JointType.HandLeft], i);
i++;
sendOneOSCHand(skel.Joints[JointType.HandRight], i);
i++;
}
}
}
//Methods
public void sendOscMessage(OscElement messageElement)
{
bundle.AddElement(messageElement);
writer.Send(bundle);
}