public void DrawStaticGesture(SGInstance gesture, bool showArms = true)
{
foreach (var hand in gesture.Hands)
{
DrawHand(hand, showArms);
}
}
public void SaveNewStaticGestureInstance(int classId, SGInstance instance) // Whenever this is called it will be a new instance.
{
string serializedInstance = JsonConvert.SerializeObject(instance);
string sql = String.Format("INSERT INTO StaticGestureInstances (class_id, json) VALUES ('{0}', '{1}')", classId, serializedInstance);
executeNonQuery(sql);
}
public SGInstanceWrapper(Frame frame)
{
Gesture = new SGInstance(frame);
Id = -1;
ClassId = -1;
InstanceName = "new instance";
}
public SGInstanceWrapper(SGInstance instance)
{
Gesture = instance;
Id = -1;
ClassId = -1;
InstanceName = "new instance";
}
public SGInstance DeepCopy()
{
string serialized = JsonConvert.SerializeObject(this);
SGInstance deserialized = JsonConvert.DeserializeObject <SGInstance>(serialized);
//deserialized.UpdateFeatureVector();
return(deserialized);
}
public DGInstanceSample(SGInstance sgInstance)
{
Hands = sgInstance.Hands;
LeftHand = sgInstance.LeftHand;
RightHand = sgInstance.RightHand;
HandConfiguration = sgInstance.HandConfiguration;
FeatureVector = sgInstance.FeatureVector; // Should probably have new feature vector here that is a superset of DG feature vector.
Features = sgInstance.Features;
}
public Dictionary <SGClassWrapper, float> GetDistancesFromAllClasses(SGInstance gestureInstance)
{
lock (_staticGestureClasses)
{
var gestureDistances = new Dictionary <SGClassWrapper, float>();
foreach (var gestureClass in _staticGestureClasses)
{
gestureDistances.Add(gestureClass, gestureClass.Gesture.DistanceTo(gestureInstance));
}
return(gestureDistances);
}
}
//TODO: Check if this works (with all the casting and stuff).
public DGInstanceSample Lerp(DGInstanceSample otherInstance, float amount)
{
SGInstance sgLerp = base.Lerp(otherInstance, amount);
var lerpedSample = new DGInstanceSample(sgLerp);
if (LeftHand != null)
{
lerpedSample.LeftPalmVelocity = LeftPalmVelocity.Lerp(otherInstance.LeftPalmVelocity, amount);
}
if (RightHand != null)
{
lerpedSample.RightPalmVelocity = RightPalmVelocity.Lerp(otherInstance.RightPalmVelocity, amount);
}
return(lerpedSample);
}
private bool handsAreStill(Frame frame)
{
if (_stillGesture == null)
{
return(false);
}
var liveStaticGesture = new SGInstance(frame);
DebugMessage = String.Format("State: {0}\nDistance to still gesture: {1}\n", State, liveStaticGesture.DistanceTo(_stillGesture));
float leftHandVelocityMagnitude = -1;
float rightHandVelocityMagnitude = -1;
foreach (var hand in frame.Hands)
{
if (hand.IsLeft)
{
leftHandVelocityMagnitude = hand.PalmVelocity.Magnitude;
}
else
{
rightHandVelocityMagnitude = hand.PalmVelocity.Magnitude;
}
}
DebugMessage += String.Format("Left hand velocity magnitude: {0}\nRight hand velocity magnitude: {1}", leftHandVelocityMagnitude, rightHandVelocityMagnitude);
float stillDistance = (frame.Hands.Count == 2) ? _twoHandStillDistance : _stillDistance;
if (palmsAreMoving(frame) || liveStaticGesture.DistanceTo(_stillGesture) > stillDistance)
{
_stillGesture = liveStaticGesture;
_stillFramesCount = 0;
return(false);
}
return(++_stillFramesCount / (float)_frameRate >= _stillSeconds);
}
public float DistanceTo(SGInstance gestureInstance)
{
// Check if same hand configuration (BothHands, LeftHandOnly, RightHandOnly)...
if (HandConfiguration != gestureInstance.HandConfiguration)
{
return(Single.PositiveInfinity);
}
float distance = 0;
int featureCount = 0; // Necessary to do this because some features (like Dictionary's) are really 5 features.
int featureWeight;
foreach (var feature in gestureInstance.FeatureVector)
{
if (featuresToSkip.Contains(feature.Name))
{
continue;
}
if (feature.Value is Vec3)
{
featureWeight = FeatureWeights[feature.Name.ToString()];
Vec3 mean = ((Newtonsoft.Json.Linq.JObject)MeanValues[feature.Name]).ToObject <Vec3>();
distance += featureWeight * ((Vec3)(feature.Value)).DistanceTo(mean) / ((float)(double)StdDevValues[feature.Name]);
featureCount += featureWeight;
}
else if (feature.Value is float)
{
featureWeight = FeatureWeights[feature.Name.ToString()];
distance += featureWeight * Math.Abs(((float)feature.Value - (float)(double)MeanValues[feature.Name]) / ((float)(double)StdDevValues[feature.Name]));
featureCount += featureWeight;
}
else if (feature.Value is Dictionary <Finger.FingerType, Vec3> )
{
// This is just for fingersTipPositions
foreach (var fingerTipPosition in (Dictionary <Finger.FingerType, Vec3>)feature.Value)
{
Finger.FingerType fingerType = fingerTipPosition.Key;
featureWeight = FeatureWeights[feature.Name.ToString() + fingerType];
var meanFingerTipPositions = ((Newtonsoft.Json.Linq.JObject)MeanValues[feature.Name]).ToObject <Dictionary <Finger.FingerType, Vec3> >();
var stdDevFingerTipPositions = ((Newtonsoft.Json.Linq.JObject)StdDevValues[feature.Name]).ToObject <Dictionary <Finger.FingerType, float> >();
//var stdDevFingerTipPositions = (Dictionary<Finger.FingerType, float>)StdDevValues[feature.Name];
Vec3 instancePos = fingerTipPosition.Value;
distance += featureWeight * (instancePos).DistanceTo(meanFingerTipPositions[fingerType]) / stdDevFingerTipPositions[fingerType];
featureCount += featureWeight;
}
}
else if (feature.Value is Dictionary <Finger.FingerType, bool> )
{
// This is just for fingers.IsExtended
foreach (var fingerExtended in (Dictionary <Finger.FingerType, bool>)feature.Value)
{
Finger.FingerType fingerType = fingerExtended.Key;
featureWeight = FeatureWeights[feature.Name.ToString() + fingerType];
bool isExtended = fingerExtended.Value;
float fingerExtendedPercentage = (((Newtonsoft.Json.Linq.JObject)MeanValues[feature.Name]).ToObject <Dictionary <Finger.FingerType, float> >())[fingerType];
if (fingerExtendedPercentage >= 0.5f) // finger should be extended
{
if (!isExtended)
{
distance += featureWeight; //distance += fingerExtendedPercentage;
}
}
else
{
if (isExtended)
{
distance += featureWeight; //distance += 1 - fingerExtendedPercentage;
}
}
featureCount += featureWeight;
}
}
}
//return distance / (gestureInstance.FeatureVector.Count - featuresToSkip.Count);
return(distance / featureCount);
}
public SGInstance Lerp(SGInstance otherInstance, float amount)
{
if (this.HandConfiguration != otherInstance.HandConfiguration)
{
return(this); // Maybe throw an exception or handle another way.
}
var lerpedSGI = this.DeepCopy(); // Just to avoid any potential null errors. Make first hand value the default.
//var lerpedSGI = new StaticGestureInstance();
lerpedSGI.Hands.Clear();
#region Left Hand
if (LeftHand != null)
{
lerpedSGI.LeftHand.ArmX = LeftHand.ArmX.Lerp(otherInstance.LeftHand.ArmX, amount);
lerpedSGI.LeftHand.ArmY = LeftHand.ArmY.Lerp(otherInstance.LeftHand.ArmY, amount);
lerpedSGI.LeftHand.ArmZ = LeftHand.ArmZ.Lerp(otherInstance.LeftHand.ArmZ, amount);
lerpedSGI.LeftHand.ElbowPos_Relative = LeftHand.ElbowPos_Relative.Lerp(otherInstance.LeftHand.ElbowPos_Relative, amount);
lerpedSGI.LeftHand.ElbowPos_World = LeftHand.ElbowPos_World.Lerp(otherInstance.LeftHand.ElbowPos_World, amount);
foreach (var fingerType in (Finger.FingerType[])Enum.GetValues(typeof(Finger.FingerType)))
{
foreach (var jointType in (Finger.FingerJoint[])Enum.GetValues(typeof(Finger.FingerJoint)))
{
var fjp_rel1 = LeftHand.FingerJointPositions_Relative[fingerType][jointType];
var fjp_rel2 = otherInstance.LeftHand.FingerJointPositions_Relative[fingerType][jointType];
lerpedSGI.LeftHand.FingerJointPositions_Relative[fingerType][jointType] = fjp_rel1.Lerp(fjp_rel2, amount);
var fjp_world1 = LeftHand.FingerJointPositions_World[fingerType][jointType];
var fjp_world2 = otherInstance.LeftHand.FingerJointPositions_World[fingerType][jointType];
lerpedSGI.LeftHand.FingerJointPositions_World[fingerType][jointType] = fjp_world1.Lerp(fjp_world2, amount);
}
lerpedSGI.LeftHand.FingerTipPositions[fingerType] = LeftHand.FingerTipPositions[fingerType].Lerp(otherInstance.LeftHand.FingerTipPositions[fingerType], amount);
}
lerpedSGI.LeftHand.ForearmCenter_Relative = LeftHand.ForearmCenter_Relative.Lerp(otherInstance.LeftHand.ForearmCenter_Relative, amount);
lerpedSGI.LeftHand.ForearmCenter_World = LeftHand.ForearmCenter_World.Lerp(otherInstance.LeftHand.ForearmCenter_World, amount);
lerpedSGI.LeftHand.HandDirection = LeftHand.HandDirection.Lerp(otherInstance.LeftHand.HandDirection, amount);
// Do I need to worry about HandTransform? Hopefully not.
lerpedSGI.LeftHand.IndexBasePos_Relative = LeftHand.IndexBasePos_Relative.Lerp(otherInstance.LeftHand.IndexBasePos_Relative, amount);
lerpedSGI.LeftHand.MiddleBasePos_Relative = LeftHand.MiddleBasePos_Relative.Lerp(otherInstance.LeftHand.MiddleBasePos_Relative, amount);
lerpedSGI.LeftHand.RingBasePos_Relative = LeftHand.RingBasePos_Relative.Lerp(otherInstance.LeftHand.RingBasePos_Relative, amount);
lerpedSGI.LeftHand.PinkyBasePos_Relative = LeftHand.PinkyBasePos_Relative.Lerp(otherInstance.LeftHand.PinkyBasePos_Relative, amount);
lerpedSGI.LeftHand.IndexBasePos_World = LeftHand.IndexBasePos_World.Lerp(otherInstance.LeftHand.IndexBasePos_World, amount);
lerpedSGI.LeftHand.MiddleBasePos_World = LeftHand.MiddleBasePos_World.Lerp(otherInstance.LeftHand.MiddleBasePos_World, amount);
lerpedSGI.LeftHand.RingBasePos_World = LeftHand.RingBasePos_World.Lerp(otherInstance.LeftHand.RingBasePos_World, amount);
lerpedSGI.LeftHand.PinkyBasePos_World = LeftHand.PinkyBasePos_World.Lerp(otherInstance.LeftHand.PinkyBasePos_World, amount);
lerpedSGI.LeftHand.PalmNormal = LeftHand.PalmNormal.Lerp(otherInstance.LeftHand.PalmNormal, amount);
lerpedSGI.LeftHand.PalmPosition = LeftHand.PalmPosition.Lerp(otherInstance.LeftHand.PalmPosition, amount);
lerpedSGI.LeftHand.PalmSphereCenter = LeftHand.PalmSphereCenter.Lerp(otherInstance.LeftHand.PalmSphereCenter, amount);
lerpedSGI.LeftHand.PalmSphereRadius = HelperMethods.Lerp(LeftHand.PalmSphereRadius, otherInstance.LeftHand.PalmSphereRadius, amount);
lerpedSGI.LeftHand.Pitch = HelperMethods.Lerp(LeftHand.Pitch, otherInstance.LeftHand.Pitch, amount);
lerpedSGI.LeftHand.Roll = HelperMethods.Lerp(LeftHand.Roll, otherInstance.LeftHand.Roll, amount);
lerpedSGI.LeftHand.Yaw = HelperMethods.Lerp(LeftHand.Yaw, otherInstance.LeftHand.Yaw, amount);
lerpedSGI.LeftHand.WristPos_Relative = LeftHand.WristPos_Relative.Lerp(otherInstance.LeftHand.WristPos_Relative, amount);
lerpedSGI.LeftHand.WristPos_World = LeftHand.WristPos_World.Lerp(otherInstance.LeftHand.WristPos_World, amount);
lerpedSGI.Hands.Add(lerpedSGI.LeftHand);
}
#endregion
#region Right Hand
if (RightHand != null)
{
lerpedSGI.RightHand.ArmX = RightHand.ArmX.Lerp(otherInstance.RightHand.ArmX, amount);
lerpedSGI.RightHand.ArmY = RightHand.ArmY.Lerp(otherInstance.RightHand.ArmY, amount);
lerpedSGI.RightHand.ArmZ = RightHand.ArmZ.Lerp(otherInstance.RightHand.ArmZ, amount);
lerpedSGI.RightHand.ElbowPos_Relative = RightHand.ElbowPos_Relative.Lerp(otherInstance.RightHand.ElbowPos_Relative, amount);
lerpedSGI.RightHand.ElbowPos_World = RightHand.ElbowPos_World.Lerp(otherInstance.RightHand.ElbowPos_World, amount);
foreach (var fingerType in (Finger.FingerType[])Enum.GetValues(typeof(Finger.FingerType)))
{
foreach (var jointType in (Finger.FingerJoint[])Enum.GetValues(typeof(Finger.FingerJoint)))
{
var fjp_rel1 = RightHand.FingerJointPositions_Relative[fingerType][jointType];
var fjp_rel2 = otherInstance.RightHand.FingerJointPositions_Relative[fingerType][jointType];
lerpedSGI.RightHand.FingerJointPositions_Relative[fingerType][jointType] = fjp_rel1.Lerp(fjp_rel2, amount);
var fjp_world1 = RightHand.FingerJointPositions_World[fingerType][jointType];
var fjp_world2 = otherInstance.RightHand.FingerJointPositions_World[fingerType][jointType];
lerpedSGI.RightHand.FingerJointPositions_World[fingerType][jointType] = fjp_world1.Lerp(fjp_world2, amount);
}
lerpedSGI.RightHand.FingerTipPositions[fingerType] = RightHand.FingerTipPositions[fingerType].Lerp(otherInstance.RightHand.FingerTipPositions[fingerType], amount);
}
lerpedSGI.RightHand.ForearmCenter_Relative = RightHand.ForearmCenter_Relative.Lerp(otherInstance.RightHand.ForearmCenter_Relative, amount);
lerpedSGI.RightHand.ForearmCenter_World = RightHand.ForearmCenter_World.Lerp(otherInstance.RightHand.ForearmCenter_World, amount);
lerpedSGI.RightHand.HandDirection = RightHand.HandDirection.Lerp(otherInstance.RightHand.HandDirection, amount);
// Do I need to worry about HandTransform? Hopefully not.
lerpedSGI.RightHand.IndexBasePos_Relative = RightHand.IndexBasePos_Relative.Lerp(otherInstance.RightHand.IndexBasePos_Relative, amount);
lerpedSGI.RightHand.MiddleBasePos_Relative = RightHand.MiddleBasePos_Relative.Lerp(otherInstance.RightHand.MiddleBasePos_Relative, amount);
lerpedSGI.RightHand.RingBasePos_Relative = RightHand.RingBasePos_Relative.Lerp(otherInstance.RightHand.RingBasePos_Relative, amount);
lerpedSGI.RightHand.PinkyBasePos_Relative = RightHand.PinkyBasePos_Relative.Lerp(otherInstance.RightHand.PinkyBasePos_Relative, amount);
lerpedSGI.RightHand.IndexBasePos_World = RightHand.IndexBasePos_World.Lerp(otherInstance.RightHand.IndexBasePos_World, amount);
lerpedSGI.RightHand.MiddleBasePos_World = RightHand.MiddleBasePos_World.Lerp(otherInstance.RightHand.MiddleBasePos_World, amount);
lerpedSGI.RightHand.RingBasePos_World = RightHand.RingBasePos_World.Lerp(otherInstance.RightHand.RingBasePos_World, amount);
lerpedSGI.RightHand.PinkyBasePos_World = RightHand.PinkyBasePos_World.Lerp(otherInstance.RightHand.PinkyBasePos_World, amount);
lerpedSGI.RightHand.PalmNormal = RightHand.PalmNormal.Lerp(otherInstance.RightHand.PalmNormal, amount);
lerpedSGI.RightHand.PalmPosition = RightHand.PalmPosition.Lerp(otherInstance.RightHand.PalmPosition, amount);
lerpedSGI.RightHand.PalmSphereCenter = RightHand.PalmSphereCenter.Lerp(otherInstance.RightHand.PalmSphereCenter, amount);
lerpedSGI.RightHand.PalmSphereRadius = HelperMethods.Lerp(RightHand.PalmSphereRadius, otherInstance.RightHand.PalmSphereRadius, amount);
lerpedSGI.RightHand.Pitch = HelperMethods.Lerp(RightHand.Pitch, otherInstance.RightHand.Pitch, amount);
lerpedSGI.RightHand.Roll = HelperMethods.Lerp(RightHand.Roll, otherInstance.RightHand.Roll, amount);
lerpedSGI.RightHand.Yaw = HelperMethods.Lerp(RightHand.Yaw, otherInstance.RightHand.Yaw, amount);
lerpedSGI.RightHand.WristPos_Relative = RightHand.WristPos_Relative.Lerp(otherInstance.RightHand.WristPos_Relative, amount);
lerpedSGI.RightHand.WristPos_World = RightHand.WristPos_World.Lerp(otherInstance.RightHand.WristPos_World, amount);
lerpedSGI.Hands.Add(lerpedSGI.RightHand);
}
#endregion
lerpedSGI.UpdateFeatureVector();
return(lerpedSGI);
}
public float DistanceTo(SGInstance otherInstance)
{
try
{
// Check if same hand configuration (BothHands, LeftHandOnly, RightHandOnly)...
if (HandConfiguration != otherInstance.HandConfiguration)
{
return(Single.PositiveInfinity);
}
float distance = 0;
int featureCount = 0; // Necessary to do this because some features (like Dictionary's) are really 5 features.
foreach (var feature in otherInstance.FeatureVector)
{
if (featuresToSkip.Contains(feature.Name))
{
continue;
}
if (feature.Value is Vec3)
{
distance += feature.Weight * ((Vec3)(feature.Value)).DistanceTo((Vec3)Features[feature.Name].Value);
featureCount += feature.Weight;
}
else if (feature.Value is float)
{
// float values include Yaw, Pitch, Roll, and Sphere Radius
distance += feature.Weight * Math.Abs((float)feature.Value - (float)Features[feature.Name].Value);
featureCount += feature.Weight;
}
else if (feature.Value is Dictionary <Finger.FingerType, Vec3> )
{
// This is just for fingersTipPositions
Dictionary <Finger.FingerType, Vec3> thisHandFingersTipPositions = (Dictionary <Finger.FingerType, Vec3>)(Features[feature.Name].Value);
foreach (var fingerTipPosition in (Dictionary <Finger.FingerType, Vec3>)feature.Value)
{
//var theFingerTipPositions = ((Newtonsoft.Json.Linq.JObject)Features[feature.Name].Value).ToObject<Dictionary<Finger.FingerType, Vec3>>();
Finger.FingerType fingerType = fingerTipPosition.Key;
Vec3 instancePos = fingerTipPosition.Value;
distance += feature.Weight * (instancePos).DistanceTo(thisHandFingersTipPositions[fingerType]);
featureCount += feature.Weight;
}
}
else if (feature.Value is Dictionary <Finger.FingerType, bool> )
{
// This is just for fingers.IsExtended
Dictionary <Finger.FingerType, bool> thisHandFingersExtended = (Dictionary <Finger.FingerType, bool>)(Features[feature.Name].Value);
foreach (var fingerExtended in (Dictionary <Finger.FingerType, bool>)feature.Value)
{
Finger.FingerType fingerType = fingerExtended.Key;
bool isExtended = fingerExtended.Value;
if (thisHandFingersExtended[fingerType]) // finger should be extended
{
if (!isExtended)
{
distance += feature.Weight; //distance += fingerExtendedPercentage;
}
}
else
{
if (isExtended)
{
distance += feature.Weight; //distance += 1 - fingerExtendedPercentage;
}
}
featureCount += feature.Weight;
}
}
}
//return distance / (gestureInstance.FeatureVector.Count - featuresToSkip.Count);
return(distance / featureCount);
}
catch (Exception ex) { return(-555); }
}
public void ProcessFrame(Frame frame)
{
if (_lastFrame == null)
{
_lastFrame = frame;
return;
}
bool handsStill = handsAreStill(frame);
if (frame.Hands.Count == 0)
{
State = DGRecorderState.WaitingForHands;
}
switch (State)
{
case DGRecorderState.WaitingForHands:
if (frame.Hands.Count > 0)
{
_stillGesture = new SGInstance(frame);
State = DGRecorderState.WaitingToStart;
}
break;
case DGRecorderState.WaitingToStart:
if (handsStill)
{
_startOfGesture = new DGInstanceSample(frame);
_gestureSamples = new List <DGInstanceSample>();
_gestureSamples.Add(_startOfGesture);
State = DGRecorderState.InStartPosition;
}
break;
case DGRecorderState.InStartPosition:
if (!handsStill)
{
State = DGRecorderState.RecordingGesture;
}
break;
case DGRecorderState.RecordingGesture:
if (handsStill)
{
// Trim the extra samples in back (from holding hand still for X seconds)
int stillFrames = (int)(_frameRate * _stillSeconds);
_gestureSamples.RemoveRange(_gestureSamples.Count - stillFrames, stillFrames);
MostRecentInstance = new DGInstance(_gestureSamples);
if (_inRecordMode)
{
Instances.Add(MostRecentInstance);
}
State = DGRecorderState.RecordingJustFinished; // Put this first so "InEndPosition" is printed while we process the frames
}
else
{
_gestureSamples.Add(new DGInstanceSample(frame));
}
break;
case DGRecorderState.RecordingJustFinished:
State = DGRecorderState.InEndPosition;
break;
case DGRecorderState.InEndPosition:
if (!handsStill)
{
State = DGRecorderState.WaitingToStart;
}
break;
}
}
