public override TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Microsoft.Kinect.Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity)
{
double columnData = cursor.X;
double rowData = cursor.Y;
double constraintData = cursor.Z;
if (cursor.IsFrozen)
{
columnData = cursor.FrozenX;
rowData = cursor.FrozenY;
constraintData = cursor.FrozenZ;
}
if (columnData < 0 || rowData < 0)
return new TypingStatus();
double stepX = 1.0 / layoutCols;
double stepY = 1.0 / layoutRows;
int col = (int)(columnData / stepX);
int row = (int)(rowData / stepY);
TypingStatus status = new TypingStatus();
status.HighlightedKey = layout[row * layoutCols + col];
if (gestures != null && gestures.Count > 0 && gestures.ElementAt(0).Type == GestureType.Tap)
{
TypingGesture gesture = gestures.ElementAt(0);
col = (int)(gesture.Position.X / stepX);
row = (int)(gesture.Position.Y / stepY);
status.SelectedKey = layout[row * layoutCols + col];
}
return status;
}
public override TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Microsoft.Kinect.Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity)
{
double columnData = cursor.X;
double rowData = cursor.Y;
double constraintData = cursor.Z;
if (columnData < 0 || rowData < 0)
return new TypingStatus();
double stepX = 1.0 / layoutCols;
double stepY = 1.0 / layoutRows;
int col = (int)(columnData / stepX);
int row = (int)(rowData / stepY);
currentPointingIndex = row * layoutCols + col;
TypingStatus status = new TypingStatus();
if (currentSequenceIndex == 36)
{
return status;
}
if (layout[currentPointingIndex] == sequence[currentSequenceIndex])
{
status.HighlightedKey = layout[currentPointingIndex];
if (currentSequenceIndex < sequence.Length)
{
currentSequenceIndex++;
if (!started)
started = true;
else
correctCount++;
}
}
else
{
if (currentPointingIndex != previousPointingIndex)
{
if (started)
{
errorCount++;
}
}
}
if (gestures != null && gestures.Count > 0 && gestures.ElementAt(0).Type == GestureType.Tap)
{
TypingGesture gesture = gestures.ElementAt(0);
col = (int)(gesture.Position.X / stepX);
row = (int)(gesture.Position.Y / stepY);
currentPointingIndex = row * layoutCols + col;
status.SelectedKey = layout[currentPointingIndex];
}
previousPointingIndex = currentPointingIndex;
return status;
}
private List<TypingGesture> ProcessGesturesPush(Point3D cursor)
{
if (cursor.X == -1 || cursor.Y == -1)
return null;
List<TypingGesture> gestures = new List<TypingGesture>();
InputInfo info = new InputInfo() { Position = cursor };
System.Windows.Media.Media3D.Vector3D displacement = cursor - lastPos;
lastPos = cursor;
if (movementSequenceSB.Length == movementSequence.Capacity)
{
movementSequenceSB.Remove(0, 1);
movementSequence.RemoveAt(0);
}
if (displacement.X > LATERAL_THRESHOLD_PUSH || displacement.Y > LATERAL_THRESHOLD_PUSH)
info.Movement = 'M'; //moving
else if (Math.Abs(displacement.Z) > DEPTH_THRESHOLD) //doing some movement on depth
{
if (displacement.Z < 0)
info.Movement = 'P'; //released
else
info.Movement = 'R'; //pressed
}
else
info.Movement = 'S'; //steady --- not moving
movementSequenceSB.Append(info.Movement);
movementSequence.Add(info);
String tmpSequence = MovementSequence;
MatchCollection taps = regexTapCrank.Matches(tmpSequence);
if (taps.Count > 0)
{
Match tap = taps[0];
gestures.Add(new TypingGesture()
{
Time = DateTime.Now,
Type = GestureType.Tap,
Position = movementSequence[tap.Index].Position
});
movementSequenceSB.Remove(tap.Index, tap.Length);
movementSequence.RemoveRange(tap.Index, tap.Length);
}
OnPropertyChanged("MovementSequence");
return gestures;
}
internal ICollection<TypingGesture> ProcessGestures(Skeleton skeleton, double deltaTimeMilliseconds, Point3D cursor, Point3D secondaryCursor,
SelectionMethod selectionM, Key highlightedKey, bool userClicked)
{
List<TypingGesture> gestures = null;
if (selectionM == SelectionMethod.Push)
gestures = ProcessGesturesPush(cursor);
else if (selectionM == SelectionMethod.Swipe)
gestures = ProcessGesturesSwipe(cursor);
else if (selectionM == SelectionMethod.Timer)
gestures = ProcessGesturesTimer(cursor, deltaTimeMilliseconds, highlightedKey);
else if (selectionM == SelectionMethod.SecondHand)
gestures = ProcessGesturesSecondHand(cursor, secondaryCursor, deltaTimeMilliseconds, highlightedKey);
else if (selectionM == SelectionMethod.Click)
gestures = ProcessGesturesClick(cursor, userClicked);
return gestures;
}
private void DoWork(Skeleton skeleton, double deltaMilliseconds)
{
Size layoutSize = new Size(35, 35);
if(planeSize == PlaneSize.p25x25)
layoutSize = new Size(25, 25);
//1- find the position of the cursor on the layout plane
CursorLocation = TypingMethod.FindCursorPosition(skeleton, anchoring, layoutSize, dexterity, selectionMethod, armStretch);
//2- find the second hand cursor
secCursorLocation = secTypingMethod.FindCursorPosition(skeleton, anchoring, layoutSize, dexterity, selectionMethod, armStretch);
//3- Looks for gestures [pressed, released]
ICollection<TypingGesture> gestures = Recognizer.ProcessGestures(skeleton, deltaMilliseconds, cursorLocation, secCursorLocation, selectionMethod, highlightedKey, HasUserClicked);
//4- Passes it on to the typing method itself
TypingStatus status = TypingMethod.ProcessNewFrame(CursorLocation, gestures, skeleton, deltaMilliseconds, dexterity);
HighlightedKey = status.HighlightedKey;
SelectedKey = status.SelectedKey;
}
public TypingEngine()
{
Methods = new List<TypingMethod>();
Methods.Add(new ABCTypingMethod());
Methods.Add(new QwertyTypingMethod());
Methods.Add(new SplitQwertyTypingMethod());
Methods.Add(new FitalyTypingMethod());
Methods.Add(new CircularTypingMethod());
Methods.Add(new TahnuTypingMethod());
Methods.Add(new LabyrinthTypingMethod());
Methods.Add(new SeatoTypingMethod());
Methods.Add(new BoxTypingMethod());
TypingMethod = Methods.Last();
secTypingMethod = new SecondHandSelectionTypingMethod();
Recognizer = new GestureRecognizer();
PropertyChanged += new PropertyChangedEventHandler(Recognizer.TypingEngine_PropertyChanged);
CursorLocation = new Point3D(0, 0, 0);
secCursorLocation = new Point3D(0, 0, 0);
planeSize = PlaneSize.p35x35;
SelectionMethod = ArmText.Typing.SelectionMethod.Click;
Dexterity = TypingDexterity.Right;
}
private List<TypingGesture> ProcessGesturesClick(Point3D cursor, bool userClicked)
{
if (cursor.X == -1 || cursor.Y == -1)
return null;
List<TypingGesture> gestures = new List<TypingGesture>();
if (!userClicked)
return gestures;
gestures.Add(new TypingGesture()
{
Position = cursor,
Type = GestureType.Tap,
Time = DateTime.Now
});
return gestures;
}
private List<TypingGesture> ProcessGesturesSecondHand(Point3D cursor, Point3D secondaryCursor, double deltaTimeMilliseconds, Key highlightedKey)
{
if (cursor.X == -1 || cursor.Y == -1)
return null;
List<TypingGesture> gestures = new List<TypingGesture>();
InputInfo info = new InputInfo() { Position = cursor };
System.Windows.Media.Media3D.Vector3D displacement = cursor - lastPos;
lastPos = cursor;
if (movementSequenceSB.Length == movementSequence.Capacity)
{
movementSequenceSB.Remove(0, 1);
movementSequence.RemoveAt(0);
}
if (displacement.X > LATERAL_THRESHOLD_TIMER || displacement.Y > LATERAL_THRESHOLD_TIMER)
info.Movement = 'M'; //moving
else
info.Movement = 'S'; //steady --- not moving
movementSequenceSB.Append(info.Movement);
movementSequence.Add(info);
if (timerState == ArmText.Typing.TimerState.Nothing)
{
String tmpSequence = MovementSequence;
MatchCollection starts = regexStartTimer.Matches(tmpSequence);
if (starts.Count > 0)
{
Match start = starts[0];
TimerState = ArmText.Typing.TimerState.Running;
secondHandSelectionInfo = movementSequence[start.Index];
movementSequenceSB.Remove(start.Index, start.Length);
movementSequence.RemoveRange(start.Index, start.Length);
}
}
else if (timerState == ArmText.Typing.TimerState.Running)
{
if (secondaryCursor.X > LATERAL_THRESHOLD_SECOND_HAND && DateTime.Now > lastSecondHandSelection + secondaryHandSelectionTimeDelta)
{
gestures.Add(new TypingGesture()
{
Time = DateTime.Now,
Type = GestureType.Tap,
Position = secondHandSelectionInfo.Position
});
lastSecondHandSelection = DateTime.Now;
TimerState = ArmText.Typing.TimerState.Nothing;
}
}
OnPropertyChanged("MovementSequence");
return gestures;
}
private List<TypingGesture> ProcessGesturesTimer(Point3D cursor, double delta, Key highlightedKey)
{
if (cursor.X == -1 || cursor.Y == -1)
return null;
List<TypingGesture> gestures = new List<TypingGesture>();
InputInfo info = new InputInfo() { Position = cursor };
System.Windows.Media.Media3D.Vector3D displacement = cursor - lastPos;
lastPos = cursor;
if (movementSequenceSB.Length == movementSequence.Capacity)
{
movementSequenceSB.Remove(0, 1);
movementSequence.RemoveAt(0);
}
if (Math.Abs(displacement.X) > LATERAL_THRESHOLD_TIMER || Math.Abs(displacement.Y) > LATERAL_THRESHOLD_TIMER)
info.Movement = 'M'; //moving
else
info.Movement = 'S'; //steady --- not moving
movementSequenceSB.Append(info.Movement);
movementSequence.Add(info);
String tmpSequence = MovementSequence;
if (timerState == ArmText.Typing.TimerState.Nothing)
{
MatchCollection starts = regexStartTimer.Matches(tmpSequence);
if (starts.Count > 0)
{
Match start = starts[0];
TimerState = ArmText.Typing.TimerState.Running;
movementSequenceSB.Remove(start.Index, start.Length);
movementSequence.RemoveRange(start.Index, start.Length);
}
}
else if (timerState == ArmText.Typing.TimerState.Running)
{
MatchCollection selections = regexTapTimer.Matches(tmpSequence);
if (selections.Count > 0)
{
Match selection = selections[0];
gestures.Add(new TypingGesture()
{
Time = DateTime.Now,
Type = GestureType.Tap,
Position = movementSequence[selection.Index].Position
});
TimerState = ArmText.Typing.TimerState.Nothing;
movementSequenceSB.Remove(selection.Index, selection.Length);
movementSequence.RemoveRange(selection.Index, selection.Length);
}
}
OnPropertyChanged("MovementSequence");
return gestures;
}
private List<TypingGesture> ProcessGesturesSwipe(Point3D cursor)
{
if (cursor.X == -1 || cursor.Y == -1)
return null;
List<TypingGesture> gestures = new List<TypingGesture>();
InputInfo info = new InputInfo() { Position = cursor };
lastPos = cursor;
if (movementSequenceSB.Length == movementSequence.Capacity)
{
movementSequenceSB.Remove(0, 1);
movementSequence.RemoveAt(0);
}
if (cursor.IsFrozen)
info.Movement = 'S'; //steady --- not moving
else
info.Movement = 'M'; //moving
movementSequenceSB.Append(info.Movement);
movementSequence.Add(info);
String tmpSequence = MovementSequence;
if (timerState == ArmText.Typing.TimerState.Nothing)
{
MatchCollection starts = regexStartSwipe.Matches(tmpSequence);
if (starts.Count > 0)
{
Match start = starts[0];
TimerState = ArmText.Typing.TimerState.Running;
secondHandSelectionInfo = movementSequence[start.Index];
movementSequenceSB.Remove(start.Index, start.Length);
movementSequence.RemoveRange(start.Index, start.Length);
}
}
else if (timerState == ArmText.Typing.TimerState.Running)
{
MatchCollection taps = regexTapSwipe.Matches(tmpSequence);
if (taps.Count > 0)
{
Match tap = taps[0];
gestures.Add(new TypingGesture()
{
Time = DateTime.Now,
Type = GestureType.Tap,
Position = movementSequence[tap.Index].Position
});
TimerState = ArmText.Typing.TimerState.Nothing;
movementSequenceSB.Remove(tap.Index, tap.Length);
movementSequence.RemoveRange(tap.Index, tap.Length);
}
}
OnPropertyChanged("MovementSequence");
return gestures;
}
private System.Windows.Media.Media3D.Vector3D CalculateCenterMass(Point3D shoulder, Point3D elbow, Point3D hand)
{
//upper arm center mass and fore arm center mass
upperCM.X = (elbow.X - shoulder.X) * UPPER_ARM_CENTER_GRAVITY_RATIO;
upperCM.Y = (elbow.Y - shoulder.Y) * UPPER_ARM_CENTER_GRAVITY_RATIO;
upperCM.Z = (elbow.Z - shoulder.Z) * UPPER_ARM_CENTER_GRAVITY_RATIO;
//lower arm
foreCM.X = (hand.X - elbow.X) * foreArmAndHandCenterOfGravityRatio + elbow.X;
foreCM.Y = (hand.Y - elbow.Y) * foreArmAndHandCenterOfGravityRatio + elbow.Y;
foreCM.Z = (hand.Z - elbow.Z) * foreArmAndHandCenterOfGravityRatio + elbow.Z;
//base on equation get whole arm center mass
armCM.X = (foreCM.X - upperCM.X) * (1 - upperArmWeightProportion) + upperCM.X;
armCM.Y = (foreCM.Y - upperCM.Y) * (1 - upperArmWeightProportion) + upperCM.Y;
armCM.Z = (foreCM.Z - upperCM.Z) * (1 - upperArmWeightProportion) + upperCM.Z;
var normalizingFactor = MALE_UPPERARM_LENGHT / ((elbow - shoulder).Length * 100);
if(maxForce == FEMALE_MAX_FORCE)
normalizingFactor = FEMALE_UPPERARM_LENGHT / ((elbow - shoulder).Length * 100);
var normalizedArmCM = armCM * normalizingFactor;
return normalizedArmCM;
}
private Dictionary<JointType, Point3D> GetArmPoints(Skeleton skeleton)
{
Dictionary<JointType, Point3D> tempJointPoints = new Dictionary<JointType, Point3D>();
//right side joint tracking
if (skeleton.Joints[Hand].TrackingState == JointTrackingState.Tracked &&
skeleton.Joints[Shoulder].TrackingState == JointTrackingState.Tracked &&
skeleton.Joints[Elbow].TrackingState == JointTrackingState.Tracked)
{
Joint rightHand = skeleton.Joints[Hand];
Joint rightWrist = skeleton.Joints[Wrist];
Point3D hand = new Point3D(
0.397 * (rightHand.Position.X - rightWrist.Position.X) + rightWrist.Position.X,
0.397 * (rightHand.Position.Y - rightWrist.Position.Y) + rightWrist.Position.Y,
0.397 * (rightHand.Position.Z - rightWrist.Position.Z) + rightWrist.Position.Z);
Point3D elbow = new Point3D(
skeleton.Joints[Elbow].Position.X,
skeleton.Joints[Elbow].Position.Y,
skeleton.Joints[Elbow].Position.Z);
Point3D shoulder = new Point3D(
skeleton.Joints[Shoulder].Position.X,
skeleton.Joints[Shoulder].Position.Y,
skeleton.Joints[Shoulder].Position.Z);
tempJointPoints.Add(Hand, new Point3D(hand.X - shoulder.X, hand.Y - shoulder.Y, hand.Z - shoulder.Z));
tempJointPoints.Add(Elbow, new Point3D(elbow.X - shoulder.X, elbow.Y - shoulder.Y, elbow.Z - shoulder.Z));
tempJointPoints.Add(Shoulder, new Point3D(0, 0, 0));
}
return tempJointPoints;
}
public override TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity)
{
throw new NotImplementedException();
}
internal virtual Point3D FindCursorPosition(Skeleton skeleton, LayoutAnchoring anchoring, Size layoutSize, TypingDexterity dexterity, SelectionMethod selectionM, ArmStretch armStretch)
{
SetJointsForDexterity(dexterity);
Joint shoulder = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Shoulder);
Joint hand = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Hand);
Joint wrist = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Wrist);
Joint hip = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Hip);
double width = layoutSize.Width / 100;
double height = layoutSize.Height / 100;
if (anchoring == LayoutAnchoring.VerticalShoulderLevel)
{
planeCenter.X = shoulder.Position.X + width / 2;
planeCenter.Y = shoulder.Position.Y;
}
else if (anchoring == LayoutAnchoring.VerticalBodyCenterLevel)
{
planeCenter.X = shoulder.Position.X + width / 2;
planeCenter.Y = (shoulder.Position.Y + hip.Position.Y) / 2;
}
else if (anchoring == LayoutAnchoring.HorizontalBottomLevel)
{
planeCenter.X = shoulder.Position.X + width / 2 + 0.05;
//because of the coordinate direction
planeCenter.Z = shoulder.Position.Z - height / 2 - 0.05;
}
double pointerPosX = (hand.Position.X + wrist.Position.X) / 2;
double distX = pointerPosX - planeCenter.X;
double pointerPosY = (hand.Position.Y + wrist.Position.Y) / 2;
double distY = pointerPosY - planeCenter.Y;
double pointerPosZ = (hand.Position.Z + wrist.Position.Z) / 2;
double distZ = pointerPosZ - planeCenter.Z;
Point3D pointer = new Point3D(pointerPosX, pointerPosY, pointerPosZ);
Point3D origin = new Point3D(shoulder.Position.X, shoulder.Position.Y, shoulder.Position.Z);
System.Windows.Media.Media3D.Vector3D distanceFromOrigin = ToolBox.CalculateDisplacement(
(System.Windows.Media.Media3D.Vector3D)pointer,
(System.Windows.Media.Media3D.Vector3D)origin);
Point3D cursorP = new Point3D(-1, -1, -1);
if (armStretch == ArmStretch.Long && distanceFromOrigin.Length < LONG_ARM_CONSTRAIN)
return lastPos = cursorP;
if (armStretch == ArmStretch.Short)
{
if (anchoring == LayoutAnchoring.VerticalShoulderLevel || anchoring == LayoutAnchoring.VerticalBodyCenterLevel)
{
if (Math.Abs(distanceFromOrigin.Z) > SHORT_ARM_CONSTRAIN)
return lastPos = cursorP;
}
else if (anchoring == LayoutAnchoring.HorizontalBottomLevel)
{
if (Math.Abs(distanceFromOrigin.Y) > SHORT_ARM_CONSTRAIN)
return lastPos = cursorP;
}
}
//if the pointer is close enough to plane
if (anchoring == LayoutAnchoring.VerticalShoulderLevel || anchoring == LayoutAnchoring.VerticalBodyCenterLevel)
{
//out of width boundary
if (Math.Abs(distX) > width / 2)
return lastPos = cursorP;
//out of height boundary
if (Math.Abs(distY) > height / 2)
return lastPos = cursorP;
double absX = distX + width / 2;
double absY = height - ((height / 2) + distY);
cursorP.X = absX / width;
cursorP.Y = absY / height;
cursorP.Z = distZ;
cursorP = FreezeMovementesForSelectionGesture(cursorP, selectionM);
lastPos = cursorP;
}
else //if (anchoring == LayoutAnchoring.HorizontalBottomLevel)
{
//out of column boundary
if (Math.Abs(distX) > width / 2)
return lastPos = cursorP;
//out of row boundary
if (Math.Abs(distZ) > height / 2)
return lastPos = cursorP;
double absX = distX + width / 2;
double absZ = ((height / 2) + distZ);
cursorP.X = absX / width;
cursorP.Y = absZ / height;
cursorP.Z = distY;
cursorP = FreezeMovementesForSelectionGesture(cursorP, selectionM);
lastPos = cursorP;
}
//if not close to plane, return an invalid cursorP
return lastPos = cursorP;
}
//this method is to be overridden by each particular typing method public abstract TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity);
private Point3D FreezeMovementesForSelectionGesture(Point3D cursorP, SelectionMethod selectionM)
{
cursorP.FrozenX = cursorP.X;
cursorP.FrozenY = cursorP.Y;
cursorP.FrozenZ = cursorP.Z;
if (selectionM == SelectionMethod.Push)
{
//If the movement looks like a push (movement in Z), then it ignores the movements in the other two dimensions
if (Math.Abs(lastPos.FrozenZ - cursorP.Z) >= TypingMethod.PUSH_AXIS_DISTANCE)
{
cursorP.IsFrozen = true;
cursorP.X = lastPos.X;
cursorP.Y = lastPos.Y;
//cursorP.Z = lastPos.Z;
}
}
else if (selectionM == SelectionMethod.Swipe)
{
//If the movement looks like a swipe (movement in X), then it ignores the movements in the other two dimensions
if (Math.Abs(lastPos.X - cursorP.X) < TypingMethod.SWIPE_THRESHOLD)
return cursorP;
//if (Math.Abs(lastPos.Y - cursorP.Y) >= TypingMethod.SWIPE_THRESHOLD)
// return cursorP;
cursorP.IsFrozen = true;
cursorP.FrozenX = lastPos.FrozenX;
cursorP.FrozenY = lastPos.FrozenY;
cursorP.FrozenZ = lastPos.FrozenZ;
}
return cursorP;
}
