コード例 #1
0
    public override TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Microsoft.Kinect.Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity)
    {
      double columnData = cursor.X;
      double rowData = cursor.Y;
      double constraintData = cursor.Z;

      if (cursor.IsFrozen)
      {
        columnData = cursor.FrozenX;
        rowData = cursor.FrozenY;
        constraintData = cursor.FrozenZ;
      }

      if (columnData < 0 || rowData < 0)
        return new TypingStatus();
      double stepX = 1.0 / layoutCols;
      double stepY = 1.0 / layoutRows;

      int col = (int)(columnData / stepX);
      int row = (int)(rowData / stepY);

      TypingStatus status = new TypingStatus();
      status.HighlightedKey = layout[row * layoutCols + col];
      if (gestures != null && gestures.Count > 0 && gestures.ElementAt(0).Type == GestureType.Tap)
      {
        TypingGesture gesture = gestures.ElementAt(0);
        col = (int)(gesture.Position.X / stepX);
        row = (int)(gesture.Position.Y / stepY);
        status.SelectedKey = layout[row * layoutCols + col];
      }

      return status;
    }
コード例 #2
0
    public override TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Microsoft.Kinect.Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity)
    {
      double columnData = cursor.X;
      double rowData = cursor.Y;
      double constraintData = cursor.Z;

      if (columnData < 0 || rowData < 0)
        return new TypingStatus();

      double stepX = 1.0 / layoutCols;
      double stepY = 1.0 / layoutRows;

      int col = (int)(columnData / stepX);
      int row = (int)(rowData / stepY);
      currentPointingIndex = row * layoutCols + col;
      TypingStatus status = new TypingStatus();
      if (currentSequenceIndex == 36)
      {
        return status;
      }
      if (layout[currentPointingIndex] == sequence[currentSequenceIndex])
      {
        status.HighlightedKey = layout[currentPointingIndex];
        if (currentSequenceIndex < sequence.Length)
        {
          currentSequenceIndex++;
          if (!started)
            started = true;
          else
            correctCount++;
        }
      }
      else 
      {
        if (currentPointingIndex != previousPointingIndex)
        {
          if (started)
          {
            errorCount++;
          }
        }
      }

      if (gestures != null && gestures.Count > 0 && gestures.ElementAt(0).Type == GestureType.Tap)
      {
        TypingGesture gesture = gestures.ElementAt(0);
        col = (int)(gesture.Position.X / stepX);
        row = (int)(gesture.Position.Y / stepY);
        currentPointingIndex = row * layoutCols + col;
        status.SelectedKey = layout[currentPointingIndex];
      }

      previousPointingIndex = currentPointingIndex;
      return status;
    }
コード例 #3
0
    private List<TypingGesture> ProcessGesturesPush(Point3D cursor)
    {
      if (cursor.X == -1 || cursor.Y == -1)
        return null;

      List<TypingGesture> gestures = new List<TypingGesture>();
      InputInfo info = new InputInfo() { Position = cursor };
      System.Windows.Media.Media3D.Vector3D displacement = cursor - lastPos;
      lastPos = cursor;

      if (movementSequenceSB.Length == movementSequence.Capacity)
      {
        movementSequenceSB.Remove(0, 1);
        movementSequence.RemoveAt(0);
      }

      if (displacement.X > LATERAL_THRESHOLD_PUSH || displacement.Y > LATERAL_THRESHOLD_PUSH)
        info.Movement = 'M'; //moving
      else if (Math.Abs(displacement.Z) > DEPTH_THRESHOLD)  //doing some movement on depth
      {
        if (displacement.Z < 0)
          info.Movement = 'P'; //released
        else
          info.Movement = 'R'; //pressed
      }
      else
        info.Movement = 'S'; //steady --- not moving
      movementSequenceSB.Append(info.Movement);
      movementSequence.Add(info);


      String tmpSequence = MovementSequence;
      MatchCollection taps = regexTapCrank.Matches(tmpSequence);
      if (taps.Count > 0)
      {
        Match tap = taps[0];
        gestures.Add(new TypingGesture()
        {
          Time = DateTime.Now,
          Type = GestureType.Tap,
          Position = movementSequence[tap.Index].Position
        });
        movementSequenceSB.Remove(tap.Index, tap.Length);
        movementSequence.RemoveRange(tap.Index, tap.Length);
      }
      OnPropertyChanged("MovementSequence");

      return gestures;
    }
コード例 #4
0
 internal ICollection<TypingGesture> ProcessGestures(Skeleton skeleton, double deltaTimeMilliseconds, Point3D cursor, Point3D secondaryCursor,
   SelectionMethod selectionM, Key highlightedKey, bool userClicked)
 {
   List<TypingGesture> gestures = null;
   if (selectionM == SelectionMethod.Push)
     gestures = ProcessGesturesPush(cursor);
   else if (selectionM == SelectionMethod.Swipe)
     gestures = ProcessGesturesSwipe(cursor);
   else if (selectionM == SelectionMethod.Timer)
     gestures = ProcessGesturesTimer(cursor, deltaTimeMilliseconds, highlightedKey);
   else if (selectionM == SelectionMethod.SecondHand)
     gestures = ProcessGesturesSecondHand(cursor, secondaryCursor, deltaTimeMilliseconds, highlightedKey);
   else if (selectionM == SelectionMethod.Click)
     gestures = ProcessGesturesClick(cursor, userClicked);
   return gestures;
 }
コード例 #5
0
    private void DoWork(Skeleton skeleton, double deltaMilliseconds)
    {
      Size layoutSize = new Size(35, 35);
      if(planeSize == PlaneSize.p25x25)
        layoutSize = new Size(25, 25);

      //1- find the position of the cursor on the layout plane
      CursorLocation = TypingMethod.FindCursorPosition(skeleton, anchoring, layoutSize, dexterity, selectionMethod, armStretch);
      //2- find the second hand cursor
      secCursorLocation = secTypingMethod.FindCursorPosition(skeleton, anchoring, layoutSize, dexterity, selectionMethod, armStretch);
      //3- Looks for gestures [pressed, released]
      ICollection<TypingGesture> gestures = Recognizer.ProcessGestures(skeleton, deltaMilliseconds, cursorLocation, secCursorLocation, selectionMethod, highlightedKey, HasUserClicked);

      //4- Passes it on to the typing method itself
      TypingStatus status = TypingMethod.ProcessNewFrame(CursorLocation, gestures, skeleton, deltaMilliseconds, dexterity);
      HighlightedKey = status.HighlightedKey;
      SelectedKey = status.SelectedKey;
      
    }
コード例 #6
0
    public TypingEngine()
    {
      Methods = new List<TypingMethod>();
      Methods.Add(new ABCTypingMethod());
      Methods.Add(new QwertyTypingMethod());
      Methods.Add(new SplitQwertyTypingMethod());
      Methods.Add(new FitalyTypingMethod());
      Methods.Add(new CircularTypingMethod());
      Methods.Add(new TahnuTypingMethod());
      Methods.Add(new LabyrinthTypingMethod());
      Methods.Add(new SeatoTypingMethod());
      Methods.Add(new BoxTypingMethod());

      TypingMethod = Methods.Last();
      secTypingMethod = new SecondHandSelectionTypingMethod();

      Recognizer = new GestureRecognizer();
      PropertyChanged += new PropertyChangedEventHandler(Recognizer.TypingEngine_PropertyChanged);

      CursorLocation = new Point3D(0, 0, 0);
      secCursorLocation = new Point3D(0, 0, 0);
      planeSize = PlaneSize.p35x35;
      SelectionMethod = ArmText.Typing.SelectionMethod.Click;
      Dexterity = TypingDexterity.Right;
    }
コード例 #7
0
    private List<TypingGesture> ProcessGesturesClick(Point3D cursor, bool userClicked)
    {
      if (cursor.X == -1 || cursor.Y == -1)
        return null;

      List<TypingGesture> gestures = new List<TypingGesture>();
      if (!userClicked)
        return gestures;

      gestures.Add(new TypingGesture()
      {
        Position = cursor,
        Type = GestureType.Tap,
        Time = DateTime.Now
      });
      return gestures;
    }
コード例 #8
0
    private List<TypingGesture> ProcessGesturesSecondHand(Point3D cursor, Point3D secondaryCursor, double deltaTimeMilliseconds, Key highlightedKey)
    {
      if (cursor.X == -1 || cursor.Y == -1)
        return null;

      List<TypingGesture> gestures = new List<TypingGesture>();
      InputInfo info = new InputInfo() { Position = cursor };
      System.Windows.Media.Media3D.Vector3D displacement = cursor - lastPos;
      lastPos = cursor;
      if (movementSequenceSB.Length == movementSequence.Capacity)
      {
        movementSequenceSB.Remove(0, 1);
        movementSequence.RemoveAt(0);
      }
      if (displacement.X > LATERAL_THRESHOLD_TIMER || displacement.Y > LATERAL_THRESHOLD_TIMER)
        info.Movement = 'M'; //moving
      else
        info.Movement = 'S'; //steady --- not moving
      movementSequenceSB.Append(info.Movement);
      movementSequence.Add(info);

      if (timerState == ArmText.Typing.TimerState.Nothing)
      {
        String tmpSequence = MovementSequence;
        MatchCollection starts = regexStartTimer.Matches(tmpSequence);
        if (starts.Count > 0)
        {
          Match start = starts[0];
          TimerState = ArmText.Typing.TimerState.Running;
          secondHandSelectionInfo = movementSequence[start.Index];
          movementSequenceSB.Remove(start.Index, start.Length);
          movementSequence.RemoveRange(start.Index, start.Length);
        }
      }
      else if (timerState == ArmText.Typing.TimerState.Running)
      {
        if (secondaryCursor.X > LATERAL_THRESHOLD_SECOND_HAND && DateTime.Now > lastSecondHandSelection + secondaryHandSelectionTimeDelta)
        {
          gestures.Add(new TypingGesture()
          {
            Time = DateTime.Now,
            Type = GestureType.Tap,
            Position = secondHandSelectionInfo.Position
          });
          lastSecondHandSelection = DateTime.Now;
          TimerState = ArmText.Typing.TimerState.Nothing;
        }
      }
      OnPropertyChanged("MovementSequence");

      return gestures;
    }
コード例 #9
0
    private List<TypingGesture> ProcessGesturesTimer(Point3D cursor, double delta, Key highlightedKey)
    {
      if (cursor.X == -1 || cursor.Y == -1)
        return null;

      List<TypingGesture> gestures = new List<TypingGesture>();
      InputInfo info = new InputInfo() { Position = cursor };
      System.Windows.Media.Media3D.Vector3D displacement = cursor - lastPos;
      lastPos = cursor;

      if (movementSequenceSB.Length == movementSequence.Capacity)
      {
        movementSequenceSB.Remove(0, 1);
        movementSequence.RemoveAt(0);
      }

      if (Math.Abs(displacement.X) > LATERAL_THRESHOLD_TIMER || Math.Abs(displacement.Y) > LATERAL_THRESHOLD_TIMER)
        info.Movement = 'M'; //moving
      else
        info.Movement = 'S'; //steady --- not moving
      movementSequenceSB.Append(info.Movement);
      movementSequence.Add(info);

      String tmpSequence = MovementSequence;
      if (timerState == ArmText.Typing.TimerState.Nothing)
      {
        MatchCollection starts = regexStartTimer.Matches(tmpSequence);
        if (starts.Count > 0)
        {
          Match start = starts[0];
          TimerState = ArmText.Typing.TimerState.Running;
          movementSequenceSB.Remove(start.Index, start.Length);
          movementSequence.RemoveRange(start.Index, start.Length);
        }
      }
      else if (timerState == ArmText.Typing.TimerState.Running)
      {
        MatchCollection selections = regexTapTimer.Matches(tmpSequence);
        if (selections.Count > 0)
        {
          Match selection = selections[0];
          gestures.Add(new TypingGesture()
          {
            Time = DateTime.Now,
            Type = GestureType.Tap,
            Position = movementSequence[selection.Index].Position
          });
          TimerState = ArmText.Typing.TimerState.Nothing;
          movementSequenceSB.Remove(selection.Index, selection.Length);
          movementSequence.RemoveRange(selection.Index, selection.Length);
        }
      }
      OnPropertyChanged("MovementSequence");

      return gestures;
    }
コード例 #10
0
    private List<TypingGesture> ProcessGesturesSwipe(Point3D cursor)
    {
      if (cursor.X == -1 || cursor.Y == -1)
        return null;

      List<TypingGesture> gestures = new List<TypingGesture>();
      InputInfo info = new InputInfo() { Position = cursor };
      lastPos = cursor;

      if (movementSequenceSB.Length == movementSequence.Capacity)
      {
        movementSequenceSB.Remove(0, 1);
        movementSequence.RemoveAt(0);
      }

      if (cursor.IsFrozen)
        info.Movement = 'S'; //steady --- not moving
      else
        info.Movement = 'M'; //moving
      movementSequenceSB.Append(info.Movement);
      movementSequence.Add(info);

      String tmpSequence = MovementSequence;
      if (timerState == ArmText.Typing.TimerState.Nothing)
      {
        MatchCollection starts = regexStartSwipe.Matches(tmpSequence);
        if (starts.Count > 0)
        {
          Match start = starts[0];
          TimerState = ArmText.Typing.TimerState.Running;
          secondHandSelectionInfo = movementSequence[start.Index];
          movementSequenceSB.Remove(start.Index, start.Length);
          movementSequence.RemoveRange(start.Index, start.Length);
        }
      }
      else if (timerState == ArmText.Typing.TimerState.Running)
      {
        MatchCollection taps = regexTapSwipe.Matches(tmpSequence);
        if (taps.Count > 0)
        {
          Match tap = taps[0];
          gestures.Add(new TypingGesture()
          {
            Time = DateTime.Now,
            Type = GestureType.Tap,
            Position = movementSequence[tap.Index].Position
          });
          TimerState = ArmText.Typing.TimerState.Nothing;
          movementSequenceSB.Remove(tap.Index, tap.Length);
          movementSequence.RemoveRange(tap.Index, tap.Length);
        }
      }
      OnPropertyChanged("MovementSequence");

      return gestures;
    }
コード例 #11
0
    private System.Windows.Media.Media3D.Vector3D CalculateCenterMass(Point3D shoulder, Point3D elbow, Point3D hand)
    {
      //upper arm center mass and fore arm center mass
      upperCM.X = (elbow.X - shoulder.X) * UPPER_ARM_CENTER_GRAVITY_RATIO;
      upperCM.Y = (elbow.Y - shoulder.Y) * UPPER_ARM_CENTER_GRAVITY_RATIO;
      upperCM.Z = (elbow.Z - shoulder.Z) * UPPER_ARM_CENTER_GRAVITY_RATIO;

      //lower arm
      foreCM.X = (hand.X - elbow.X) * foreArmAndHandCenterOfGravityRatio + elbow.X;
      foreCM.Y = (hand.Y - elbow.Y) * foreArmAndHandCenterOfGravityRatio + elbow.Y;
      foreCM.Z = (hand.Z - elbow.Z) * foreArmAndHandCenterOfGravityRatio + elbow.Z;

      //base on equation get whole arm center mass
      armCM.X = (foreCM.X - upperCM.X) * (1 - upperArmWeightProportion) + upperCM.X;
      armCM.Y = (foreCM.Y - upperCM.Y) * (1 - upperArmWeightProportion) + upperCM.Y;
      armCM.Z = (foreCM.Z - upperCM.Z) * (1 - upperArmWeightProportion) + upperCM.Z;

      var normalizingFactor = MALE_UPPERARM_LENGHT / ((elbow - shoulder).Length * 100);
      if(maxForce == FEMALE_MAX_FORCE)
        normalizingFactor = FEMALE_UPPERARM_LENGHT / ((elbow - shoulder).Length * 100);
      var normalizedArmCM = armCM * normalizingFactor;

      return normalizedArmCM;
    }
コード例 #12
0
    private Dictionary<JointType, Point3D> GetArmPoints(Skeleton skeleton)
    {
      Dictionary<JointType, Point3D> tempJointPoints = new Dictionary<JointType, Point3D>();
      //right side joint tracking
      if (skeleton.Joints[Hand].TrackingState == JointTrackingState.Tracked &&
        skeleton.Joints[Shoulder].TrackingState == JointTrackingState.Tracked &&
        skeleton.Joints[Elbow].TrackingState == JointTrackingState.Tracked)
      {
        Joint rightHand = skeleton.Joints[Hand];
        Joint rightWrist = skeleton.Joints[Wrist];
        Point3D hand = new Point3D(
          0.397 * (rightHand.Position.X - rightWrist.Position.X) + rightWrist.Position.X,
          0.397 * (rightHand.Position.Y - rightWrist.Position.Y) + rightWrist.Position.Y,
          0.397 * (rightHand.Position.Z - rightWrist.Position.Z) + rightWrist.Position.Z);

        Point3D elbow = new Point3D(
          skeleton.Joints[Elbow].Position.X,
          skeleton.Joints[Elbow].Position.Y,
          skeleton.Joints[Elbow].Position.Z);
        Point3D shoulder = new Point3D(
          skeleton.Joints[Shoulder].Position.X,
          skeleton.Joints[Shoulder].Position.Y,
          skeleton.Joints[Shoulder].Position.Z);

        tempJointPoints.Add(Hand, new Point3D(hand.X - shoulder.X, hand.Y - shoulder.Y, hand.Z - shoulder.Z));
        tempJointPoints.Add(Elbow, new Point3D(elbow.X - shoulder.X, elbow.Y - shoulder.Y, elbow.Z - shoulder.Z));
        tempJointPoints.Add(Shoulder, new Point3D(0, 0, 0));
      }
      return tempJointPoints;
    }
コード例 #13
0
 public override TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity)
 {
   throw new NotImplementedException();
 }
コード例 #14
0
    internal virtual Point3D FindCursorPosition(Skeleton skeleton, LayoutAnchoring anchoring, Size layoutSize, TypingDexterity dexterity, SelectionMethod selectionM, ArmStretch armStretch)
    {
      SetJointsForDexterity(dexterity);

      Joint shoulder = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Shoulder);
      Joint hand = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Hand);
      Joint wrist = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Wrist);
      Joint hip = skeleton.Joints.SingleOrDefault(tmp => tmp.JointType == Hip);

      double width = layoutSize.Width / 100;
      double height = layoutSize.Height / 100;

      if (anchoring == LayoutAnchoring.VerticalShoulderLevel)
      {
        planeCenter.X = shoulder.Position.X + width / 2;
        planeCenter.Y = shoulder.Position.Y;
      }
      else if (anchoring == LayoutAnchoring.VerticalBodyCenterLevel)
      {
        planeCenter.X = shoulder.Position.X + width / 2;
        planeCenter.Y = (shoulder.Position.Y + hip.Position.Y) / 2;
      }
      else if (anchoring == LayoutAnchoring.HorizontalBottomLevel)
      {
        planeCenter.X = shoulder.Position.X + width / 2 + 0.05;
        //because of the coordinate direction
        planeCenter.Z = shoulder.Position.Z - height / 2 - 0.05;
      }

      double pointerPosX = (hand.Position.X + wrist.Position.X) / 2;
      double distX = pointerPosX - planeCenter.X;
      double pointerPosY = (hand.Position.Y + wrist.Position.Y) / 2;
      double distY = pointerPosY - planeCenter.Y;
      double pointerPosZ = (hand.Position.Z + wrist.Position.Z) / 2;
      double distZ = pointerPosZ - planeCenter.Z;

      Point3D pointer = new Point3D(pointerPosX, pointerPosY, pointerPosZ);
      Point3D origin = new Point3D(shoulder.Position.X, shoulder.Position.Y, shoulder.Position.Z);
      System.Windows.Media.Media3D.Vector3D distanceFromOrigin = ToolBox.CalculateDisplacement(
        (System.Windows.Media.Media3D.Vector3D)pointer,
        (System.Windows.Media.Media3D.Vector3D)origin);

      Point3D cursorP = new Point3D(-1, -1, -1);
      if (armStretch == ArmStretch.Long && distanceFromOrigin.Length < LONG_ARM_CONSTRAIN)
        return lastPos = cursorP;
      if (armStretch == ArmStretch.Short)
      {
        if (anchoring == LayoutAnchoring.VerticalShoulderLevel || anchoring == LayoutAnchoring.VerticalBodyCenterLevel)
        {
          if (Math.Abs(distanceFromOrigin.Z) > SHORT_ARM_CONSTRAIN)
            return lastPos = cursorP;
        }
        else if (anchoring == LayoutAnchoring.HorizontalBottomLevel)
        {
          if (Math.Abs(distanceFromOrigin.Y) > SHORT_ARM_CONSTRAIN)
            return lastPos = cursorP;
        }
      }

      //if the pointer is close enough to plane
      if (anchoring == LayoutAnchoring.VerticalShoulderLevel || anchoring == LayoutAnchoring.VerticalBodyCenterLevel)
      {
        //out of width boundary
        if (Math.Abs(distX) > width / 2)
          return lastPos = cursorP;
        //out of height boundary
        if (Math.Abs(distY) > height / 2)
          return lastPos = cursorP;

        double absX = distX + width / 2;
        double absY = height - ((height / 2) + distY);

        cursorP.X = absX / width;
        cursorP.Y = absY / height;
        cursorP.Z = distZ;
        cursorP = FreezeMovementesForSelectionGesture(cursorP, selectionM);
        lastPos = cursorP;
      }
      else //if (anchoring == LayoutAnchoring.HorizontalBottomLevel)
      {
        //out of column boundary
        if (Math.Abs(distX) > width / 2)
          return lastPos = cursorP;
        //out of row boundary
        if (Math.Abs(distZ) > height / 2)
          return lastPos = cursorP;

        double absX = distX + width / 2;
        double absZ = ((height / 2) + distZ);

        cursorP.X = absX / width;
        cursorP.Y = absZ / height;
        cursorP.Z = distY;
        cursorP = FreezeMovementesForSelectionGesture(cursorP, selectionM);
        lastPos = cursorP;
      }
      //if not close to plane, return an invalid cursorP
      return lastPos = cursorP;
    }
コード例 #15
0
 //this method is to be overridden by each particular typing method
 public abstract TypingStatus ProcessNewFrame(Point3D cursor, ICollection<TypingGesture> gestures, Skeleton stableSkeleton, double deltaTimeMilliseconds, TypingDexterity dexterity);
コード例 #16
0
    private Point3D FreezeMovementesForSelectionGesture(Point3D cursorP, SelectionMethod selectionM)
    {
      cursorP.FrozenX = cursorP.X;
      cursorP.FrozenY = cursorP.Y;
      cursorP.FrozenZ = cursorP.Z;

      if (selectionM == SelectionMethod.Push)
      {
        //If the movement looks like a push (movement in Z), then it ignores the movements in the other two dimensions
        if (Math.Abs(lastPos.FrozenZ - cursorP.Z) >= TypingMethod.PUSH_AXIS_DISTANCE)
        {
          cursorP.IsFrozen = true;
          cursorP.X = lastPos.X;
          cursorP.Y = lastPos.Y;
          //cursorP.Z = lastPos.Z;
        }
      }
      else if (selectionM == SelectionMethod.Swipe)
      {
        //If the movement looks like a swipe (movement in X), then it ignores the movements in the other two dimensions
        if (Math.Abs(lastPos.X  - cursorP.X) < TypingMethod.SWIPE_THRESHOLD)
          return cursorP;
        //if (Math.Abs(lastPos.Y - cursorP.Y) >= TypingMethod.SWIPE_THRESHOLD)
        //  return cursorP;

        cursorP.IsFrozen = true;
        cursorP.FrozenX = lastPos.FrozenX;
        cursorP.FrozenY = lastPos.FrozenY;
        cursorP.FrozenZ = lastPos.FrozenZ;
      }
      return cursorP;
    }