public HandTrackingValue(ClassificationList classificationList, NormalizedLandmarkList landmarkList, NormalizedRect rect, List <Detection> detections)
{
Handedness = classificationList;
HandLandmarkList = landmarkList;
HandRect = rect;
PalmDetections = detections;
}
public void CorrectAndPredict(NormalizedLandmarkList normalizedLandmarkList)
{
if (landmarkFilters == null)
{
setupFilters(normalizedLandmarkList);
correctAndPredict(normalizedLandmarkList);
}
else
{
var oldPosition = landmarkFilters[0].GetPosition();
var newLandmark = normalizedLandmarkList.Landmark[0];
var newPosition = new Vector3(newLandmark.X, newLandmark.Y, newLandmark.Z);
var displacement = Vector3.Distance(oldPosition, newPosition);
//Debug.Log(displacement);
if (displacement > displacementLimit)
{
if (warmup < 20)
{
warmup++;
correctAndPredict(normalizedLandmarkList);
}
else
{
Debug.Log("Filtered error landmark data(d): " + displacement);
}
}
else
{
correctAndPredict(normalizedLandmarkList);
}
}
}
public PoseTrackingValue(Detection poseDetection, NormalizedLandmarkList poseLandmarks, LandmarkList poseWorldLandmarks, NormalizedRect roiFromLandmarks)
{
this.poseDetection = poseDetection;
this.poseLandmarks = poseLandmarks;
this.poseWorldLandmarks = poseWorldLandmarks;
this.roiFromLandmarks = roiFromLandmarks;
}
public void Draw(NormalizedLandmarkList faceLandmarks, NormalizedLandmarkList poseLandmarks,
NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks,
NormalizedLandmarkList leftIrisLandmarks, NormalizedLandmarkList rightIrisLandmarks, bool visualizeZ = false, int circleVertices = 128)
{
Draw(
faceLandmarks?.Landmark, poseLandmarks?.Landmark, leftHandLandmarks?.Landmark, rightHandLandmarks?.Landmark, leftIrisLandmarks?.Landmark, rightIrisLandmarks?.Landmark, visualizeZ
);
}
private void correctAndPredict(NormalizedLandmarkList normalizedLandmarkList)
{
for (int i = 0; i < count; i++)
{
landmarkFilters[i].Correct(normalizedLandmarkList.Landmark[i]);
landmarkFilters[i].Predict();
}
}
public void Draw(Transform screenTransform, ClassificationList handedness, NormalizedLandmarkList handLandmarkList,
NormalizedRect handRect, List <Detection> palmDetections, bool isFlipped = false)
{
handednessAnnotation.GetComponent <ClassificationAnnotationController>().Draw(screenTransform, handedness);
handLandmarkListAnnotation.GetComponent <HandLandmarkListAnnotationController>().Draw(screenTransform, handLandmarkList, isFlipped);
handRectAnnotation.GetComponent <RectAnnotationController>().Draw(screenTransform, handRect, isFlipped);
palmDetectionsAnnotation.GetComponent <DetectionListAnnotationController>().Draw(screenTransform, palmDetections, isFlipped);
}
private void setupFilters(NormalizedLandmarkList normalizedLandmarkList)
{
landmarkFilters = new List <LandmarkFilter>();
this.count = normalizedLandmarkList.Landmark.Count;
for (int i = 0; i < count; i++)
{
landmarkFilters.Add(new LandmarkFilter(timeInterval, noise));
}
}
public HolisticValue(NormalizedLandmarkList PoseLandmarks, NormalizedRect PoseRoi, Detection PoseDetection,
NormalizedLandmarkList FaceLandmarks, NormalizedLandmarkList LeftHandLandmarks, NormalizedLandmarkList RightHandLandmarks)
{
this.PoseLandmarks = PoseLandmarks;
this.PoseRoi = PoseRoi;
this.PoseDetection = PoseDetection;
this.FaceLandmarks = FaceLandmarks;
this.LeftHandLandmarks = LeftHandLandmarks;
this.RightHandLandmarks = RightHandLandmarks;
}
public void DrawNow(NormalizedLandmarkList faceLandmarkList, NormalizedLandmarkList poseLandmarkList,
NormalizedLandmarkList leftHandLandmarkList, NormalizedLandmarkList rightHandLandmarkList)
{
DrawNow(
faceLandmarkList?.Landmark,
poseLandmarkList?.Landmark,
leftHandLandmarkList?.Landmark,
rightHandLandmarkList?.Landmark
);
SyncNow();
}
public static List <Vector3> ToList(this NormalizedLandmarkList input)
{
List <Vector3> output = new List <Vector3>();
var landmarks = input.Landmark;
for (int i = 0; i < landmarks.Count; i++)
{
var v = new Vector3(landmarks[i].X, landmarks[i].Y, landmarks[i].Z);
output.Add(v);
}
return(output);
}
private IList <NormalizedLandmark> GetIrisLandmarks(NormalizedLandmarkList landmarkList, Side side)
{
var irisLandmarks = new List <NormalizedLandmark>(5);
var offset = 468 + (side == Side.Left ? 0 : 5);
for (var i = 0; i < 5; i++)
{
irisLandmarks.Add(landmarkList.Landmark[offset + i]);
}
return(irisLandmarks);
}
public HolisticTrackingValue(Detection poseDetection, NormalizedLandmarkList poseLandmarks,
NormalizedLandmarkList faceLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks,
NormalizedLandmarkList leftIrisLandmarks, NormalizedLandmarkList rightIrisLandmarks,
LandmarkList poseWorldLandmarks, NormalizedRect poseRoi)
{
this.poseDetection = poseDetection;
this.poseLandmarks = poseLandmarks;
this.faceLandmarks = faceLandmarks;
this.leftIrisLandmarks = leftIrisLandmarks;
this.rightIrisLandmarks = rightIrisLandmarks;
this.leftHandLandmarks = leftHandLandmarks;
this.rightHandLandmarks = rightHandLandmarks;
this.poseWorldLandmarks = poseWorldLandmarks;
this.poseRoi = poseRoi;
}
protected override IEnumerator WaitForNextValue()
{
List <Detection> faceDetections = null;
NormalizedRect faceRect = null;
NormalizedLandmarkList faceLandmarksWithIris = null;
if (runningMode == RunningMode.Sync)
{
var _ = graphRunner.TryGetNext(out faceDetections, out faceRect, out faceLandmarksWithIris, true);
}
else if (runningMode == RunningMode.NonBlockingSync)
{
yield return(new WaitUntil(() => graphRunner.TryGetNext(out faceDetections, out faceRect, out faceLandmarksWithIris, false)));
}
_faceDetectionsAnnotationController.DrawNow(faceDetections);
_faceRectAnnotationController.DrawNow(faceRect);
_faceLandmarksWithIrisAnnotationController.DrawNow(faceLandmarksWithIris);
}
public void Draw(Transform screenTransform, NormalizedLandmarkList poseLandmarks, NormalizedRect poseRoi, Detection poseDetection,
NormalizedLandmarkList faceLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks, bool isFlipped = false)
{
poseLandmarksAnnotation.GetComponent <FullBodyPoseLandmarkListAnnotationController>().Draw(screenTransform, poseLandmarks, isFlipped);
poseRoiAnnotation.GetComponent <RectAnnotationController>().Draw(screenTransform, poseRoi, isFlipped);
poseDetectionAnnotation.GetComponent <DetectionAnnotationController>().Draw(screenTransform, poseDetection, isFlipped);
faceLandmarksAnnotation.GetComponent <FaceLandmarkListAnnotationController>().Draw(screenTransform, faceLandmarks, isFlipped);
leftHandLandmarksAnnotation.GetComponent <HandLandmarkListAnnotationController>().Draw(screenTransform, leftHandLandmarks, isFlipped);
rightHandLandmarksAnnotation.GetComponent <HandLandmarkListAnnotationController>().Draw(screenTransform, rightHandLandmarks, isFlipped);
if (faceLandmarks.Landmark.Count == 0)
{
ClearIrisAnnotations();
}
else if (faceLandmarks.Landmark.Count > 468)
{
var leftIrisLandmarks = GetIrisLandmarks(faceLandmarks, Side.Left);
leftIrisLandmarksAnnotation.GetComponent <IrisAnnotationController>().Draw(screenTransform, leftIrisLandmarks, isFlipped);
var rightIrisLandmarks = GetIrisLandmarks(faceLandmarks, Side.Right);
rightIrisLandmarksAnnotation.GetComponent <IrisAnnotationController>().Draw(screenTransform, rightIrisLandmarks, isFlipped);
}
}
public void DrawLeftIrisLandmarkListLater(NormalizedLandmarkList leftIrisLandmarkList)
{
DrawLeftIrisLandmarkListLater(leftIrisLandmarkList?.Landmark);
}
public void DrawNow(NormalizedLandmarkList target)
{
DrawNow(target?.Landmark);
}
public void DrawLater(NormalizedLandmarkList target)
{
DrawLater(target?.Landmark);
}
public override string GetMark()
{
// if detect hands change state machine.
//Debug.Log(currDirection);
if (handTrackingValue != null && handTrackingValue.PalmDetections != null && handTrackingValue.PalmDetections.Count > 0)
{
// blue color one...
float width = (float)handTrackingValue.PalmDetections[0].LocationData.RelativeBoundingBox.Width;
float height = (float)handTrackingValue.PalmDetections[0].LocationData.RelativeBoundingBox.Height;
// center of the X and Y
float currX = width / 2 + (float)handTrackingValue.PalmDetections[0].LocationData.RelativeBoundingBox.Xmin;
float currY = height / 2 + (float)handTrackingValue.PalmDetections[0].LocationData.RelativeBoundingBox.Ymin;
// the filter to remove noise rect...
if (width >= noise_radio_ratio && height >= noise_radio_ratio)
{
// send landmark here:
if (handTrackingValue.HandLandmarkLists != null && handTrackingValue.HandLandmarkLists.Count > 0)
{
NormalizedLandmarkList landmarks = handTrackingValue.HandLandmarkLists[0];
for (int i = 0; i < landmarks.Landmark.Count; i++)
{
float tempX, tempY, tempZ;
tempX = landmarks.Landmark[i].X;
tempY = landmarks.Landmark[i].Y;
tempZ = landmarks.Landmark[i].Z;
string temp = handsLandmarkMessage + "X=&" + tempX + "Y=&" + tempY + "Z=&" + tempZ;
MessageMgr.GetIns().Dispatch("HandMessage", temp);
}
//Debug.Log(landmarks.Landmark[0].X + " , " + landmarks.Landmark[0].Y);
}
if (currState == handState.outScreen)
{
/*
* outScreen state.
* once we detect the hand show in screen, There will be two situation,
* 1. first time came into the screen.
* 2. the hand left screen because of the move command..
* For the first situation, We set a 3s frozon time,( just change the handState to move. and set time);
* For the second situation. We should not change the original time,
* because it's an instant state, we don't need to update location data, the location will be modified inside move state.
*
* by combining this two situations, we need set a relative small time, (say frozen_time/2) to fit the both cases.
*/
currState = handState.move;
originTime = Time.time - freeze_time / 2;
}
else if (currState == handState.move)
{
/*
*
* move state
* we set a frozon time, within it the program won't detect any move you've down.
* but we will track your current position.
* Time.time return current program running time. 1.0000s, 2.38491s......
*/
originX = currX;
originY = currY;
if ((Time.time - originTime) > freeze_time)
{
// Frozon time end, change state to idle, ready to detect new action, Sync the time to current time.
currState = handState.idle;
originTime = Time.time;
currDirection = moveDirection.direction_idle;
}
else
{
// Frozon time, do nothing here..
}
}
else if (currState == handState.idle)
{
/*
*
* idle state
* In this state, we detect hand shape instantly, (hand spread out, fist)
* And we are ready to detect hand move direction (left, right, up, down)
* If we detect a long range move, we change state to idle, change hand position to move direction, and start frozon time.
*
*/
// detect hand shape by calculate vector.
if (handTrackingValue != null && handTrackingValue.HandLandmarkLists != null && handTrackingValue.HandLandmarkLists.Count > 0)
{
NormalizedLandmarkList landmarks = handTrackingValue.HandLandmarkLists[0];
Vector2[] cordinates = new Vector2[landmarks.Landmark.Count];
for (int i = 0; i < cordinates.Length; i++)
{
cordinates[i] = new Vector2(landmarks.Landmark[i].X, landmarks.Landmark[i].Y);
}
Vector2 vector68 = cordinates[8] - cordinates[6];
Vector2 vector56 = cordinates[6] - cordinates[5];
Vector2 vector1012 = cordinates[12] - cordinates[10];
Vector2 vector910 = cordinates[10] - cordinates[9];
Vector2 vector1416 = cordinates[16] - cordinates[14];
Vector2 vector1314 = cordinates[14] - cordinates[13];
Vector2 vector1820 = cordinates[20] - cordinates[18];
Vector2 vector1718 = cordinates[18] - cordinates[17];
// the angle between some knuckles.
float angle_index = Vector2.Angle(vector68, vector56);
float angle_middle = Vector2.Angle(vector910, vector1012);
float angle_ring = Vector2.Angle(vector1314, vector1416);
float angle_pinky = Vector2.Angle(vector1718, vector1820);
if (angle_index <= hand_spread_out_angle && angle_middle <= hand_spread_out_angle &&
angle_ring <= hand_spread_out_angle && angle_pinky <= hand_spread_out_angle)
{
//Debug.Log("finger_spread_out");
currShape = handShape.finger_spread_out;
}
else if (angle_index >= fist_angle && angle_middle >= fist_angle &&
angle_ring >= fist_angle && angle_pinky >= fist_angle)
{
//Debug.Log("fist");
currShape = handShape.fist;
}
else
{
// none shape
currShape = handShape.none;
}
}
// calculate hand move direction.
float diffX = currX - originX;
float diffY = currY - originY;
if (Mathf.Abs(diffX) > 0.15 || Mathf.Abs(originY - currY) > 0.15)
{
currState = handState.move;
originX = currX;
originY = currY;
originTime = Time.time;
// define which derection it move...
// first we just compare diff scale regrad horizontal and vertical...
if (Mathf.Pow(diffX, 2) - Mathf.Pow(diffY, 2) > 0)
{
// x is larger.
if (diffX > 0)
{
currDirection = moveDirection.right;
}
else
{
currDirection = moveDirection.left;
}
}
else
{
if (diffY > 0)
{
currDirection = moveDirection.down;
}
else
{
currDirection = moveDirection.up;
}
}
}
string temp = handShapeDirectionMessage + "Gesture=&" + currShape + "Direction=&" + currDirection;
MessageMgr.GetIns().Dispatch("HandMessage", temp);
Debug.Log(currShape + " , " + currDirection);
}
}
}
else
{
currState = handState.outScreen;
}
return(null);
}
public void Draw(Transform screenTransform, NormalizedLandmarkList poseLandmarkList, Detection poseDetection, bool isFlipped = false)
{
poseLandmarkListAnnotation.GetComponent <PoseLandmarkListAnnotationController>().Draw(screenTransform, poseLandmarkList, isFlipped);
poseDetectionAnnotation.GetComponent <DetectionAnnotationController>().Draw(screenTransform, poseDetection, isFlipped);
}
public void Draw(NormalizedLandmarkList target, bool visualizeZ = false, int vertices = 128)
{
Draw(target?.Landmark, visualizeZ, vertices);
}
void OnRightIrisLandmarksOutput(NormalizedLandmarkList rightIrisLandmarks)
{
holisticAnnotationController.DrawRightIrisLandmarkListLater(rightIrisLandmarks);
}
public IrisTrackingValue(List <Detection> faceDetections, NormalizedRect faceRect, NormalizedLandmarkList faceLandmarksWithIris)
{
this.faceRect = faceRect;
this.faceDetections = faceDetections;
this.faceLandmarksWithIris = faceLandmarksWithIris;
}
void OnFaceLandmarksWithIrisOutput(NormalizedLandmarkList faceLandmarkListWithIris)
{
faceLandmarksWithIrisAnnotationController.DrawLater(faceLandmarkListWithIris);
}
public void DrawLater(NormalizedLandmarkList target)
{
UpdateCurrentTarget(target?.Landmark, ref currentTarget);
}
public HandTrackingValue(ClassificationList classificationList, NormalizedLandmarkList landmarkList, NormalizedRect rect) :
this(classificationList, landmarkList, rect, new List <Detection>())
{
}
public void DrawFaceLandmarkListLater(NormalizedLandmarkList faceLandmarkList)
{
DrawFaceLandmarkListLater(faceLandmarkList?.Landmark);
}
void OnLeftIrisLandmarksOutput(NormalizedLandmarkList leftIrisLandmarks)
{
holisticAnnotationController.DrawLeftIrisLandmarkListLater(leftIrisLandmarks);
}
public void DrawPoseLandmarkListLater(NormalizedLandmarkList poseLandmarkList)
{
DrawPoseLandmarkListLater(poseLandmarkList?.Landmark);
}
public void Draw(NormalizedLandmarkList targets, bool visualizeZ = true)
{
Draw(targets.Landmark, visualizeZ);
}
public void DrawRightIrisLandmarkListLater(NormalizedLandmarkList rightIrisLandmarkList)
{
DrawRightIrisLandmarkListLater(rightIrisLandmarkList?.Landmark);
}
