private void SetBoneRenderer() { BoneRenderer myBoneRenderer; if (this.gameObject.GetComponent <BoneRenderer>() == null) { myBoneRenderer = this.gameObject.AddComponent <BoneRenderer>(); } else { myBoneRenderer = this.gameObject.GetComponent <BoneRenderer>(); } boneTransforms = new List <Transform>(); Stack <KNNBone> boneStack = new Stack <KNNBone>(); boneStack.Push(this.rootBone); while (boneStack.Count > 0) { KNNBone topBone = boneStack.Pop(); foreach (KNNBone childBone in topBone.children) { boneStack.Push(childBone); } boneTransforms.Add(topBone.transform); } myBoneRenderer.transforms = boneTransforms.ToArray(); }
public void SetKNNSkeleton(KNNBone rootBone, bool ignoreRotation, KDTree <float, int> lHandQueryTree, KDTree <float, int> rHandQueryTree, int featureVectorLength) { this.ignoreRotation = ignoreRotation; this.rootBone = rootBone; this.lHandQueryTree = lHandQueryTree; this.rHandQueryTree = rHandQueryTree; this.featureVectorLength = featureVectorLength; SetBoneRenderer(); }
private string ComposeString() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendLine("#bones timesteps framesPerTimeStep ignoreRotation"); List <KNNBone> boneList = new List <KNNBone>(); Stack <KNNBone> boneStack = new Stack <KNNBone>(); boneStack.Push(this.rootBone); while (boneStack.Count > 0) { KNNBone top = boneStack.Pop(); foreach (KNNBone child in top.children) { boneStack.Push(child); } boneList.Add(top); } stringBuilder.AppendLine(boneList.Count.ToString() + " " + this.rootBone.rotations.Count.ToString() + " " + this.lHandQueryTree.Navigator.Point.Length.ToString() + " " + ignoreRotation.ToString()); stringBuilder.AppendLine("#name id parentid offsetx offsety offsetz rotations(quaternions * timesteps)"); foreach (KNNBone currBone in boneList) { string currLine = currBone.name + " "; currLine += boneList.IndexOf(currBone) + " "; currLine += boneList.IndexOf(currBone.parent) + " "; currLine += currBone.offset.x.ToString() + " " + currBone.offset.y.ToString() + " " + currBone.offset.z.ToString(); foreach (Quaternion currRot in currBone.rotations) { currLine += " " + currRot.x + " " + currRot.y + " " + currRot.z + " " + currRot.w; } stringBuilder.AppendLine(currLine); } string nextLine = this.KdTreeToString(this.lHandQueryTree); stringBuilder.AppendLine(nextLine); nextLine = this.KdTreeToString(this.rHandQueryTree); stringBuilder.AppendLine(nextLine); return(stringBuilder.ToString()); }
public void createKNNRig(string headBoneName, string rHandBoneName, string lHandBoneName, float slidingWindowSizeInMS, float slidingWindowOffsetInMS, float pollingRate, bool ignoreRotation, bool bothHandsInFeatureVec, string outputPath) { var lHandTreeNodes = new List <int>(); var lHandTreePoints = new List <float[]>(); var rHandTreeNodes = new List <int>(); var rHandTreePoints = new List <float[]>(); GameObject rootBoneObj = new GameObject(); KNNBone rootBone = rootBoneObj.AddComponent <KNNBone>(); Debug.Assert(files != null && files.Count > 0); loadFromFile(files[0]); setToFrame(0, true, true, true); rootBone.initKNNBone(root.transform); foreach (string file in files) { loadFromFile(file); FillKNNTrees(rootBone, ignoreRotation, bothHandsInFeatureVec, headBoneName, rHandBoneName, lHandBoneName, slidingWindowSizeInMS, slidingWindowOffsetInMS, pollingRate, lHandTreeNodes, rHandTreeNodes, lHandTreePoints, rHandTreePoints); } float slidingWindowSize = slidingWindowSizeInMS / 1000.0f; float targetFrameTime = 1.0f / pollingRate; int framesPerWindow = (int)((slidingWindowSize) / targetFrameTime); int featureVecLength = 9 * framesPerWindow; var rHandTree = new KDTree <float, int>(featureVecLength, rHandTreePoints.ToArray(), rHandTreeNodes.ToArray(), Metrics.WeightedL2Norm); var lHandTree = new KDTree <float, int>(featureVecLength, lHandTreePoints.ToArray(), lHandTreeNodes.ToArray(), Metrics.WeightedL2Norm); GameObject kNNSkeleton_ = new GameObject("KNN-Skeleton"); KNNSkeleton finalKNNSkeleton = kNNSkeleton_.AddComponent <KNNSkeleton>(); finalKNNSkeleton.SetKNNSkeleton(rootBone, ignoreRotation, lHandTree, rHandTree, featureVecLength); finalKNNSkeleton.Save(outputPath); DestroyImmediate(kNNSkeleton_); DestroyImmediate(rootBoneObj); GameObject kNNRig = new GameObject("KNN-Rig"); KNNRig kNNRigComponent = kNNRig.AddComponent <KNNRig>(); kNNRigComponent.skeletonPath = outputPath; }
public void initKNNBone(Transform rootTransform) { this.name = rootTransform.name; this.transform.localPosition = rootTransform.localPosition; this.transform.localRotation = rootTransform.localRotation; this.offset = this.transform.localPosition; this.children = new List <KNNBone>(); this.rotations = new List <Quaternion>(); for (int childIdx = 0; childIdx < rootTransform.childCount; childIdx++) { GameObject childObj = new GameObject(); KNNBone childBone = (KNNBone)childObj.AddComponent(typeof(KNNBone)); childBone.parent = this; childBone.transform.parent = this.transform; childBone.initKNNBone(rootTransform.GetChild(childIdx)); children.Add(childBone); } }
public void Parse(string file) { StreamReader reader = new StreamReader(file); string currLine = reader.ReadLine(); // skip comment currLine = reader.ReadLine(); string[] words = currLine.Split(' '); int nrOfBones = int.Parse(words[0]); int timeSteps = int.Parse(words[1]); this.featureVectorLength = int.Parse(words[2]); this.ignoreRotation = bool.Parse(words[3]); currLine = reader.ReadLine(); // skip comment GameObject rootBone = new GameObject(); this.rootBone = rootBone.AddComponent <KNNBone>(); this.rootBone.initKNNBoneFromFile(file); this.rootBone.transform.parent = this.transform; for (int currBoneIdx = 0; currBoneIdx < nrOfBones; currBoneIdx++) { currLine = reader.ReadLine(); // ignore since it's done in the initBoneFromFile } currLine = reader.ReadLine(); if (this.lHandQueryTree == null || this.lHandQueryTree.Count == 0) { this.lHandQueryTree = ParseKDTree(currLine, featureVectorLength, timeSteps); } currLine = reader.ReadLine(); if (this.rHandQueryTree == null || this.rHandQueryTree.Count == 0) { this.rHandQueryTree = ParseKDTree(currLine, featureVectorLength, timeSteps); } reader.Close(); }
private void FillKNNTrees(KNNBone rootBone, bool ignoreRotation, bool bothHandsInFeatureVec, string headBoneName, string rHandBoneName, string lHandBoneName, float slidingWindowSizeInMS, float slidingWindowOffsetInMS, float pollingRate, List <int> lHandTreeNodes, List <int> rHandTreeNodes, List <float[]> lHandTreePoints, List <float[]> rHandTreePoints) { float sourceFrameTime = bvhParser.frameTime; float targetFrameTime = 1.0f / pollingRate; float totalTime = bvhParser.frames * sourceFrameTime; int totalNewFrames = (int)(totalTime / targetFrameTime); float slidingWindowOffset = slidingWindowOffsetInMS / 1000.0f; float slidingWindowSize = slidingWindowSizeInMS / 1000.0f; int totalWindows = (int)((totalTime - (slidingWindowSize)) / (slidingWindowOffset)); int framesPerWindow = (int)((slidingWindowSize) / targetFrameTime); int nrOfFeatures = bothHandsInFeatureVec ? 18 : 9; int featureVecLength = nrOfFeatures * framesPerWindow; setToFrame(0, true, true, true); for (int currentWindow = 0; currentWindow < totalWindows; currentWindow++) { float[] lHandPoints = new float[featureVecLength]; float[] rHandPoints = new float[featureVecLength]; for (int currFrame = 0; currFrame < framesPerWindow; currFrame++) { float startTime = slidingWindowOffset * (float)currentWindow; float sourceFrame = getSourceFrameFromTargetTime(sourceFrameTime, startTime + (float)currFrame * targetFrameTime); setToFrame(sourceFrame, true, true, true); Vector3 rHandPosition = root.Find(rHandBoneName).position; Vector3 lHandPosition = root.Find(lHandBoneName).position; if (sourceFrame >= 1.0f) { setToFrame(sourceFrame - 1.0f, true, true, true); } Vector3 prevRHandPosition = root.Find(rHandBoneName).position; Vector3 prevLHandPosition = root.Find(lHandBoneName).position; if (sourceFrame >= 2.0f) { setToFrame(sourceFrame - 2.0f, true, true, true); } Vector3 prevPrevRHandPosition = root.Find(rHandBoneName).position; Vector3 prevPrevLHandPosition = root.Find(lHandBoneName).position; Vector3 lHandVelocity = (lHandPosition - prevLHandPosition) / sourceFrameTime; Vector3 rHandVelocity = (rHandPosition - prevRHandPosition) / sourceFrameTime; Vector3 prevLHandVelocity = (prevLHandPosition - prevPrevLHandPosition) / sourceFrameTime; Vector3 prevRHandVelocity = (prevRHandPosition - prevPrevRHandPosition) / sourceFrameTime; Vector3 lHandAcceleration = (lHandVelocity - prevLHandVelocity) / sourceFrameTime; Vector3 rHandAcceleration = (rHandVelocity - prevRHandVelocity) / sourceFrameTime; if (ignoreRotation) { SphericalCoords rHandSpPosition = SphericalCoords.CartesianToSpherical(rHandPosition); SphericalCoords lHandSpPosition = SphericalCoords.CartesianToSpherical(lHandPosition); float rHandAngle = rHandSpPosition.theta * Mathf.Rad2Deg; float lHandAngle = lHandSpPosition.theta * Mathf.Rad2Deg; //rotate velocity and acceleration angle around "theta" lHandVelocity = Quaternion.AngleAxis(lHandAngle, Vector3.up) * lHandVelocity; rHandVelocity = Quaternion.AngleAxis(rHandAngle, Vector3.up) * rHandVelocity; lHandAcceleration = Quaternion.AngleAxis(lHandAngle, Vector3.up) * lHandAcceleration; rHandAcceleration = Quaternion.AngleAxis(rHandAngle, Vector3.up) * rHandAcceleration; rHandSpPosition.theta = 0; lHandSpPosition.theta = 0; rHandPosition = rHandSpPosition.ToCartesian(); lHandPosition = lHandSpPosition.ToCartesian(); } int pointOffset = currFrame * nrOfFeatures; lHandPoints[pointOffset] = lHandPosition.x; lHandPoints[pointOffset + 1] = lHandPosition.y; lHandPoints[pointOffset + 2] = lHandPosition.z; lHandPoints[pointOffset + 3] = lHandVelocity.x; lHandPoints[pointOffset + 4] = lHandVelocity.y; lHandPoints[pointOffset + 5] = lHandVelocity.z; lHandPoints[pointOffset + 6] = lHandAcceleration.x; lHandPoints[pointOffset + 7] = lHandAcceleration.y; lHandPoints[pointOffset + 8] = lHandAcceleration.z; rHandPoints[pointOffset] = rHandPosition.x; rHandPoints[pointOffset + 1] = rHandPosition.y; rHandPoints[pointOffset + 2] = rHandPosition.z; rHandPoints[pointOffset + 3] = rHandVelocity.x; rHandPoints[pointOffset + 4] = rHandVelocity.y; rHandPoints[pointOffset + 5] = rHandVelocity.z; rHandPoints[pointOffset + 6] = rHandAcceleration.x; rHandPoints[pointOffset + 7] = rHandAcceleration.y; rHandPoints[pointOffset + 8] = rHandAcceleration.z; if (bothHandsInFeatureVec) { lHandPoints[pointOffset + 9] = rHandPosition.x; lHandPoints[pointOffset + 10] = rHandPosition.y; lHandPoints[pointOffset + 11] = rHandPosition.z; lHandPoints[pointOffset + 12] = rHandVelocity.x; lHandPoints[pointOffset + 13] = rHandVelocity.y; lHandPoints[pointOffset + 14] = rHandVelocity.z; lHandPoints[pointOffset + 15] = rHandAcceleration.x; lHandPoints[pointOffset + 16] = rHandAcceleration.y; lHandPoints[pointOffset + 17] = rHandAcceleration.z; rHandPoints[pointOffset + 9] = lHandPosition.x; rHandPoints[pointOffset + 10] = lHandPosition.y; rHandPoints[pointOffset + 11] = lHandPosition.z; rHandPoints[pointOffset + 12] = lHandVelocity.x; rHandPoints[pointOffset + 13] = lHandVelocity.y; rHandPoints[pointOffset + 14] = lHandVelocity.z; rHandPoints[pointOffset + 15] = lHandAcceleration.x; rHandPoints[pointOffset + 16] = lHandAcceleration.y; rHandPoints[pointOffset + 17] = lHandAcceleration.z; } } int lastPositionSourceIndex = (int)((currentWindow * slidingWindowOffset + slidingWindowSize) / sourceFrameTime); setToFrame(lastPositionSourceIndex, true, true, true); rootBone.AddRotationsFromTransform(root); lHandTreePoints.Add(lHandPoints); lHandTreeNodes.Add(lHandTreeNodes.Count); rHandTreePoints.Add(rHandPoints); rHandTreeNodes.Add(rHandTreeNodes.Count); } }
public void InitRig() { leftArmFeatureVec = new LinkedList <FeatureItem>(); rightArmFeatureVec = new LinkedList <FeatureItem>(); if (rHandSkeleton == null || rHandSkeleton.rootBone == null || rHandSkeleton.lHandQueryTree == null || rHandSkeleton.rHandQueryTree == null || rHandSkeleton.lHandQueryTree.Count == 0 || rHandSkeleton.rHandQueryTree.Count == 0) { if (rHandSkeleton == null) { GameObject kNNSkeleton_ = new GameObject("Right-KNN-Skeleton"); this.rHandSkeleton = kNNSkeleton_.AddComponent <KNNSkeleton>(); } this.rHandSkeleton.SetKNNSkeleton(skeletonPath); this.rHandSkeleton.transform.parent = this.transform; this.rHandSkeleton.rootBone.transform.parent = this.rHandSkeleton.transform; } if (lHandSkeleton == null || lHandSkeleton.rootBone == null || lHandSkeleton.lHandQueryTree == null || lHandSkeleton.rHandQueryTree == null || lHandSkeleton.lHandQueryTree.Count == 0 || lHandSkeleton.rHandQueryTree.Count == 0) { if (lHandSkeleton == null) { GameObject kNNSkeleton_ = new GameObject("Left-KNN-Skeleton"); this.lHandSkeleton = kNNSkeleton_.AddComponent <KNNSkeleton>(); } this.lHandSkeleton.SetKNNSkeleton(skeletonPath); this.lHandSkeleton.transform.parent = this.transform; this.lHandSkeleton.rootBone.transform.parent = this.lHandSkeleton.transform; } if (finalSkeletonBones == null) { GameObject kNNSkeleton_ = new GameObject("final-KNN-Skeleton-Bones"); GameObject rootBone = new GameObject("final-KNN-Skeleton-Bones"); this.finalSkeletonBones = rootBone.AddComponent <KNNBone>(); this.finalSkeletonBones.initKNNBoneFromFile(skeletonPath); this.finalSkeletonBones.transform.parent = kNNSkeleton_.transform; kNNSkeleton_.transform.parent = this.transform; } Stack <KNNBone> boneStackFinal = new Stack <KNNBone>(); Stack <KNNBone> boneStackLeft = new Stack <KNNBone>(); Stack <KNNBone> boneStackRight = new Stack <KNNBone>(); boneStackFinal.Push(this.finalSkeletonBones); boneStackLeft.Push(this.lHandSkeleton.rootBone); boneStackRight.Push(this.rHandSkeleton.rootBone); allBones = new List <Tuple <KNNBone, KNNBone, KNNBone> >(); //assuming they all have the same hierarchy while (boneStackFinal.Count > 0) { KNNBone topBoneFinal = boneStackFinal.Pop(); KNNBone topBoneLeft = boneStackLeft.Pop(); KNNBone topBoneRight = boneStackRight.Pop(); foreach (KNNBone childBone in topBoneFinal.children) { boneStackFinal.Push(childBone); } foreach (KNNBone childBone in topBoneLeft.children) { boneStackLeft.Push(childBone); } foreach (KNNBone childBone in topBoneRight.children) { boneStackRight.Push(childBone); } if (topBoneLeft.name == topBoneRight.name && topBoneFinal.name == topBoneLeft.name) { allBones.Add(Tuple.Create(topBoneLeft, topBoneRight, topBoneFinal)); } } rigTransforms = new List <Tuple <Transform, Transform, Quaternion> >(); Stack <KNNBone> boneStack = new Stack <KNNBone>(); Stack <Transform> transformStack = new Stack <Transform>(); boneStack.Push(this.finalSkeletonBones); transformStack.Push(rig); while (boneStack.Count > 0) { KNNBone topBone = boneStack.Pop(); Transform topTransform = null; if (transformStack.Count > 0) { topTransform = transformStack.Pop(); } foreach (KNNBone childBone in topBone.children) { boneStack.Push(childBone); if (topTransform != null) { Transform childTransform = topTransform.Find(childBone.name); if (childTransform != null) { transformStack.Push(childTransform); } } } if (topTransform != null && topTransform.name == topBone.name) { rigTransforms.Add(Tuple.Create(topBone.transform, topTransform, topTransform.rotation)); } } Transform targets = null; Transform lHandTmpTarget = null; Transform rHandTmpTarget = null; Transform headTmpTarget = null; if (this.transform.Find("Targets") == null) { targets = new GameObject("Targets").transform; targets.transform.parent = this.transform; } else { targets = this.transform.Find("Targets"); lHandTmpTarget = targets.Find("LHandTarget"); rHandTmpTarget = targets.Find("RHandTarget"); headTmpTarget = targets.Find("HeadTarget"); } if (lHandTarget == null) { if (lHandTmpTarget == null) { lHandTmpTarget = new GameObject("LHandTarget").transform; lHandTmpTarget.transform.parent = targets.transform; } lHandTarget = lHandTmpTarget; } if (rHandTarget == null) { if (rHandTmpTarget == null) { rHandTmpTarget = new GameObject("RHandTarget").transform; rHandTmpTarget.transform.parent = targets.transform; } rHandTarget = rHandTmpTarget; } if (headTarget == null) { if (headTmpTarget == null) { headTmpTarget = new GameObject("HeadTarget").transform; headTmpTarget.transform.parent = targets.transform; } headTarget = headTmpTarget; } BoneRenderer leftBoneRenderer = lHandSkeleton.GetComponent <BoneRenderer>(); BoneRenderer rightBoneRenderer = rHandSkeleton.GetComponent <BoneRenderer>(); BoneRenderer finalBoneRenderer = finalSkeletonBones.GetComponent <BoneRenderer>(); if (leftBoneRenderer != null) { leftBoneRenderer.boneColor = Color.green; } if (rightBoneRenderer != null) { rightBoneRenderer.boneColor = Color.white; } }