private void SetBoneRenderer()
{
BoneRenderer myBoneRenderer;
if (this.gameObject.GetComponent <BoneRenderer>() == null)
{
myBoneRenderer = this.gameObject.AddComponent <BoneRenderer>();
}
else
{
myBoneRenderer = this.gameObject.GetComponent <BoneRenderer>();
}
boneTransforms = new List <Transform>();
Stack <KNNBone> boneStack = new Stack <KNNBone>();
boneStack.Push(this.rootBone);
while (boneStack.Count > 0)
{
KNNBone topBone = boneStack.Pop();
foreach (KNNBone childBone in topBone.children)
{
boneStack.Push(childBone);
}
boneTransforms.Add(topBone.transform);
}
myBoneRenderer.transforms = boneTransforms.ToArray();
}
public void SetKNNSkeleton(KNNBone rootBone, bool ignoreRotation, KDTree <float, int> lHandQueryTree, KDTree <float, int> rHandQueryTree, int featureVectorLength)
{
this.ignoreRotation = ignoreRotation;
this.rootBone = rootBone;
this.lHandQueryTree = lHandQueryTree;
this.rHandQueryTree = rHandQueryTree;
this.featureVectorLength = featureVectorLength;
SetBoneRenderer();
}
private string ComposeString()
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendLine("#bones timesteps framesPerTimeStep ignoreRotation");
List <KNNBone> boneList = new List <KNNBone>();
Stack <KNNBone> boneStack = new Stack <KNNBone>();
boneStack.Push(this.rootBone);
while (boneStack.Count > 0)
{
KNNBone top = boneStack.Pop();
foreach (KNNBone child in top.children)
{
boneStack.Push(child);
}
boneList.Add(top);
}
stringBuilder.AppendLine(boneList.Count.ToString() + " " + this.rootBone.rotations.Count.ToString() + " " + this.lHandQueryTree.Navigator.Point.Length.ToString() + " " + ignoreRotation.ToString());
stringBuilder.AppendLine("#name id parentid offsetx offsety offsetz rotations(quaternions * timesteps)");
foreach (KNNBone currBone in boneList)
{
string currLine = currBone.name + " ";
currLine += boneList.IndexOf(currBone) + " ";
currLine += boneList.IndexOf(currBone.parent) + " ";
currLine += currBone.offset.x.ToString() + " " + currBone.offset.y.ToString() + " " + currBone.offset.z.ToString();
foreach (Quaternion currRot in currBone.rotations)
{
currLine += " " + currRot.x + " " + currRot.y + " " + currRot.z + " " + currRot.w;
}
stringBuilder.AppendLine(currLine);
}
string nextLine = this.KdTreeToString(this.lHandQueryTree);
stringBuilder.AppendLine(nextLine);
nextLine = this.KdTreeToString(this.rHandQueryTree);
stringBuilder.AppendLine(nextLine);
return(stringBuilder.ToString());
}
public void createKNNRig(string headBoneName, string rHandBoneName, string lHandBoneName, float slidingWindowSizeInMS, float slidingWindowOffsetInMS, float pollingRate, bool ignoreRotation, bool bothHandsInFeatureVec, string outputPath)
{
var lHandTreeNodes = new List <int>();
var lHandTreePoints = new List <float[]>();
var rHandTreeNodes = new List <int>();
var rHandTreePoints = new List <float[]>();
GameObject rootBoneObj = new GameObject();
KNNBone rootBone = rootBoneObj.AddComponent <KNNBone>();
Debug.Assert(files != null && files.Count > 0);
loadFromFile(files[0]);
setToFrame(0, true, true, true);
rootBone.initKNNBone(root.transform);
foreach (string file in files)
{
loadFromFile(file);
FillKNNTrees(rootBone, ignoreRotation, bothHandsInFeatureVec, headBoneName, rHandBoneName, lHandBoneName, slidingWindowSizeInMS, slidingWindowOffsetInMS, pollingRate, lHandTreeNodes, rHandTreeNodes, lHandTreePoints, rHandTreePoints);
}
float slidingWindowSize = slidingWindowSizeInMS / 1000.0f;
float targetFrameTime = 1.0f / pollingRate;
int framesPerWindow = (int)((slidingWindowSize) / targetFrameTime);
int featureVecLength = 9 * framesPerWindow;
var rHandTree = new KDTree <float, int>(featureVecLength, rHandTreePoints.ToArray(), rHandTreeNodes.ToArray(), Metrics.WeightedL2Norm);
var lHandTree = new KDTree <float, int>(featureVecLength, lHandTreePoints.ToArray(), lHandTreeNodes.ToArray(), Metrics.WeightedL2Norm);
GameObject kNNSkeleton_ = new GameObject("KNN-Skeleton");
KNNSkeleton finalKNNSkeleton = kNNSkeleton_.AddComponent <KNNSkeleton>();
finalKNNSkeleton.SetKNNSkeleton(rootBone, ignoreRotation, lHandTree, rHandTree, featureVecLength);
finalKNNSkeleton.Save(outputPath);
DestroyImmediate(kNNSkeleton_);
DestroyImmediate(rootBoneObj);
GameObject kNNRig = new GameObject("KNN-Rig");
KNNRig kNNRigComponent = kNNRig.AddComponent <KNNRig>();
kNNRigComponent.skeletonPath = outputPath;
}
public void initKNNBone(Transform rootTransform)
{
this.name = rootTransform.name;
this.transform.localPosition = rootTransform.localPosition;
this.transform.localRotation = rootTransform.localRotation;
this.offset = this.transform.localPosition;
this.children = new List <KNNBone>();
this.rotations = new List <Quaternion>();
for (int childIdx = 0; childIdx < rootTransform.childCount; childIdx++)
{
GameObject childObj = new GameObject();
KNNBone childBone = (KNNBone)childObj.AddComponent(typeof(KNNBone));
childBone.parent = this;
childBone.transform.parent = this.transform;
childBone.initKNNBone(rootTransform.GetChild(childIdx));
children.Add(childBone);
}
}
public void Parse(string file)
{
StreamReader reader = new StreamReader(file);
string currLine = reader.ReadLine(); // skip comment
currLine = reader.ReadLine();
string[] words = currLine.Split(' ');
int nrOfBones = int.Parse(words[0]);
int timeSteps = int.Parse(words[1]);
this.featureVectorLength = int.Parse(words[2]);
this.ignoreRotation = bool.Parse(words[3]);
currLine = reader.ReadLine(); // skip comment
GameObject rootBone = new GameObject();
this.rootBone = rootBone.AddComponent <KNNBone>();
this.rootBone.initKNNBoneFromFile(file);
this.rootBone.transform.parent = this.transform;
for (int currBoneIdx = 0; currBoneIdx < nrOfBones; currBoneIdx++)
{
currLine = reader.ReadLine(); // ignore since it's done in the initBoneFromFile
}
currLine = reader.ReadLine();
if (this.lHandQueryTree == null || this.lHandQueryTree.Count == 0)
{
this.lHandQueryTree = ParseKDTree(currLine, featureVectorLength, timeSteps);
}
currLine = reader.ReadLine();
if (this.rHandQueryTree == null || this.rHandQueryTree.Count == 0)
{
this.rHandQueryTree = ParseKDTree(currLine, featureVectorLength, timeSteps);
}
reader.Close();
}
private void FillKNNTrees(KNNBone rootBone, bool ignoreRotation, bool bothHandsInFeatureVec, string headBoneName, string rHandBoneName, string lHandBoneName, float slidingWindowSizeInMS, float slidingWindowOffsetInMS, float pollingRate, List <int> lHandTreeNodes, List <int> rHandTreeNodes, List <float[]> lHandTreePoints, List <float[]> rHandTreePoints)
{
float sourceFrameTime = bvhParser.frameTime;
float targetFrameTime = 1.0f / pollingRate;
float totalTime = bvhParser.frames * sourceFrameTime;
int totalNewFrames = (int)(totalTime / targetFrameTime);
float slidingWindowOffset = slidingWindowOffsetInMS / 1000.0f;
float slidingWindowSize = slidingWindowSizeInMS / 1000.0f;
int totalWindows = (int)((totalTime - (slidingWindowSize)) / (slidingWindowOffset));
int framesPerWindow = (int)((slidingWindowSize) / targetFrameTime);
int nrOfFeatures = bothHandsInFeatureVec ? 18 : 9;
int featureVecLength = nrOfFeatures * framesPerWindow;
setToFrame(0, true, true, true);
for (int currentWindow = 0; currentWindow < totalWindows; currentWindow++)
{
float[] lHandPoints = new float[featureVecLength];
float[] rHandPoints = new float[featureVecLength];
for (int currFrame = 0; currFrame < framesPerWindow; currFrame++)
{
float startTime = slidingWindowOffset * (float)currentWindow;
float sourceFrame = getSourceFrameFromTargetTime(sourceFrameTime, startTime + (float)currFrame * targetFrameTime);
setToFrame(sourceFrame, true, true, true);
Vector3 rHandPosition = root.Find(rHandBoneName).position;
Vector3 lHandPosition = root.Find(lHandBoneName).position;
if (sourceFrame >= 1.0f)
{
setToFrame(sourceFrame - 1.0f, true, true, true);
}
Vector3 prevRHandPosition = root.Find(rHandBoneName).position;
Vector3 prevLHandPosition = root.Find(lHandBoneName).position;
if (sourceFrame >= 2.0f)
{
setToFrame(sourceFrame - 2.0f, true, true, true);
}
Vector3 prevPrevRHandPosition = root.Find(rHandBoneName).position;
Vector3 prevPrevLHandPosition = root.Find(lHandBoneName).position;
Vector3 lHandVelocity = (lHandPosition - prevLHandPosition) / sourceFrameTime;
Vector3 rHandVelocity = (rHandPosition - prevRHandPosition) / sourceFrameTime;
Vector3 prevLHandVelocity = (prevLHandPosition - prevPrevLHandPosition) / sourceFrameTime;
Vector3 prevRHandVelocity = (prevRHandPosition - prevPrevRHandPosition) / sourceFrameTime;
Vector3 lHandAcceleration = (lHandVelocity - prevLHandVelocity) / sourceFrameTime;
Vector3 rHandAcceleration = (rHandVelocity - prevRHandVelocity) / sourceFrameTime;
if (ignoreRotation)
{
SphericalCoords rHandSpPosition = SphericalCoords.CartesianToSpherical(rHandPosition);
SphericalCoords lHandSpPosition = SphericalCoords.CartesianToSpherical(lHandPosition);
float rHandAngle = rHandSpPosition.theta * Mathf.Rad2Deg;
float lHandAngle = lHandSpPosition.theta * Mathf.Rad2Deg;
//rotate velocity and acceleration angle around "theta"
lHandVelocity = Quaternion.AngleAxis(lHandAngle, Vector3.up) * lHandVelocity;
rHandVelocity = Quaternion.AngleAxis(rHandAngle, Vector3.up) * rHandVelocity;
lHandAcceleration = Quaternion.AngleAxis(lHandAngle, Vector3.up) * lHandAcceleration;
rHandAcceleration = Quaternion.AngleAxis(rHandAngle, Vector3.up) * rHandAcceleration;
rHandSpPosition.theta = 0;
lHandSpPosition.theta = 0;
rHandPosition = rHandSpPosition.ToCartesian();
lHandPosition = lHandSpPosition.ToCartesian();
}
int pointOffset = currFrame * nrOfFeatures;
lHandPoints[pointOffset] = lHandPosition.x;
lHandPoints[pointOffset + 1] = lHandPosition.y;
lHandPoints[pointOffset + 2] = lHandPosition.z;
lHandPoints[pointOffset + 3] = lHandVelocity.x;
lHandPoints[pointOffset + 4] = lHandVelocity.y;
lHandPoints[pointOffset + 5] = lHandVelocity.z;
lHandPoints[pointOffset + 6] = lHandAcceleration.x;
lHandPoints[pointOffset + 7] = lHandAcceleration.y;
lHandPoints[pointOffset + 8] = lHandAcceleration.z;
rHandPoints[pointOffset] = rHandPosition.x;
rHandPoints[pointOffset + 1] = rHandPosition.y;
rHandPoints[pointOffset + 2] = rHandPosition.z;
rHandPoints[pointOffset + 3] = rHandVelocity.x;
rHandPoints[pointOffset + 4] = rHandVelocity.y;
rHandPoints[pointOffset + 5] = rHandVelocity.z;
rHandPoints[pointOffset + 6] = rHandAcceleration.x;
rHandPoints[pointOffset + 7] = rHandAcceleration.y;
rHandPoints[pointOffset + 8] = rHandAcceleration.z;
if (bothHandsInFeatureVec)
{
lHandPoints[pointOffset + 9] = rHandPosition.x;
lHandPoints[pointOffset + 10] = rHandPosition.y;
lHandPoints[pointOffset + 11] = rHandPosition.z;
lHandPoints[pointOffset + 12] = rHandVelocity.x;
lHandPoints[pointOffset + 13] = rHandVelocity.y;
lHandPoints[pointOffset + 14] = rHandVelocity.z;
lHandPoints[pointOffset + 15] = rHandAcceleration.x;
lHandPoints[pointOffset + 16] = rHandAcceleration.y;
lHandPoints[pointOffset + 17] = rHandAcceleration.z;
rHandPoints[pointOffset + 9] = lHandPosition.x;
rHandPoints[pointOffset + 10] = lHandPosition.y;
rHandPoints[pointOffset + 11] = lHandPosition.z;
rHandPoints[pointOffset + 12] = lHandVelocity.x;
rHandPoints[pointOffset + 13] = lHandVelocity.y;
rHandPoints[pointOffset + 14] = lHandVelocity.z;
rHandPoints[pointOffset + 15] = lHandAcceleration.x;
rHandPoints[pointOffset + 16] = lHandAcceleration.y;
rHandPoints[pointOffset + 17] = lHandAcceleration.z;
}
}
int lastPositionSourceIndex = (int)((currentWindow * slidingWindowOffset + slidingWindowSize) / sourceFrameTime);
setToFrame(lastPositionSourceIndex, true, true, true);
rootBone.AddRotationsFromTransform(root);
lHandTreePoints.Add(lHandPoints); lHandTreeNodes.Add(lHandTreeNodes.Count);
rHandTreePoints.Add(rHandPoints); rHandTreeNodes.Add(rHandTreeNodes.Count);
}
}
public void InitRig()
{
leftArmFeatureVec = new LinkedList <FeatureItem>();
rightArmFeatureVec = new LinkedList <FeatureItem>();
if (rHandSkeleton == null || rHandSkeleton.rootBone == null || rHandSkeleton.lHandQueryTree == null || rHandSkeleton.rHandQueryTree == null || rHandSkeleton.lHandQueryTree.Count == 0 || rHandSkeleton.rHandQueryTree.Count == 0)
{
if (rHandSkeleton == null)
{
GameObject kNNSkeleton_ = new GameObject("Right-KNN-Skeleton");
this.rHandSkeleton = kNNSkeleton_.AddComponent <KNNSkeleton>();
}
this.rHandSkeleton.SetKNNSkeleton(skeletonPath);
this.rHandSkeleton.transform.parent = this.transform;
this.rHandSkeleton.rootBone.transform.parent = this.rHandSkeleton.transform;
}
if (lHandSkeleton == null || lHandSkeleton.rootBone == null || lHandSkeleton.lHandQueryTree == null || lHandSkeleton.rHandQueryTree == null || lHandSkeleton.lHandQueryTree.Count == 0 || lHandSkeleton.rHandQueryTree.Count == 0)
{
if (lHandSkeleton == null)
{
GameObject kNNSkeleton_ = new GameObject("Left-KNN-Skeleton");
this.lHandSkeleton = kNNSkeleton_.AddComponent <KNNSkeleton>();
}
this.lHandSkeleton.SetKNNSkeleton(skeletonPath);
this.lHandSkeleton.transform.parent = this.transform;
this.lHandSkeleton.rootBone.transform.parent = this.lHandSkeleton.transform;
}
if (finalSkeletonBones == null)
{
GameObject kNNSkeleton_ = new GameObject("final-KNN-Skeleton-Bones");
GameObject rootBone = new GameObject("final-KNN-Skeleton-Bones");
this.finalSkeletonBones = rootBone.AddComponent <KNNBone>();
this.finalSkeletonBones.initKNNBoneFromFile(skeletonPath);
this.finalSkeletonBones.transform.parent = kNNSkeleton_.transform;
kNNSkeleton_.transform.parent = this.transform;
}
Stack <KNNBone> boneStackFinal = new Stack <KNNBone>();
Stack <KNNBone> boneStackLeft = new Stack <KNNBone>();
Stack <KNNBone> boneStackRight = new Stack <KNNBone>();
boneStackFinal.Push(this.finalSkeletonBones);
boneStackLeft.Push(this.lHandSkeleton.rootBone);
boneStackRight.Push(this.rHandSkeleton.rootBone);
allBones = new List <Tuple <KNNBone, KNNBone, KNNBone> >();
//assuming they all have the same hierarchy
while (boneStackFinal.Count > 0)
{
KNNBone topBoneFinal = boneStackFinal.Pop();
KNNBone topBoneLeft = boneStackLeft.Pop();
KNNBone topBoneRight = boneStackRight.Pop();
foreach (KNNBone childBone in topBoneFinal.children)
{
boneStackFinal.Push(childBone);
}
foreach (KNNBone childBone in topBoneLeft.children)
{
boneStackLeft.Push(childBone);
}
foreach (KNNBone childBone in topBoneRight.children)
{
boneStackRight.Push(childBone);
}
if (topBoneLeft.name == topBoneRight.name && topBoneFinal.name == topBoneLeft.name)
{
allBones.Add(Tuple.Create(topBoneLeft, topBoneRight, topBoneFinal));
}
}
rigTransforms = new List <Tuple <Transform, Transform, Quaternion> >();
Stack <KNNBone> boneStack = new Stack <KNNBone>();
Stack <Transform> transformStack = new Stack <Transform>();
boneStack.Push(this.finalSkeletonBones);
transformStack.Push(rig);
while (boneStack.Count > 0)
{
KNNBone topBone = boneStack.Pop();
Transform topTransform = null;
if (transformStack.Count > 0)
{
topTransform = transformStack.Pop();
}
foreach (KNNBone childBone in topBone.children)
{
boneStack.Push(childBone);
if (topTransform != null)
{
Transform childTransform = topTransform.Find(childBone.name);
if (childTransform != null)
{
transformStack.Push(childTransform);
}
}
}
if (topTransform != null && topTransform.name == topBone.name)
{
rigTransforms.Add(Tuple.Create(topBone.transform, topTransform, topTransform.rotation));
}
}
Transform targets = null;
Transform lHandTmpTarget = null;
Transform rHandTmpTarget = null;
Transform headTmpTarget = null;
if (this.transform.Find("Targets") == null)
{
targets = new GameObject("Targets").transform;
targets.transform.parent = this.transform;
}
else
{
targets = this.transform.Find("Targets");
lHandTmpTarget = targets.Find("LHandTarget");
rHandTmpTarget = targets.Find("RHandTarget");
headTmpTarget = targets.Find("HeadTarget");
}
if (lHandTarget == null)
{
if (lHandTmpTarget == null)
{
lHandTmpTarget = new GameObject("LHandTarget").transform;
lHandTmpTarget.transform.parent = targets.transform;
}
lHandTarget = lHandTmpTarget;
}
if (rHandTarget == null)
{
if (rHandTmpTarget == null)
{
rHandTmpTarget = new GameObject("RHandTarget").transform;
rHandTmpTarget.transform.parent = targets.transform;
}
rHandTarget = rHandTmpTarget;
}
if (headTarget == null)
{
if (headTmpTarget == null)
{
headTmpTarget = new GameObject("HeadTarget").transform;
headTmpTarget.transform.parent = targets.transform;
}
headTarget = headTmpTarget;
}
BoneRenderer leftBoneRenderer = lHandSkeleton.GetComponent <BoneRenderer>();
BoneRenderer rightBoneRenderer = rHandSkeleton.GetComponent <BoneRenderer>();
BoneRenderer finalBoneRenderer = finalSkeletonBones.GetComponent <BoneRenderer>();
if (leftBoneRenderer != null)
{
leftBoneRenderer.boneColor = Color.green;
}
if (rightBoneRenderer != null)
{
rightBoneRenderer.boneColor = Color.white;
}
}
