public static Tuple <Dictionary <int, List <Dictionary <JointType, CvPoint3D64f> > >, Dictionary <int, List <DateTime> > > ExportFromProject(FrameSequence frameseq, int startIndex, int endIndex)
{
TimeSpan period = new TimeSpan((long)(10000000 / frameseq.frameRate));
List <DateTime> timestamps = new List <DateTime>();
for (DateTime time = frameseq.Frames[startIndex].Time; time < frameseq.Frames[endIndex].Time; time += period)
{
timestamps.Add(time);
}
HashSet <Tuple <int, JointType> > uniqueUserJoint = new HashSet <Tuple <int, JointType> >();
List <Frame> frames = frameseq.Slice(startIndex, endIndex);
foreach (Frame frame in frames)
{
for (int i = 0; i < frameseq.recordNum; i++)
{
foreach (SerializableBody body in frame.GetBodyList(i))
{
foreach (JointType jointType in Utility.GetValidJoints(body.Joints).Keys)
{
// TrackingIdと勝手に作ったuser idの対応辞書が必要. 以下の処理はとりあえず
Tuple <int, JointType> userJoint = new Tuple <int, JointType>(body.integratedId, jointType);
if (!uniqueUserJoint.Contains(userJoint))
{
uniqueUserJoint.Add(userJoint);
}
}
}
}
}
List <Tuple <int, JointType> > userJointPairs = (
from pair in uniqueUserJoint
orderby pair.Item1, pair.Item2
select pair
).ToList();
Dictionary <int, List <Dictionary <JointType, CvPoint3D64f> > > allBodies = new Dictionary <int, List <Dictionary <JointType, CvPoint3D64f> > >();
Dictionary <int, List <DateTime> > allTimes = new Dictionary <int, List <DateTime> >();
SkeletonInterpolator skeletonInterpolator = new SkeletonInterpolator(0.5, true);
foreach (int user in userJointPairs.Select(p => p.Item1).Distinct())
{
skeletonInterpolator.pivot = new Dictionary <JointType, CvPoint3D64f>();
// pivot初期化処理
Dictionary <JointType, Joint>[] firstJoints = new Dictionary <JointType, Joint> [frameseq.recordNum];
for (int no = 0; no < frameseq.recordNum; no++)
{
var bodies = frameseq.GetNextData(no, timestamps[0]).bodies;
if (bodies.Count() > 0)
{
SerializableBody body = bodies.Where(b => b.integratedId == user).FirstOrDefault();
if (body != null)
{
var validJoints = Utility.GetValidJoints(body.Joints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D());
if (skeletonInterpolator.pivot.Count < validJoints.Count)
{
skeletonInterpolator.pivot = validJoints;
}
}
}
}
List <Dictionary <JointType, CvPoint3D64f> > jointsSeq = new List <Dictionary <JointType, CvPoint3D64f> >();
List <DateTime> times = new List <DateTime>();
foreach (DateTime time in timestamps)
{
Dictionary <JointType, CvPoint3D64f> joints = skeletonInterpolator.IntegrateSkeleton(time, user, frameseq);
// jointsがnullの場合にはダミーデータを突っ込んで長さを稼ぐ
if (joints == null)
{
jointsSeq.Add(new Dictionary <JointType, CvPoint3D64f>()
{
{ JointType.SpineBase, new CvPoint3D64f(float.MaxValue, float.MaxValue, float.MaxValue) }
});
}
else
{
jointsSeq.Add(joints);
}
times.Add(time);
}
allBodies[user] = jointsSeq;
allTimes[user] = times;
}
return(Tuple.Create(allBodies, allTimes));
}
public static Tuple<Dictionary<int, List<Dictionary<JointType, CvPoint3D64f>>>, Dictionary<int, List<DateTime>>> ExportFromProject(FrameSequence frameseq, int startIndex, int endIndex)
{
TimeSpan period = new TimeSpan((long)(10000000 / frameseq.frameRate));
List<DateTime> timestamps = new List<DateTime>();
for (DateTime time = frameseq.Frames[startIndex].Time; time < frameseq.Frames[endIndex].Time; time += period)
{
timestamps.Add(time);
}
HashSet<Tuple<int, JointType>> uniqueUserJoint = new HashSet<Tuple<int, JointType>>();
List<Frame> frames = frameseq.Slice(startIndex, endIndex);
foreach(Frame frame in frames){
for (int i = 0; i < frameseq.recordNum; i++)
{
foreach (SerializableBody body in frame.GetBodyList(i))
{
foreach (JointType jointType in Utility.GetValidJoints(body.Joints).Keys)
{
// TrackingIdと勝手に作ったuser idの対応辞書が必要. 以下の処理はとりあえず
Tuple<int, JointType> userJoint = new Tuple<int, JointType>(body.integratedId, jointType);
if (!uniqueUserJoint.Contains(userJoint))
{
uniqueUserJoint.Add(userJoint);
}
}
}
}
}
List<Tuple<int, JointType>> userJointPairs = (
from pair in uniqueUserJoint
orderby pair.Item1, pair.Item2
select pair
).ToList();
Dictionary<int, List<Dictionary<JointType, CvPoint3D64f>>> allBodies = new Dictionary<int, List<Dictionary<JointType, CvPoint3D64f>>>();
Dictionary<int, List<DateTime>> allTimes = new Dictionary<int, List<DateTime>>();
SkeletonInterpolator skeletonInterpolator = new SkeletonInterpolator(0.5, true);
foreach (int user in userJointPairs.Select(p => p.Item1).Distinct())
{
skeletonInterpolator.pivot = new Dictionary<JointType, CvPoint3D64f>();
// pivot初期化処理
Dictionary<JointType, Joint>[] firstJoints = new Dictionary<JointType,Joint>[frameseq.recordNum];
for (int no = 0; no < frameseq.recordNum; no++)
{
var bodies = frameseq.GetNextData(no, timestamps[0]).bodies;
if (bodies.Count() > 0)
{
SerializableBody body = bodies.Where(b => b.integratedId == user).FirstOrDefault();
if (body != null)
{
var validJoints = Utility.GetValidJoints(body.Joints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D());
if (skeletonInterpolator.pivot.Count < validJoints.Count)
{
skeletonInterpolator.pivot = validJoints;
}
}
}
}
List<Dictionary<JointType, CvPoint3D64f>> jointsSeq = new List<Dictionary<JointType, CvPoint3D64f>>();
List<DateTime> times = new List<DateTime>();
foreach(DateTime time in timestamps)
{
Dictionary<JointType, CvPoint3D64f> joints = skeletonInterpolator.IntegrateSkeleton(time, user, frameseq);
// jointsがnullの場合にはダミーデータを突っ込んで長さを稼ぐ
if (joints == null)
{
jointsSeq.Add(new Dictionary<JointType, CvPoint3D64f>() { { JointType.SpineBase, new CvPoint3D64f(float.MaxValue, float.MaxValue, float.MaxValue) } });
}
else
{
jointsSeq.Add(joints);
}
times.Add(time);
}
allBodies[user] = jointsSeq;
allTimes[user] = times;
}
return Tuple.Create(allBodies, allTimes);
}
public Dictionary <JointType, CvPoint3D64f> IntegrateSkeleton(DateTime time, int userInt, FrameSequence frameSeq)
{
List <CvMat> ToWorldConversions = frameSeq.ToWorldConversions;
List <CameraIntrinsics> cameraInfo = frameSeq.CameraInfo;
List <UserSegmentation> segm = frameSeq.Segmentations;
Dictionary <JointType, CvPoint3D64f>[] jointsArr = new Dictionary <JointType, CvPoint3D64f> [frameSeq.recordNum];
Dictionary <JointType, CvPoint3D64f>[] pivotCandidate = new Dictionary <JointType, CvPoint3D64f> [frameSeq.recordNum];
double[] reliabilityArr = new double[frameSeq.recordNum];
double[] weightArr = new double[frameSeq.recordNum];
for (int recordNo = 0; recordNo < frameSeq.recordNum; recordNo++)
{
MotionData prevData = frameSeq.GetPrevData(recordNo, time);
MotionData nextData = frameSeq.GetNextData(recordNo, time);
if (prevData == null || nextData == null || prevData.bodies.Length * nextData.bodies.Length == 0 || !prevData.isValid || !nextData.isValid)
{
jointsArr[recordNo] = null;
reliabilityArr[recordNo] = 0;
weightArr[recordNo] = 0;
continue;
}
SerializableBody prevBody = prevData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault();
SerializableBody nextBody = nextData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault();
if (prevBody == null || nextBody == null || prevBody.Equals(default(SerializableBody)) || nextBody.Equals(default(SerializableBody)))
{
jointsArr[recordNo] = null;
reliabilityArr[recordNo] = 0;
weightArr[recordNo] = 0;
continue;
}
// 統計情報によるフィルタリング
Dictionary <JointType, Joint> prevJoints;
Dictionary <JointType, Joint> nextJoints;
if (prevBody.Joints == null || nextBody.Joints == null)
{
continue;
}
// pivotが設定されてるとき、つまり、本番統合のとき
if (pivot != null)
{
// mirror矯正
prevJoints = this.CorrectMirrorJoint(prevBody.Joints, ToWorldConversions[recordNo]);
nextJoints = this.CorrectMirrorJoint(nextBody.Joints, ToWorldConversions[recordNo]);
// next pivot候補つめつめ
pivotCandidate[recordNo] = Utility.GetValidJoints(nextJoints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D());
}
else
{
prevJoints = prevBody.Joints;
nextJoints = nextBody.Joints;
}
// 統計情報があるとき
if (frameSeq.BodyStat != null)
{
prevJoints = frameSeq.BodyStat.FilterBonesByStatistics(prevJoints);
nextJoints = frameSeq.BodyStat.FilterBonesByStatistics(nextJoints);
}
else
{
prevJoints = Utility.GetValidJoints(prevBody.Joints);
nextJoints = Utility.GetValidJoints(nextBody.Joints);
}
jointsArr[recordNo] = this.InterpolateSkeleton(prevData, nextData, prevJoints, nextJoints, time, ToWorldConversions[recordNo]);
reliabilityArr[recordNo] = this.GetSkeletonReliability(prevData, nextData, prevJoints, nextJoints, time, cameraInfo[recordNo]);
weightArr[recordNo] = this.GetVarianceWeight(prevData, nextData, prevJoints, nextJoints, time);
}
// pivot更新
pivot = new Dictionary <JointType, CvPoint3D64f>();
foreach (var candidate in pivotCandidate)
{
if (candidate != null && pivot.Count <= candidate.Count)
{
pivot = candidate;
}
}
double maxWeight = weightArr.Max();
double[] modifiedReliabilityList = weightArr.Select(w => Math.Max(0, (w / maxWeight) - _weightBase)).Zip(reliabilityArr, (a, b) => a * b).ToArray();
var keys = jointsArr.Where(j => j != null).SelectMany(j => j.Keys).Distinct().ToList();
if (maxWeight == 0)
{
return(null);
}
return(CalcEx.LinearMedianSkeletons(jointsArr, modifiedReliabilityList));
}
public Dictionary<JointType, CvPoint3D64f> IntegrateSkeleton(DateTime time, int userInt, FrameSequence frameSeq)
{
List<CvMat> ToWorldConversions = frameSeq.ToWorldConversions;
List<CameraIntrinsics> cameraInfo = frameSeq.CameraInfo;
List<UserSegmentation> segm = frameSeq.Segmentations;
Dictionary<JointType, CvPoint3D64f>[] jointsArr = new Dictionary<JointType, CvPoint3D64f>[frameSeq.recordNum];
Dictionary<JointType, CvPoint3D64f>[] pivotCandidate = new Dictionary<JointType, CvPoint3D64f>[frameSeq.recordNum];
double[] reliabilityArr = new double[frameSeq.recordNum];
double[] weightArr = new double[frameSeq.recordNum];
for (int recordNo = 0; recordNo < frameSeq.recordNum; recordNo++)
{
MotionData prevData = frameSeq.GetPrevData(recordNo, time);
MotionData nextData = frameSeq.GetNextData(recordNo, time);
if (prevData == null || nextData == null || prevData.bodies.Length * nextData.bodies.Length == 0 || !prevData.isValid || !nextData.isValid)
{
jointsArr[recordNo] = null;
reliabilityArr[recordNo] = 0;
weightArr[recordNo] = 0;
continue;
}
SerializableBody prevBody = prevData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault();
SerializableBody nextBody = nextData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault();
if (prevBody == null || nextBody == null || prevBody.Equals(default(SerializableBody)) || nextBody.Equals(default(SerializableBody)))
{
jointsArr[recordNo] = null;
reliabilityArr[recordNo] = 0;
weightArr[recordNo] = 0;
continue;
}
// 統計情報によるフィルタリング
Dictionary<JointType, Joint> prevJoints;
Dictionary<JointType, Joint> nextJoints;
if (prevBody.Joints == null || nextBody.Joints == null)
continue;
// pivotが設定されてるとき、つまり、本番統合のとき
if (pivot != null)
{
// mirror矯正
prevJoints = this.CorrectMirrorJoint(prevBody.Joints, ToWorldConversions[recordNo]);
nextJoints = this.CorrectMirrorJoint(nextBody.Joints, ToWorldConversions[recordNo]);
// next pivot候補つめつめ
pivotCandidate[recordNo] = Utility.GetValidJoints(nextJoints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D());
}
else
{
prevJoints = prevBody.Joints;
nextJoints = nextBody.Joints;
}
// 統計情報があるとき
if (frameSeq.BodyStat != null)
{
prevJoints = frameSeq.BodyStat.FilterBonesByStatistics(prevJoints);
nextJoints = frameSeq.BodyStat.FilterBonesByStatistics(nextJoints);
}
else
{
prevJoints = Utility.GetValidJoints(prevBody.Joints);
nextJoints = Utility.GetValidJoints(nextBody.Joints);
}
jointsArr[recordNo] = this.InterpolateSkeleton(prevData, nextData, prevJoints, nextJoints, time, ToWorldConversions[recordNo]);
reliabilityArr[recordNo] = this.GetSkeletonReliability(prevData, nextData, prevJoints, nextJoints, time, cameraInfo[recordNo]);
weightArr[recordNo] = this.GetVarianceWeight(prevData, nextData, prevJoints, nextJoints, time);
}
// pivot更新
pivot = new Dictionary<JointType, CvPoint3D64f>();
foreach (var candidate in pivotCandidate)
{
if (candidate != null && pivot.Count <= candidate.Count)
{
pivot = candidate;
}
}
double maxWeight = weightArr.Max();
double[] modifiedReliabilityList = weightArr.Select(w => Math.Max(0, (w / maxWeight) - _weightBase)).Zip(reliabilityArr, (a, b) => a * b).ToArray();
var keys = jointsArr.Where(j => j != null).SelectMany(j => j.Keys).Distinct().ToList();
if (maxWeight == 0)
return null;
return CalcEx.LinearMedianSkeletons(jointsArr, modifiedReliabilityList);
}
