public static Tuple <Dictionary <int, List <Dictionary <JointType, CvPoint3D64f> > >, Dictionary <int, List <DateTime> > > ExportFromProject(FrameSequence frameseq, int startIndex, int endIndex) { TimeSpan period = new TimeSpan((long)(10000000 / frameseq.frameRate)); List <DateTime> timestamps = new List <DateTime>(); for (DateTime time = frameseq.Frames[startIndex].Time; time < frameseq.Frames[endIndex].Time; time += period) { timestamps.Add(time); } HashSet <Tuple <int, JointType> > uniqueUserJoint = new HashSet <Tuple <int, JointType> >(); List <Frame> frames = frameseq.Slice(startIndex, endIndex); foreach (Frame frame in frames) { for (int i = 0; i < frameseq.recordNum; i++) { foreach (SerializableBody body in frame.GetBodyList(i)) { foreach (JointType jointType in Utility.GetValidJoints(body.Joints).Keys) { // TrackingIdと勝手に作ったuser idの対応辞書が必要. 以下の処理はとりあえず Tuple <int, JointType> userJoint = new Tuple <int, JointType>(body.integratedId, jointType); if (!uniqueUserJoint.Contains(userJoint)) { uniqueUserJoint.Add(userJoint); } } } } } List <Tuple <int, JointType> > userJointPairs = ( from pair in uniqueUserJoint orderby pair.Item1, pair.Item2 select pair ).ToList(); Dictionary <int, List <Dictionary <JointType, CvPoint3D64f> > > allBodies = new Dictionary <int, List <Dictionary <JointType, CvPoint3D64f> > >(); Dictionary <int, List <DateTime> > allTimes = new Dictionary <int, List <DateTime> >(); SkeletonInterpolator skeletonInterpolator = new SkeletonInterpolator(0.5, true); foreach (int user in userJointPairs.Select(p => p.Item1).Distinct()) { skeletonInterpolator.pivot = new Dictionary <JointType, CvPoint3D64f>(); // pivot初期化処理 Dictionary <JointType, Joint>[] firstJoints = new Dictionary <JointType, Joint> [frameseq.recordNum]; for (int no = 0; no < frameseq.recordNum; no++) { var bodies = frameseq.GetNextData(no, timestamps[0]).bodies; if (bodies.Count() > 0) { SerializableBody body = bodies.Where(b => b.integratedId == user).FirstOrDefault(); if (body != null) { var validJoints = Utility.GetValidJoints(body.Joints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D()); if (skeletonInterpolator.pivot.Count < validJoints.Count) { skeletonInterpolator.pivot = validJoints; } } } } List <Dictionary <JointType, CvPoint3D64f> > jointsSeq = new List <Dictionary <JointType, CvPoint3D64f> >(); List <DateTime> times = new List <DateTime>(); foreach (DateTime time in timestamps) { Dictionary <JointType, CvPoint3D64f> joints = skeletonInterpolator.IntegrateSkeleton(time, user, frameseq); // jointsがnullの場合にはダミーデータを突っ込んで長さを稼ぐ if (joints == null) { jointsSeq.Add(new Dictionary <JointType, CvPoint3D64f>() { { JointType.SpineBase, new CvPoint3D64f(float.MaxValue, float.MaxValue, float.MaxValue) } }); } else { jointsSeq.Add(joints); } times.Add(time); } allBodies[user] = jointsSeq; allTimes[user] = times; } return(Tuple.Create(allBodies, allTimes)); }
public static Tuple<Dictionary<int, List<Dictionary<JointType, CvPoint3D64f>>>, Dictionary<int, List<DateTime>>> ExportFromProject(FrameSequence frameseq, int startIndex, int endIndex) { TimeSpan period = new TimeSpan((long)(10000000 / frameseq.frameRate)); List<DateTime> timestamps = new List<DateTime>(); for (DateTime time = frameseq.Frames[startIndex].Time; time < frameseq.Frames[endIndex].Time; time += period) { timestamps.Add(time); } HashSet<Tuple<int, JointType>> uniqueUserJoint = new HashSet<Tuple<int, JointType>>(); List<Frame> frames = frameseq.Slice(startIndex, endIndex); foreach(Frame frame in frames){ for (int i = 0; i < frameseq.recordNum; i++) { foreach (SerializableBody body in frame.GetBodyList(i)) { foreach (JointType jointType in Utility.GetValidJoints(body.Joints).Keys) { // TrackingIdと勝手に作ったuser idの対応辞書が必要. 以下の処理はとりあえず Tuple<int, JointType> userJoint = new Tuple<int, JointType>(body.integratedId, jointType); if (!uniqueUserJoint.Contains(userJoint)) { uniqueUserJoint.Add(userJoint); } } } } } List<Tuple<int, JointType>> userJointPairs = ( from pair in uniqueUserJoint orderby pair.Item1, pair.Item2 select pair ).ToList(); Dictionary<int, List<Dictionary<JointType, CvPoint3D64f>>> allBodies = new Dictionary<int, List<Dictionary<JointType, CvPoint3D64f>>>(); Dictionary<int, List<DateTime>> allTimes = new Dictionary<int, List<DateTime>>(); SkeletonInterpolator skeletonInterpolator = new SkeletonInterpolator(0.5, true); foreach (int user in userJointPairs.Select(p => p.Item1).Distinct()) { skeletonInterpolator.pivot = new Dictionary<JointType, CvPoint3D64f>(); // pivot初期化処理 Dictionary<JointType, Joint>[] firstJoints = new Dictionary<JointType,Joint>[frameseq.recordNum]; for (int no = 0; no < frameseq.recordNum; no++) { var bodies = frameseq.GetNextData(no, timestamps[0]).bodies; if (bodies.Count() > 0) { SerializableBody body = bodies.Where(b => b.integratedId == user).FirstOrDefault(); if (body != null) { var validJoints = Utility.GetValidJoints(body.Joints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D()); if (skeletonInterpolator.pivot.Count < validJoints.Count) { skeletonInterpolator.pivot = validJoints; } } } } List<Dictionary<JointType, CvPoint3D64f>> jointsSeq = new List<Dictionary<JointType, CvPoint3D64f>>(); List<DateTime> times = new List<DateTime>(); foreach(DateTime time in timestamps) { Dictionary<JointType, CvPoint3D64f> joints = skeletonInterpolator.IntegrateSkeleton(time, user, frameseq); // jointsがnullの場合にはダミーデータを突っ込んで長さを稼ぐ if (joints == null) { jointsSeq.Add(new Dictionary<JointType, CvPoint3D64f>() { { JointType.SpineBase, new CvPoint3D64f(float.MaxValue, float.MaxValue, float.MaxValue) } }); } else { jointsSeq.Add(joints); } times.Add(time); } allBodies[user] = jointsSeq; allTimes[user] = times; } return Tuple.Create(allBodies, allTimes); }
public Dictionary <JointType, CvPoint3D64f> IntegrateSkeleton(DateTime time, int userInt, FrameSequence frameSeq) { List <CvMat> ToWorldConversions = frameSeq.ToWorldConversions; List <CameraIntrinsics> cameraInfo = frameSeq.CameraInfo; List <UserSegmentation> segm = frameSeq.Segmentations; Dictionary <JointType, CvPoint3D64f>[] jointsArr = new Dictionary <JointType, CvPoint3D64f> [frameSeq.recordNum]; Dictionary <JointType, CvPoint3D64f>[] pivotCandidate = new Dictionary <JointType, CvPoint3D64f> [frameSeq.recordNum]; double[] reliabilityArr = new double[frameSeq.recordNum]; double[] weightArr = new double[frameSeq.recordNum]; for (int recordNo = 0; recordNo < frameSeq.recordNum; recordNo++) { MotionData prevData = frameSeq.GetPrevData(recordNo, time); MotionData nextData = frameSeq.GetNextData(recordNo, time); if (prevData == null || nextData == null || prevData.bodies.Length * nextData.bodies.Length == 0 || !prevData.isValid || !nextData.isValid) { jointsArr[recordNo] = null; reliabilityArr[recordNo] = 0; weightArr[recordNo] = 0; continue; } SerializableBody prevBody = prevData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault(); SerializableBody nextBody = nextData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault(); if (prevBody == null || nextBody == null || prevBody.Equals(default(SerializableBody)) || nextBody.Equals(default(SerializableBody))) { jointsArr[recordNo] = null; reliabilityArr[recordNo] = 0; weightArr[recordNo] = 0; continue; } // 統計情報によるフィルタリング Dictionary <JointType, Joint> prevJoints; Dictionary <JointType, Joint> nextJoints; if (prevBody.Joints == null || nextBody.Joints == null) { continue; } // pivotが設定されてるとき、つまり、本番統合のとき if (pivot != null) { // mirror矯正 prevJoints = this.CorrectMirrorJoint(prevBody.Joints, ToWorldConversions[recordNo]); nextJoints = this.CorrectMirrorJoint(nextBody.Joints, ToWorldConversions[recordNo]); // next pivot候補つめつめ pivotCandidate[recordNo] = Utility.GetValidJoints(nextJoints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D()); } else { prevJoints = prevBody.Joints; nextJoints = nextBody.Joints; } // 統計情報があるとき if (frameSeq.BodyStat != null) { prevJoints = frameSeq.BodyStat.FilterBonesByStatistics(prevJoints); nextJoints = frameSeq.BodyStat.FilterBonesByStatistics(nextJoints); } else { prevJoints = Utility.GetValidJoints(prevBody.Joints); nextJoints = Utility.GetValidJoints(nextBody.Joints); } jointsArr[recordNo] = this.InterpolateSkeleton(prevData, nextData, prevJoints, nextJoints, time, ToWorldConversions[recordNo]); reliabilityArr[recordNo] = this.GetSkeletonReliability(prevData, nextData, prevJoints, nextJoints, time, cameraInfo[recordNo]); weightArr[recordNo] = this.GetVarianceWeight(prevData, nextData, prevJoints, nextJoints, time); } // pivot更新 pivot = new Dictionary <JointType, CvPoint3D64f>(); foreach (var candidate in pivotCandidate) { if (candidate != null && pivot.Count <= candidate.Count) { pivot = candidate; } } double maxWeight = weightArr.Max(); double[] modifiedReliabilityList = weightArr.Select(w => Math.Max(0, (w / maxWeight) - _weightBase)).Zip(reliabilityArr, (a, b) => a * b).ToArray(); var keys = jointsArr.Where(j => j != null).SelectMany(j => j.Keys).Distinct().ToList(); if (maxWeight == 0) { return(null); } return(CalcEx.LinearMedianSkeletons(jointsArr, modifiedReliabilityList)); }
public Dictionary<JointType, CvPoint3D64f> IntegrateSkeleton(DateTime time, int userInt, FrameSequence frameSeq) { List<CvMat> ToWorldConversions = frameSeq.ToWorldConversions; List<CameraIntrinsics> cameraInfo = frameSeq.CameraInfo; List<UserSegmentation> segm = frameSeq.Segmentations; Dictionary<JointType, CvPoint3D64f>[] jointsArr = new Dictionary<JointType, CvPoint3D64f>[frameSeq.recordNum]; Dictionary<JointType, CvPoint3D64f>[] pivotCandidate = new Dictionary<JointType, CvPoint3D64f>[frameSeq.recordNum]; double[] reliabilityArr = new double[frameSeq.recordNum]; double[] weightArr = new double[frameSeq.recordNum]; for (int recordNo = 0; recordNo < frameSeq.recordNum; recordNo++) { MotionData prevData = frameSeq.GetPrevData(recordNo, time); MotionData nextData = frameSeq.GetNextData(recordNo, time); if (prevData == null || nextData == null || prevData.bodies.Length * nextData.bodies.Length == 0 || !prevData.isValid || !nextData.isValid) { jointsArr[recordNo] = null; reliabilityArr[recordNo] = 0; weightArr[recordNo] = 0; continue; } SerializableBody prevBody = prevData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault(); SerializableBody nextBody = nextData.bodies.Where(b => b.integratedId == userInt).FirstOrDefault(); if (prevBody == null || nextBody == null || prevBody.Equals(default(SerializableBody)) || nextBody.Equals(default(SerializableBody))) { jointsArr[recordNo] = null; reliabilityArr[recordNo] = 0; weightArr[recordNo] = 0; continue; } // 統計情報によるフィルタリング Dictionary<JointType, Joint> prevJoints; Dictionary<JointType, Joint> nextJoints; if (prevBody.Joints == null || nextBody.Joints == null) continue; // pivotが設定されてるとき、つまり、本番統合のとき if (pivot != null) { // mirror矯正 prevJoints = this.CorrectMirrorJoint(prevBody.Joints, ToWorldConversions[recordNo]); nextJoints = this.CorrectMirrorJoint(nextBody.Joints, ToWorldConversions[recordNo]); // next pivot候補つめつめ pivotCandidate[recordNo] = Utility.GetValidJoints(nextJoints).ToDictionary(p => p.Key, p => (CvPoint3D64f)p.Value.Position.ToCvPoint3D()); } else { prevJoints = prevBody.Joints; nextJoints = nextBody.Joints; } // 統計情報があるとき if (frameSeq.BodyStat != null) { prevJoints = frameSeq.BodyStat.FilterBonesByStatistics(prevJoints); nextJoints = frameSeq.BodyStat.FilterBonesByStatistics(nextJoints); } else { prevJoints = Utility.GetValidJoints(prevBody.Joints); nextJoints = Utility.GetValidJoints(nextBody.Joints); } jointsArr[recordNo] = this.InterpolateSkeleton(prevData, nextData, prevJoints, nextJoints, time, ToWorldConversions[recordNo]); reliabilityArr[recordNo] = this.GetSkeletonReliability(prevData, nextData, prevJoints, nextJoints, time, cameraInfo[recordNo]); weightArr[recordNo] = this.GetVarianceWeight(prevData, nextData, prevJoints, nextJoints, time); } // pivot更新 pivot = new Dictionary<JointType, CvPoint3D64f>(); foreach (var candidate in pivotCandidate) { if (candidate != null && pivot.Count <= candidate.Count) { pivot = candidate; } } double maxWeight = weightArr.Max(); double[] modifiedReliabilityList = weightArr.Select(w => Math.Max(0, (w / maxWeight) - _weightBase)).Zip(reliabilityArr, (a, b) => a * b).ToArray(); var keys = jointsArr.Where(j => j != null).SelectMany(j => j.Keys).Distinct().ToList(); if (maxWeight == 0) return null; return CalcEx.LinearMedianSkeletons(jointsArr, modifiedReliabilityList); }