public void AddFrame(MotionFrame frame)
{
if (_isRecording)
{
_frameQueue.AddWork(frame);
}
}
public void AddFrame(MotionFrame frame)
{
if (_isRecording)
{
_frameQueue.AddWork(frame);
}
}
// the frame is from clips
private float RootMotionCost(MotionFrame frame, MotionFrame current,
PlayerSetting playerSetting)
{
var velocity = Mathf.Abs(frame.Velocity - current.Velocity);
return(playerSetting.RootMotionCostFactor * velocity);
}
private void ClearFrameViewer(MotionFrame frame)
{
if (CurrentFrameViewer != null)
{
CurrentFrameViewer.Clear();
}
}
private void UpdateFrameViewer(MotionSensorDevice device, MotionFrame frame)
{
if (CurrentFrameViewer != null)
{
CurrentFrameViewer.UpdateMotionFrame(device, frame);
}
}
public MotionFrame MotionFrameByIndex(int globalIndex)
{
if (cost_esti_.Count != EstimateCostCount() || globalIndex >= cost_esti_.Count)
{
throw new Exception("value count invalid");
}
int index = globalIndex;
int a = totalBeats_ - 1;
while (index >= a)
{
index -= a;
a--;
}
int i0 = totalBeats_ - 1 - a;
int i1 = i0 + 1 + index;
var frame = new MotionFrame()
{
minCost = cost_esti_[globalIndex],
index0 = i0,
index1 = i1
};
return(frame);
}
void ProcessFrame(MotionFrame frame)
{
int currentFrameId = _description.FrameCount;
if (currentFrameId == 0)
{
_description.RecordingStartDateTimeUTC = frame.TimeUTC;
}
string filename = "frame" + currentFrameId.ToString("D8") + ".mfx";
filename = Path.Combine(_scratchDirectory, filename);
var bytes = _serializer.Serialize(frame);
using (var FSFile = new FileStream(filename, FileMode.Create, FileAccess.Write,
FileShare.None, BlockSize, FileOptions.None))
{
FSFile.Write(bytes, 0, bytes.Length);
FSFile.Close();
//File.WriteAllBytes(filename, bytes);
}
currentFrameId++;
_description.FrameCount = currentFrameId;
_description.RecordingStopDateTimeUTC = frame.TimeUTC;
if (currentFrameId % 10 == 0)
{
SaveDescription();
}
}
private void LoadFrameWorker(string filename)
{
MotionFrame frame = null;
using (var stream = new MemoryStream())
{
lock (_zipFileLock)
{
if (_zipFile != null)
{
_zipFile[filename].Extract(stream);
}
else
{
return;
}
}
stream.Position = 0;
frame = _serializer.Deserialize(stream);
}
if (frame == null)
{
throw new InvalidOperationException("MotionFrame " + filename + " not deserialized correctly");
}
Debug.WriteLine("Buffering frame " + filename);
lock (_bufferedFrames)
{
_bufferedFrames.Add(frame);
}
}
private void DisplayFrame(MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action) delegate
{
DisplayFrame(frame);
});
return;
}
lastFrame = frame;
UpdateFrameViewer(sensorDevice, frame);
bool newIsRecordingOn = cbxRecord.IsChecked.Value;
if (!isRecordingOn && newIsRecordingOn)
{
StartRecording();
isRecordingOn = newIsRecordingOn;
}
else if (isRecordingOn && !newIsRecordingOn)
{
StopRecording();
isRecordingOn = newIsRecordingOn;
}
UpdateCompressionSizes(frame);
CheckFPS();
txtFPS.Text = fps.ToString("F1");
}
private void SaveFrame(MotionFrame frame, string filename)
{
var bytes = serializer.Serialize(frame);
lastSizeDepth = serializer.DepthUserFrameSize;
lastSizeAll = bytes.Length;
File.WriteAllBytes(filename, bytes);
}
protected void OnMotionFrameAvailable(MotionFrame frame)
{
if (MotionFrameAvailable == null)
{
return;
}
MotionFrameAvailable(this, new MotionFrameAvailableEventArgs(frame));
}
private void PlayFrames()
{
if (Status != PointCloudPlayerStatus.Playing)
{
return;
}
if (CurrentFrameIndex >= MaxFrameIndex)
{
Status = PointCloudPlayerStatus.Stopped;
Seek(0);
_soundPlayer.Stop();
}
var currentTime = DateTime.Now;
var timeOffset = currentTime - _playbackStartUTC;
_soundPlayer.Seek(timeOffset);
var targetPlayerTime = MinTimeUTC + timeOffset;
MotionFrame playFrame = null;
lock (_bufferedFrames)
{
while (_bufferedFrames.Count > 0)
{
var frame = _bufferedFrames.Peek();
if (frame.TimeUTC <= targetPlayerTime)
{
playFrame = _bufferedFrames.Dequeue();
CurrentFrameIndex++;
}
else
{
break;
}
}
if (_bufferedFrames.Count == 0)
{
Trace.WriteLine("empty buffer");
}
}
if (playFrame != null)
{
CurrentTimeUTC = playFrame.TimeUTC;
var offsetMS = (playFrame.TimeUTC - targetPlayerTime).TotalMilliseconds;
Trace.WriteLine("Playing frame " + CurrentFrameIndex + " id: " + playFrame.Id + " time offset: " + offsetMS.ToString("F4"));
OnMotionFrameAvailable(playFrame);
}
}
private void UpdateFrameViewer(MotionFrame frame)
{
if (pointCloudFrameViewer != null)
{
currentConfiguration = new DeviceConfiguration()
{
DepthBufferFormat = new BufferFormat(frame.DepthFrame.Width, frame.DepthFrame.Height, frame.DepthFrame.PixelFormat),
VideoBufferFormat = new BufferFormat(frame.RGBFrame.Width, frame.RGBFrame.Height, frame.RGBFrame.PixelFormat),
};
pointCloudFrameViewer.UpdateMotionFrame(currentConfiguration, frame);
}
}
private void loadPhaseSpace(PhaseSpaceDataReader reader)
{
WaitForForm waitForm = new WaitForForm(ctrl => {
try {
_dataSet.ClearFrame();
_dataSet.ClearObject();
Dictionary <int, uint> index2id = new Dictionary <int, uint>();
int count = 1;
while (!reader.EndOfStream)
{
PhaseSpaceFrame inFrame = reader.ReadFrame();
MotionFrame outFrame = new MotionFrame(_dataSet, inFrame.Time);
for (int i = 0; i < inFrame.Markers.Length; i++)
{
uint id;
if (!index2id.TryGetValue(i, out id))
{
MotionObjectInfo newInfo = new MotionObjectInfo(typeof(PointObject));
newInfo.Name = PathEx.CombineName("unnamed", (i + 1).ToString());
_dataSet.AddObject(newInfo);
id = index2id[i] = newInfo.Id;
}
if (inFrame.Markers[i].Condition > 0)
{
outFrame[id] = new PointObject(new Vector3(inFrame.Markers[i].X, inFrame.Markers[i].Y, inFrame.Markers[i].Z));
}
}
_dataSet.AddFrame(outFrame);
ctrl.ReportProgress(99 - 990000 / (count + 10000), string.Format("Load Frame: {0} ({1} sec)", count, inFrame.Time.ToString("0.00")));
count++;
}
ctrl.ReportProgress(100, string.Format("Done"));
ctrl.DialogResult = DialogResult.OK;
} catch (Exception) {
_dataSet.ClearObject();
_dataSet.ClearFrame();
_dataSet.DoObjectInfoSetChanged();
_dataSet.DoFrameListChanged();
throw;
}
});
if (waitForm.ShowDialog() == DialogResult.OK)
{
_dataSet.DoObjectInfoSetChanged();
_dataSet.DoFrameListChanged();
}
}
public float CalculateAllCost(MotionFrame motionFrame, MotionFrame currentFrame,
PlayerSetting playerSetting)
{
float allCost = 0;
for (int j = 0; j < motionFrame.Joints.Length; j++)
{
allCost += BoneCost(motionFrame.Joints[j], currentFrame.Joints[j], playerSetting);
}
allCost += RootMotionCost(motionFrame, currentFrame, playerSetting);
allCost += TrajectoryCost(motionFrame, currentFrame, playerSetting);
return(allCost);
}
public void UpdateMotionFrame(MotionSensorDevice device, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action)delegate
{
UpdateMotionFrame(device, frame);
});
return;
}
depthImage.Source = frame.DepthFrame.AsDepthUserBitmapSource(frame.UserFrame);
rgbImage.Source = frame.RGBFrame.AsRgbBitmapSource();
skeletonImage.Source = frame.Skeletons.AsSkeletonBitmapSource(device, frame.DepthFrame.Width, frame.DepthFrame.Height);
}
public void ApplyFrameToJoints(MotionFrame frame)
{
//Debug.Log(frame.Velocity);
foreach (var jointPoint in frame.Joints)
{
if (!SkeletonJoints.Keys.Contains(jointPoint.Name))
{
//Debug.LogError($"{jointPoint.Name} is not in the {Skeleton.name}");
continue;
}
var joint = SkeletonJoints[jointPoint.Name];
ApplyJointPointToJoint(jointPoint, joint);
}
}
public void UpdateMotionFrame(MotionSensorDevice device, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action) delegate
{
UpdateMotionFrame(device, frame);
});
return;
}
depthImage.Source = frame.DepthFrame.AsDepthUserBitmapSource(frame.UserFrame);
rgbImage.Source = frame.RGBFrame.AsRgbBitmapSource();
skeletonImage.Source = frame.Skeletons.AsSkeletonBitmapSource(device, frame.DepthFrame.Width, frame.DepthFrame.Height);
}
public void UpdateMotionFrame(DeviceConfiguration config, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action) delegate
{
UpdateMotionFrame(config, frame);
});
return;
}
sensorImage.ProcessDepthFrame(frame.DepthFrame);
depthImage.Source = sensorImage.DepthImageSource;
rgbImage.Source = frame.RGBFrame.AsRgbBitmapSource();
skeletonImage.Source = frame.Skeletons.AsSkeletonBitmapSource(frame.DepthFrame.Width, frame.DepthFrame.Height);
}
public void UpdateMotionFrame(DeviceConfiguration config, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action)delegate
{
UpdateMotionFrame(config, frame);
});
return;
}
sensorImage.ProcessDepthFrame(frame.DepthFrame);
depthImage.Source = sensorImage.DepthImageSource;
rgbImage.Source = frame.RGBFrame.AsRgbBitmapSource();
skeletonImage.Source = frame.Skeletons.AsSkeletonBitmapSource(frame.DepthFrame.Width, frame.DepthFrame.Height);
}
public void UpdateMotionFrame(MotionSensorDevice device, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action) delegate
{
UpdateMotionFrame(device, frame);
});
return;
}
if (!pointCloudImage.IsInitialized)
{
Activate(device);
}
pointCloudImage.SetMotionFrame(frame);
}
public void UpdateMotionFrame(DeviceConfiguration config, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action)delegate
{
UpdateMotionFrame(config, frame);
});
return;
}
if (!pointCloudImage.IsInitialized)
{
Activate(config);
}
pointCloudImage.SetMotionFrame(frame);
}
public void UpdateMotionFrame(DeviceConfiguration config, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action) delegate
{
UpdateMotionFrame(config, frame);
});
return;
}
if (!pointCloudImage.IsInitialized)
{
Activate(config);
}
pointCloudImage.SetMotionFrame(frame);
}
public void UpdateMotionFrame(MotionSensorDevice device, MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action)delegate
{
UpdateMotionFrame(device, frame);
});
return;
}
if (!pointCloudImage.IsInitialized)
{
Activate(device);
}
pointCloudImage.SetMotionFrame(frame);
}
private MotionFrame GetBakedMotionFrame(PlayerInput playerInput,
float timer)//, MotionClipType motionClipType)
{
//does motionFrame have cliptype?
MotionFrame motionFrame = null;
for (int i = 0; i < MotionClips.Count; i++)
{
MotionClipData motionClipData = MotionClips[i];
var normalizedTime = (timer % motionClipData.MotionClipLengthInMilliseconds) / motionClipData.MotionClipLengthInMilliseconds;
if (isJump || isDash)
{
if (isJump && motionClipData.Name.Contains("jump"))
{
MotionName = motionClipData.Name;
motionFrame = motionClipData.MotionFrames[0];
NowClipType = motionClipData.ClipType;
break;
}
if (isDash && motionClipData.Name.Contains("dash"))
{
MotionName = motionClipData.Name;
motionFrame = motionClipData.MotionFrames[0];
NowClipType = motionClipData.ClipType;
break;
}
}
else if (motionClipData.Name == MotionName)
{
int frame = Mathf.FloorToInt(motionClipData.MotionFrames.Length * normalizedTime);
motionFrame = motionClipData.MotionFrames[frame];//PlayerMotionFrame;
NowClipType = motionClipData.ClipType;
break;
}
}
return(motionFrame);
/*
* else if (playerInput.Crouch && motionClipData.Name.Contains("crouch"))
* {
* MotionName = motionClipData.Name;
* motionFrame = motionClipData.MotionFrames[0];
* break;
* }
*/
}
void ProcessFrame(MotionFrame frame)
{
if (isRecordingOn)
{
SaveFrame(frame, "Recording/frame" + frame.Id.ToString("D8") + ".mfx");
savedFrameId = frame.Id;
}
else
{
lastSizeDepth = frame.DepthFrame.Data.Length;
int originalPlayerIndexSize = 0;
if (frame.UserFrame != null)
{
originalPlayerIndexSize = frame.UserFrame.Data.Length;
}
int originalRGBSize = frame.RGBFrame.Data.Length;
lastSizeAll = lastSizeDepth + originalRGBSize + originalPlayerIndexSize;
}
}
private void DisplayFrame(MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action) delegate
{
DisplayFrame(frame);
});
return;
}
lastFrame = frame;
UpdateFrameViewer(frame);
UpdateCompressionSizes(frame);
CheckFPS();
txtFPS.Text = fps.ToString("F1");
}
private void UpdateCompressionSizes(MotionFrame frame)
{
int originalDepthSize = frame.DepthFrame.Data.Length;
int originalPlayerIndexSize = 0;
if (frame.UserFrame != null)
{
originalPlayerIndexSize = frame.UserFrame.Data.Length;
}
int originalRGBSize = frame.RGBFrame.Data.Length;
double ratio = originalDepthSize / (double)lastSizeDepth;
txtSizeDepth.Text = "Depth: " + lastSizeDepth.ToString() + " bytes Ratio: " + ratio.ToString("F1") + " : 1";
int totalSize = originalDepthSize + originalRGBSize + originalPlayerIndexSize;
ratio = totalSize / (double)lastSizeAll;
txtSizeAll.Text = "All: " + lastSizeAll.ToString() + " bytes Ratio: " + ratio.ToString("F1") + " : 1";
txtFrameId.Text = savedFrameId.ToString();
}
public void GetClipTrajectoryData(MotionFrame frame)
{
frame.TrajectoryDatas = new MotionTrajectoryData[MotionTrajectoryData.Length()];
for (var i = 0; i < MotionTrajectoryData.Length(); i++)
{
var timeStamp = 1f / (float)(MotionTrajectoryData.Length() - i);
var trajectoryData = new MotionTrajectoryData();
//trajectoryData.LocalPosition = 1000000f * frame.Velocity * frame.Direction * timeStamp;
//trajectoryData.Velocity = 1000000f * frame.Velocity * frame.Direction;
trajectoryData.LocalPosition = 1000f * frame.Velocity * frame.Direction * timeStamp;
trajectoryData.Velocity = 1000f * frame.Velocity * frame.Direction;
if (frame.AngularVelocity != 0f)
{
trajectoryData.Direction = (Quaternion.Euler(0, frame.AngularVelocity * timeStamp, 0) * Vector3.forward).normalized;
}
frame.TrajectoryDatas[i] = trajectoryData;
}
}
private float TrajectoryCost(MotionFrame frame, MotionFrame current,
PlayerSetting playerSetting)
{
float trajectoryCost = 0;
for (int i = 0; i < frame.TrajectoryDatas.Length; i++)
{
//position cost
var traPos = frame.TrajectoryDatas[i].LocalPosition - current.TrajectoryDatas[i].LocalPosition;
var traPosCost = traPos.sqrMagnitude * playerSetting.trajectoryPosFactor;
//rotation cost
var traRot = Vector3.Dot(frame.TrajectoryDatas[i].Direction, current.TrajectoryDatas[i].Direction);
var traRotCost = traRot * playerSetting.trajectoryRotFactor;
//velocity cost
var traVel = frame.TrajectoryDatas[i].Velocity - current.TrajectoryDatas[i].Velocity;
var traVelCost = traVel.sqrMagnitude * playerSetting.trajectoryVelFactor;
trajectoryCost += (traPosCost + traRotCost + traVelCost);
}
return(trajectoryCost);
}
private void UpdateFrameViewer(MotionSensorDevice device, MotionFrame frame)
{
if (CurrentFrameViewer != null)
{
CurrentFrameViewer.UpdateMotionFrame(device, frame);
}
}
private void SaveFrame(MotionFrame frame, string filename)
{
var bytes = serializer.Serialize(frame);
lastSizeDepth = serializer.DepthUserFrameSize;
lastSizeAll = bytes.Length;
File.WriteAllBytes(filename, bytes);
}
private void ClearFrameViewer(MotionFrame frame)
{
if (CurrentFrameViewer != null)
{
CurrentFrameViewer.Clear();
}
}
public ScriptVariable Call(IList <ScriptVariable> args, ScriptConsole console)
{
_progress.Initialize(0, "Initializing...");
if (args == null)
{
throw new ArgumentNullException("args", "args cannot be null");
}
if (args.Count < 1)
{
args = new List <ScriptVariable> {
null
}
}
;
if (args[0] == null)
{
args[0] = new ListVariable();
}
IList <ScriptVariable> objectNamesVar = args[0].ToList();
if (objectNamesVar.Any(n => n.IsNull()))
{
throw new ArgumentException(global::MotionDataHandler.Properties.Settings.Default.Msg_ObjectNameCannotBeNull, "args");
}
List <string> objectNames = objectNamesVar.Select(n => n.ToString()).ToList();
MotionDataSet dataSet = console.MotionDataSet;
MotionProcEnv env2 = new MotionProcEnv(console);
List <MotionObjectInfo> infoList = new List <MotionObjectInfo>();
foreach (string objectName in objectNames)
{
MotionObjectInfo info = dataSet.GetObjectInfoByName(objectName);
if (info == null)
{
throw new ArgumentException(global::MotionDataHandler.Properties.Settings.Default.Msg_ObjectNotFound + ": " + objectName, "args");
}
infoList.Add(info);
}
foreach (MotionObjectInfo info in infoList)
{
if (!_operation.FilterSelection(info))
{
throw new ArgumentException(global::MotionDataHandler.Properties.Settings.Default.Msg_InvalidTargetObjectSpecified + ": " + info.Name, "args");
}
}
string errorMessage = "";
if (!_operation.ValidateSelection(infoList, ref errorMessage))
{
if (errorMessage == null)
{
errorMessage = "";
}
throw new ArgumentException(global::MotionDataHandler.Properties.Settings.Default.Msg_ImproperObjectSelection + ": " + errorMessage, "args");
}
IList <ProcParam <MotionProcEnv> > parameters = _operation.GetParameters() ?? new ProcParam <MotionProcEnv> [0];
if (args.Count != parameters.Count + 1)
{
throw new ArgumentException(string.Format(global::MotionDataHandler.Properties.Settings.Default.Msg_NumberOfArgumentsRequired, parameters.Count + 1));
}
for (int i = 0; i < parameters.Count; i++)
{
if (!parameters[i].FromScriptVariable(env2, args[i + 1], ref errorMessage))
{
throw new ArgumentException(string.Format(global::MotionDataHandler.Properties.Settings.Default.Msg_InvalidNthArgument + ": {1}", i + 1, errorMessage ?? ""), "args");
}
}
if (!_operation.ValidateArguments(parameters, ref errorMessage))
{
throw new ArgumentException(string.Format(global::MotionDataHandler.Properties.Settings.Default.Msg_InvalidArgument + ": {0}", errorMessage ?? ""), "args");
}
IMotionOperationGeneral general = _operation as IMotionOperationGeneral;
if (general != null)
{
_progress.Initialize(0, "Operation");
general.Operate(infoList, parameters, dataSet, _progress);
return(new ListVariable(infoList.Select(info => new StringVariable(info.Name))));
}
IMotionOperationEditObject edit = _operation as IMotionOperationEditObject;
if (edit != null)
{
_progress.Initialize(dataSet.FrameLength, "Edit Object");
foreach (MotionFrame frame in dataSet.EnumerateFrame())
{
IList <MotionObject> results = edit.EditObject(infoList, parameters, new ReadOnlyMotionFrame(frame), false);
int count = Math.Min(results.Count, infoList.Count);
for (int i = 0; i < count; i++)
{
frame[infoList[i]] = results[i];
}
_progress.CurrentValue++;
}
dataSet.DoFrameListChanged();
return(new ListVariable(infoList.Select(info => new StringVariable(info.Name))));
}
IMotionOperationOutputSequence output = _operation as IMotionOperationOutputSequence;
if (output != null)
{
_progress.Initialize(0, "Output");
IList <Sequence.SequenceData> sequences = output.OutputSequence(infoList, parameters, dataSet.EnumerateFrame().Select(frame => new ReadOnlyMotionFrame(frame)), _progress);
foreach (Sequence.SequenceData sequence in sequences)
{
console.SequenceController.AddSequence(sequence);
}
return(new ListVariable(sequences.Select(s => new StringVariable(s.Title))));
}
IMotionOperationCreateObject create = _operation as IMotionOperationCreateObject;
if (create != null)
{
_progress.Initialize(dataSet.FrameLength, "Create Object");
IList <MotionObjectInfo> newInfoList = create.GetNewObjectInfoList(infoList, parameters);
MotionFrame firstFrame = dataSet.GetFrameByIndex(0);
if (firstFrame != null)
{
IList <MotionObject> newObjects = create.CreateObjects(infoList, parameters, new ReadOnlyMotionFrame(firstFrame), false) ?? new MotionObject[0];
if (newObjects.Count != newInfoList.Count)
{
throw new InvalidOperationException(global::MotionDataHandler.Properties.Settings.Default.Msg_CreateObjectLengthMismatch);
}
}
foreach (MotionObjectInfo newInfo in newInfoList)
{
dataSet.AddObject(newInfo);
}
foreach (MotionFrame frame in dataSet.EnumerateFrame())
{
IList <MotionObject> newObjects = create.CreateObjects(infoList, parameters, new ReadOnlyMotionFrame(frame), false) ?? new MotionObject[0];
int count = Math.Min(newObjects.Count, newInfoList.Count);
for (int i = 0; i < count; i++)
{
frame[newInfoList[i]] = newObjects[i];
}
_progress.CurrentValue++;
}
dataSet.DoObjectInfoSetChanged();
dataSet.DoFrameListChanged();
return(new ListVariable(newInfoList.Select(info => new StringVariable(info.Name))));
}
return(null);
}
protected void OnMotionFrameAvailable(MotionFrame frame)
{
if (MotionFrameAvailable == null)
return;
MotionFrameAvailable(this, new MotionFrameAvailableEventArgs(frame));
}
private void PlayFrames()
{
if (Status != PointCloudPlayerStatus.Playing)
{
return;
}
if (CurrentFrameIndex >= MaxFrameIndex)
{
Status = PointCloudPlayerStatus.Stopped;
Seek(0);
_soundPlayer.Stop();
OnPlaybackEnded();
}
var timeOffset = DateTime.Now - _playbackStartUTC;
var targetPlayerTime = MinTimeUTC + timeOffset;
MotionFrame playFrame = null;
int lastFrameIndex = CurrentFrameIndex;
lock (_bufferedFrames)
{
while (_bufferedFrames.Count > 0)
{
var frame = _bufferedFrames.OrderBy(f => f.Id).First();
if (frame.TimeUTC <= targetPlayerTime)
{
playFrame = frame;
_bufferedFrames.Remove(frame);
CurrentFrameIndex++;
}
else
{
break;
}
}
}
if (CurrentFrameIndex - lastFrameIndex > 1)
{
Trace.WriteLine("Skipped a frame from " + lastFrameIndex + " to " + CurrentFrameIndex);
}
if (playFrame != null)
{
CurrentTimeUTC = playFrame.TimeUTC;
var offsetMS = (playFrame.TimeUTC - targetPlayerTime).TotalMilliseconds;
Debug.WriteLine("Playing frame " + CurrentFrameIndex + " id: " + playFrame.Id + " time offset: " + offsetMS.ToString("F4"));
//_soundPlayer.Stop();
if (Math.Abs(offsetMS) > 100)
{
var span = CurrentTimeUTC - MinTimeUTC;
if (CurrentFrameIndex % 30 == 0)
{
Trace.WriteLine("Reseeking audio to " + span.ToString());
}
_soundPlayer.Seek(span);
}
//_soundPlayer.Play();
OnMotionFrameAvailable(playFrame);
}
}
private void UpdateFrameViewer(MotionFrame frame)
{
if (pointCloudFrameViewer != null)
{
currentConfiguration = new DeviceConfiguration()
{
DepthBufferFormat = new BufferFormat(frame.DepthFrame.Width, frame.DepthFrame.Height, frame.DepthFrame.PixelFormat),
VideoBufferFormat = new BufferFormat(frame.RGBFrame.Width, frame.RGBFrame.Height, frame.RGBFrame.PixelFormat),
};
pointCloudFrameViewer.UpdateMotionFrame(currentConfiguration, frame);
}
}
private void DisplayFrame(MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action)delegate
{
DisplayFrame(frame);
});
return;
}
lastFrame = frame;
UpdateFrameViewer(frame);
UpdateCompressionSizes(frame);
CheckFPS();
txtFPS.Text = fps.ToString("F1");
}
/*
* private MotionFrame GetBakedMotionFrame(string motionName, float normalizedTime, MotionClipType clipType)
* {
* for (int i = 0; i < MotionClips.Count; i++) {
* var clip = MotionClips[i];
*
* if (clipType != null) {
* if (clipType == clip.ClipType) {
* return clip.MotionFrames[0];
* }
* } else if (clip.Name.Equals(motionName)) {
* int frameBasedOnTime = Mathf.FloorToInt(clip.MotionFrames.Length * normalizedTime);
*
* return clip.MotionFrames[frameBasedOnTime];
* }
* }
*
* return null;
* }
*/
public void ExtractMotionClips(AnimClip animationClip)
{
var motionClip = new MotionClipData();
motionClip.Name = animationClip.name;
//motionClip.MotionClipLengthInMilliseconds = animationClip.ClipLengthInMilliseconds; //no value for animationClip.ClipLengthInMilliseconds
motionClip.ClipType = animationClip.ClipType;
motionClip.MotionFrames = new MotionFrame[animationClip.Frames.Count - 10];
// The very first frame
var firstMotionFrame = new MotionFrame();
var firstFrame = animationClip.Frames[0];
var stubAnimationjointPoint = new AnimationJointPoint {
Position = Vector3.zero
};
firstMotionFrame.Joints = (from jp in firstFrame.JointPoints
select MakeMotionJoint(jp, stubAnimationjointPoint)).ToArray();
foreach (var jt in firstMotionFrame.Joints)
{
jt.BaseRotation = jt.Rotation;
}
var rootMotionJoint = firstMotionFrame.Joints.First(x => x.Name.Equals(RootName));
firstMotionFrame.AngularVelocity = Vector3.Angle(Vector3.forward, rootMotionJoint.Velocity) / 180f;
firstMotionFrame.Velocity = rootMotionJoint.Velocity.sqrMagnitude;
firstMotionFrame.Direction = rootMotionJoint.Velocity.normalized;
firstMotionFrame.Time = firstFrame.Time;
GetClipTrajectoryData(firstMotionFrame);
//motionClip.MotionFrames[0] = firstMotionFrame;
// All the other ones
for (int i = 10; i < animationClip.Frames.Count; i++)
{
var frame = animationClip.Frames[i];
var lastFrame = animationClip.Frames[i - 1];
var motionFrame = new MotionFrame();
motionFrame.Time = frame.Time;
var joints = (from jp in frame.JointPoints
from jp2 in lastFrame.JointPoints
where jp.Name.Equals(jp2.Name)
select MakeMotionJoint(jp, jp2)).ToArray();
foreach (var jt in joints)
{
var firstJt = firstMotionFrame.Joints.First(x => x.Name.Equals(jt.Name));
jt.BaseRotation = firstJt.Rotation;
}
motionFrame.Joints = joints;
var root = joints.First(x => x.Name.Equals(RootName));
motionFrame.AngularVelocity = Vector3.Angle(Vector3.forward, root.Velocity) / 180f;
motionFrame.Velocity = root.Velocity.sqrMagnitude;
motionFrame.Direction = root.Velocity.normalized;
GetClipTrajectoryData(motionFrame);
motionClip.MotionFrames[i - 10] = motionFrame;
}
motionClip.MotionClipLengthInMilliseconds = animationClip.Frames.Last().Time;
MotionClips.Add(motionClip);
}
void ProcessFrame(MotionFrame frame)
{
int currentFrameId = _description.FrameCount;
if (currentFrameId == 0)
{
_description.RecordingStartDateTimeUTC = frame.TimeUTC;
}
string filename = "frame" + currentFrameId.ToString("D8") + ".mfx";
filename = Path.Combine(_scratchDirectory, filename);
var bytes = _serializer.Serialize(frame);
using (var FSFile = new FileStream(filename, FileMode.Create, FileAccess.Write,
FileShare.None, BlockSize, FileOptions.None))
{
FSFile.Write(bytes, 0, bytes.Length);
FSFile.Close();
//File.WriteAllBytes(filename, bytes);
}
currentFrameId++;
_description.FrameCount = currentFrameId;
_description.RecordingStopDateTimeUTC = frame.TimeUTC;
if (currentFrameId % 10 == 0)
{
SaveDescription();
}
}
private void UpdateCompressionSizes(MotionFrame frame)
{
int originalDepthSize = frame.DepthFrame.Data.Length;
int originalPlayerIndexSize = 0;
if (frame.UserFrame != null)
originalPlayerIndexSize = frame.UserFrame.Data.Length;
int originalRGBSize = frame.RGBFrame.Data.Length;
double ratio = originalDepthSize / (double)lastSizeDepth;
txtSizeDepth.Text = "Depth: " + lastSizeDepth.ToString() + " bytes Ratio: " + ratio.ToString("F1") + " : 1";
int totalSize = originalDepthSize + originalRGBSize + originalPlayerIndexSize;
ratio = totalSize / (double)lastSizeAll;
txtSizeAll.Text = "All: " + lastSizeAll.ToString() + " bytes Ratio: " + ratio.ToString("F1") + " : 1";
txtFrameId.Text = savedFrameId.ToString();
}
void Start()
{
//NextFrame.Value = MotionFrames[NextIndex];
//GoalFrame.Value = MotionFrames[GoalIndex];
PlayerMotionFrame = new MotionFrame();
}
void ProcessFrame(MotionFrame frame)
{
if (isRecordingOn)
{
SaveFrame(frame, "Recording/frame" + frame.Id.ToString("D8") + ".mfx");
savedFrameId = frame.Id;
}
else
{
lastSizeDepth = frame.DepthFrame.Data.Length;
int originalPlayerIndexSize = 0;
if (frame.UserFrame != null)
originalPlayerIndexSize = frame.UserFrame.Data.Length;
int originalRGBSize = frame.RGBFrame.Data.Length;
lastSizeAll = lastSizeDepth + originalRGBSize + originalPlayerIndexSize;
}
}
private void UpdateCompressionSizes(MotionFrame frame)
{
txtFrameId.Text = "0";
}
public MotionFrameAvailableEventArgs(MotionFrame motionFrame)
{
this.MotionFrame = motionFrame;
}
private void DisplayFrame(MotionFrame frame)
{
if (!Dispatcher.CheckAccess())
{
Dispatcher.BeginInvoke((Action)delegate
{
DisplayFrame(frame);
});
return;
}
lastFrame = frame;
UpdateFrameViewer(sensorDevice, frame);
bool newIsRecordingOn = cbxRecord.IsChecked.Value;
if (!isRecordingOn && newIsRecordingOn)
{
StartRecording();
isRecordingOn = newIsRecordingOn;
}
else if (isRecordingOn && !newIsRecordingOn)
{
StopRecording();
isRecordingOn = newIsRecordingOn;
}
UpdateCompressionSizes(frame);
CheckFPS();
txtFPS.Text = fps.ToString("F1");
}
public MotionFrameAvailableEventArgs(MotionFrame motionFrame)
{
this.MotionFrame = motionFrame;
}
