/// <summary>
/// Calculates dropped frame statistics based on the difference in
/// timestamps between two consecutive headers.
/// </summary>
/// <param name="stream">
/// The KStudioSeekableEventStream object of the video stream
/// </param>
/// <param name="frameTime">The number of milliseconds per frame</param>
/// <returns>
/// FrameAnalysis object with collected frame statistics.
/// </returns>
public override FrameAnalysis CountDroppedFrames()
{
double prevFrameTime = 0;
double cumulativeTime = 0;
int eventIndex = 0;
FrameAnalysis result = new FrameAnalysis();
KStudioSeekableEventStream stream = (KStudioSeekableEventStream)_stream;
prevFrameTime = stream.StartRelativeTime.TotalMilliseconds;
foreach (KStudioEventHeader header in stream.EventHeaders)
{
double currentFrameTime = header.RelativeTime.TotalMilliseconds;
double deltaTime = currentFrameTime - prevFrameTime;
cumulativeTime += deltaTime - FrameTime;
result.FrameCount++;
result.FrameMap.Add(eventIndex);
while (cumulativeTime > FrameTime)
{
result.DroppedFrames++;
cumulativeTime -= FrameTime;
result.FrameMap.Add(eventIndex);
result.DroppedIndices.Add(result.FrameCount++);
}
eventIndex++;
prevFrameTime = currentFrameTime;
}
return(result);
}
public override void Extract(string outputPath)
{
KStudioSeekableEventStream stream = (KStudioSeekableEventStream)_stream;
int frameCount = (int)stream.EventCount;
double[] irTiming = new double[frameCount];
var values = Enumerable.Range(0, frameCount);
Parallel.ForEach(values, (value, pls, index) => {
//var currEvent = stream.ReadEvent((uint) index);
var currEvent = stream.EventHeaders[(int)index];
irTiming[index] = currEvent.RelativeTime.TotalMilliseconds;
// Update progress
OnProgressUpdated(new KinectFileProgressChangedEventArgs {
Progress = (int)((float)(index + 1) / frameCount * 100),
StatusName = Name
});
});
// Write timings
string filepath = outputPath + "/Kinect_Output";
Utils.WriteTimingToFile(filepath + "/IR_timing.txt", irTiming);
}
public bool UpdateTime(TimeSpan time, out uint foundIndex)
{
DebugHelper.AssertUIThread();
LoadEvents();
bool result = false;
foundIndex = 0;
EventData newSelectedNode = null;
if (this.events != null)
{
KStudioSeekableEventStream seekableStream = this.stream as KStudioSeekableEventStream;
if (seekableStream != null)
{
TimeSpan foundTime;
if (seekableStream.FindEvent(time, out foundIndex, out foundTime))
{
if ((foundIndex >= 0) && (foundIndex < this.events.Length))
{
newSelectedNode = this.events[(int)foundIndex];
result = true;
}
}
}
}
this.UpdateSelection(newSelectedNode);
return(result);
}
/// <summary>
/// Initializes a new instance of the StreamData class and sets all properties based on the KStudioEventStream provided
/// </summary>
/// <param name="stream">KStudioEventStream to get data from</param>
public StreamData(KStudioEventStream stream)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
this.EventStream = stream;
this.PublicMetadata = stream.GetMetadata(KStudioMetadataType.Public);
this.PersonalMetadata = stream.GetMetadata(KStudioMetadataType.PersonallyIdentifiableInformation);
this.EventHeaders = new ObservableCollection <KStudioEventHeader>();
KStudioSeekableEventStream seekStream = stream as KStudioSeekableEventStream;
if (seekStream != null)
{
this.StartTime = seekStream.StartRelativeTime;
this.EndTime = seekStream.EndRelativeTime;
List <KStudioEventHeader> headers = seekStream.EventHeaders.ToList();
foreach (KStudioEventHeader header in headers)
{
this.EventHeaders.Add(header);
}
}
}
public override void Extract(string outputPath)
{
KStudioSeekableEventStream stream = (KStudioSeekableEventStream)_stream;
int frameCount = _frameAnalysis.FrameCount;
double[] colorTiming = new double[frameCount];
var values = Enumerable.Range(0, frameCount);
Parallel.ForEach(values, new ParallelOptions {
MaxDegreeOfParallelism = 8
},
(value, pls, index) => {
WriteableBitmap colorBitmap = new WriteableBitmap(ColorWidth, ColorHeight, 96.0, 96.0,
PixelFormats.Bgr32,
null);
byte[] colorData = new byte[ColorWidth * ColorHeight * 2];
// Determine if this frame was a dropped frame
int frameIndex = _frameAnalysis.FrameMap[(int)index];
var currEvent = stream.ReadEvent((uint)frameIndex);
currEvent.CopyEventDataToArray(colorData, 0);
NativeMethods.YUY2ToBGR(colorData, 1920 * 1080 / 2, colorBitmap.BackBuffer, 1920 * 1080 * 4);
colorBitmap.WritePixels(new Int32Rect(0, 0, 1920, 1080), colorBitmap.BackBuffer, 1920 * 1080 * 4, colorBitmap.BackBufferStride);
// create a png bitmap encoder which knows how to save a .png file
//BitmapEncoder encoder = new PngBitmapEncoder();
BitmapEncoder encoder = new BmpBitmapEncoder();
// create frame from the writable bitmap and add to encoder
encoder.Frames.Add(BitmapFrame.Create(colorBitmap));
string filePath = outputPath + "/Kinect_Output/Color/";
Directory.CreateDirectory(filePath);
string path = Path.Combine(filePath, "ColorFrame_" + index + ".bmp");
// write the new file to disk
try {
using (FileStream fs = new FileStream(path, FileMode.Create)) {
encoder.Save(fs);
}
}
catch (IOException) { }
colorTiming[frameIndex] = currEvent.RelativeTime.TotalMilliseconds;
// Update progress
OnProgressUpdated(new KinectFileProgressChangedEventArgs {
Progress = (int)((float)((int)index + 1) / frameCount * 100),
StatusName = Name
});
});
// Write timings
string filepath = outputPath + "/Kinect_Output";
Utils.WriteTimingToFile(filepath + "/color_timing.txt", colorTiming);
}
public override void Extract(string outputPath)
{
KStudioSeekableEventStream stream = (KStudioSeekableEventStream)_stream;
string filePath = outputPath + "\\Kinect_Output\\Depth\\";
Utils.ExistOrCreateDirectory(filePath);
int frameCount = (int)stream.EventCount;
double[] depthTiming = new double[frameCount];
var values = Enumerable.Range(0, frameCount);
Parallel.ForEach(values, (value, pls, index) => {
WriteableBitmap depthBitmap = new WriteableBitmap(DepthWidth, DepthHeight, 96.0, 96.0,
PixelFormats.Gray16, null);
var currEvent = stream.ReadEvent((uint)index);
IntPtr buffer = Marshal.AllocHGlobal((int)currEvent.EventDataSize);
currEvent.CopyEventDataToBuffer(currEvent.EventDataSize, buffer);
depthBitmap.WritePixels(new System.Windows.Int32Rect(0, 0, DepthWidth, DepthHeight), buffer, (int)currEvent.EventDataSize, depthBitmap.BackBufferStride);
// create a png bitmap encoder which knows how to save a .png file
BitmapEncoder encoder = new PngBitmapEncoder();
// create frame from the writable bitmap and add to encoder
encoder.Frames.Add(BitmapFrame.Create(depthBitmap));
string path = Path.Combine(filePath, "DepthFrame_" + index + ".png");
// write the new file to disk
try {
using (FileStream fs = new FileStream(path, FileMode.Create)) {
encoder.Save(fs);
}
}
catch (IOException ex)
{
Console.WriteLine(ex.StackTrace);
}
depthTiming[index] = currEvent.RelativeTime.TotalMilliseconds;
// Update progress
OnProgressUpdated(new KinectFileProgressChangedEventArgs {
Progress = (int)((float)(index + 1) / frameCount * 100),
StatusName = Name
});
});
// Write timings
string filepath = outputPath + "\\Kinect_Output";
Utils.WriteTimingToFile(filepath + "\\depth_timing.txt", depthTiming);
}
private void OnSwimLanesSourceChanged(IEnumerable values)
{
DebugHelper.AssertUIThread();
foreach (DataBar dataBar in this.dataBars)
{
this.RemoveDataBar(dataBar);
}
this.dataBars.Clear();
if (values != null)
{
foreach (object value in values)
{
DataBar dataBar = null;
FrameworkElement sidebar = null;
KStudioSeekableEventStream seekableEventStream = value as KStudioSeekableEventStream;
if (seekableEventStream != null)
{
sidebar = new ContentControl()
{
Content = seekableEventStream,
ContentTemplate = this.SidebarTemplate,
ContentTemplateSelector = this.SidebarTemplateSelector,
};
IEventLaneDataSource eventDataSource = seekableEventStream.UserState as IEventLaneDataSource;
if (eventDataSource != null)
{
EventLane2 eventDataBar = new EventLane2(eventDataSource.MinTime, eventDataSource.MaxTime)
{
DataContext = seekableEventStream,
DataSource = eventDataSource,
};
dataBar = eventDataBar;
}
}
if (dataBar == null)
{
dataBar = new DataBar();
}
this.dataBars.Add(dataBar);
this.AddDataBar(dataBar, sidebar, GridLength.Auto);
}
}
}
/// <summary>
/// Calculates dropped frame statistics based on the difference in
/// timestamps between two consecutive headers.
/// </summary>
/// <param name="stream">
/// The KStudioSeekableEventStream object of the audio stream
/// </param>
/// <param name="frameTime">
/// The number of milliseconds per audio audio frame. This is not
/// necessarily the time per event header as multiple audio samples can
/// be stored in a single event.
/// </param>
/// <returns>
/// FrameAnalysis object with collected frame statistics.
/// </returns>
public override FrameAnalysis CountDroppedFrames()
{
double prevFrameTime = 0;
double cumulativeTime = 0;
int lastGoodEvent = 0;
int eventIndex = 0;
int eventFrameSize = 14432;
FrameAnalysis result = new FrameAnalysis();
KStudioSeekableEventStream stream = (KStudioSeekableEventStream)_stream;
foreach (KStudioEventHeader header in stream.EventHeaders)
{
double currentFrameTime = header.RelativeTime.TotalMilliseconds;
double deltaTime = currentFrameTime - prevFrameTime;
double missedTime = 0;
int j = 0;
// Determine if there are missed frames.
do
{
j += eventFrameSize - 16;
missedTime += FrameTime;
} while (j <= (int)header.EventDataSize);
missedTime = deltaTime - missedTime;
cumulativeTime += missedTime;
if (cumulativeTime > FrameTime)
{
while (cumulativeTime > FrameTime)
{
cumulativeTime -= FrameTime;
result.DroppedFrames++;
result.FrameMap.Add(lastGoodEvent);
result.DroppedIndices.Add(result.FrameCount++);
}
}
else
{
lastGoodEvent = eventIndex;
result.FrameMap.Add(lastGoodEvent);
result.FrameCount++;
}
eventIndex++;
prevFrameTime = currentFrameTime;
}
return(result);
}
private void DirtyWork(object sender, DoWorkEventArgs e)
{
outputData = new ushort[WIDTH * HEIGHT];
var client = KStudio.CreateClient();
if (string.IsNullOrEmpty(fileName))
{
return;
}
var file = client.OpenEventFile(fileName);
foreach (var item in file.EventStreams)
{
if (item.DataTypeName.Equals("Nui Depth"))
{
state = "Depth";
KStudioSeekableEventStream stream = item as KStudioSeekableEventStream;
this.frameCount = (int)stream.EventCount;
timing = new ushort[frameCount];
for (uint i = 0; i < frameCount; i++)
{
b.ReportProgress((int)((float)i / frameCount * 100));
Thread.Sleep(100);
var curr_event = stream.ReadEvent(i);
//unsafe
{
int size = outputData.Length * sizeof(ushort);
IntPtr ip = Marshal.AllocHGlobal(size);
uint bufferSize = 0;
curr_event.AccessUnderlyingEventDataBuffer(out bufferSize, out ip);
Copy(ip, outputData, 0, outputData.Length);
}
this.timing[i] = (ushort)curr_event.RelativeTime.TotalMilliseconds;
string filePath = Environment.CurrentDirectory + "/Xef2Mat_Output/DepthFrame" + i.ToString("D4") + ".mat";
MATWriter.ToMatFile("Dep" + i.ToString("D4"), filePath, outputData, HEIGHT, WIDTH);
}
}
if (item.DataTypeName.Equals("Nui IR"))
{
state = "IR";
KStudioSeekableEventStream stream = item as KStudioSeekableEventStream;
this.frameCount = (int)stream.EventCount;
timing = new ushort[frameCount];
for (uint i = 0; i < frameCount; i++)
{
b.ReportProgress((int)((float)i / frameCount * 100));
var curr_event = stream.ReadEvent(i);
//unsafe
{
int size = outputData.Length * sizeof(ushort);
IntPtr ip = Marshal.AllocHGlobal(size);
uint bufferSize = 0;
curr_event.AccessUnderlyingEventDataBuffer(out bufferSize, out ip);
Copy(ip, outputData, 0, outputData.Length);
}
this.timing[i] = (ushort)curr_event.RelativeTime.TotalMilliseconds;
string filePath = Environment.CurrentDirectory + "/Xef2Mat_Output/IRFrame" + i.ToString("D4") + ".mat";
MATWriter.ToMatFile("IR" + i.ToString("D4"), filePath, outputData, HEIGHT, WIDTH);
}
}
}
if (frameCount > 0)
{
state = "TimeStamp";
b.ReportProgress(100);
string filePath = Environment.CurrentDirectory + "/Xef2Mat_Output/TimeStamp.mat";
MATWriter.ToMatFile("Time", filePath, this.timing, this.timing.Length, 1);
}
}
private void LoadEvents()
{
if (this.events == null)
{
KStudioSeekableEventStream seekableStream = this.stream as KStudioSeekableEventStream;
if (seekableStream != null)
{
IReadOnlyList <KStudioEventHeader> eventHeaders = seekableStream.EventHeaders;
this.events = new EventData[eventHeaders.Count];
bool doFrameNumber = false;
if (seekableStream.TagSize >= sizeof(uint))
{
// At this time, assume if there is at least 4 bytes of tag data that it is a frame number
doFrameNumber = true;
}
int count = eventHeaders.Count;
if (count > 0)
{
ulong tick;
uint eventIndex;
{
KStudioEventHeader eventHeader = eventHeaders[0];
tick = (ulong)eventHeader.RelativeTime.Ticks * EventStreamState.cTimeSpanTicksToTimelineTicks;
eventIndex = eventHeader.EventIndex;
}
int lastIndex = count - 1;
for (int i = 0; i < lastIndex; ++i)
{
KStudioEventHeader eventHeader = eventHeaders[i + 1];
ulong nextTick = (ulong)eventHeader.RelativeTime.Ticks * EventStreamState.cTimeSpanTicksToTimelineTicks;
uint? frameNumber = null;
if (doFrameNumber)
{
uint bufferSize;
IntPtr bufferPtr;
eventHeader.AccessUnderlyingTagDataBuffer(out bufferSize, out bufferPtr);
Debug.Assert(bufferSize >= sizeof(uint));
unsafe
{
frameNumber = *((uint *)bufferPtr.ToPointer());
}
}
EventData eventDataNode = new EventData((int)eventIndex, frameNumber, tick, nextTick - tick);
tick = nextTick;
eventIndex = eventHeader.EventIndex;
this.events[i] = eventDataNode;
}
{
KStudioEventHeader eventHeader = eventHeaders[lastIndex];
uint?frameNumber = null;
if (doFrameNumber)
{
uint bufferSize;
IntPtr bufferPtr;
eventHeader.AccessUnderlyingTagDataBuffer(out bufferSize, out bufferPtr);
Debug.Assert(bufferSize >= sizeof(uint));
unsafe
{
frameNumber = *((uint *)bufferPtr.ToPointer());
}
}
ulong lastDuration;
if (this.duration == 0)
{
lastDuration = EventStreamState.cLastEventDuration;
}
else
{
lastDuration = this.duration - tick;
}
EventData eventDataNode = new EventData((int)eventIndex, frameNumber, tick, lastDuration);
this.events[lastIndex] = eventDataNode;
}
#if TODODEB
EventHandler handler = this.TimeRangeChanged;
if (handler != null)
{
handler(this, EventArgs.Empty);
}
handler = this.RenderInvalidated;
if (handler != null)
{
handler(this, EventArgs.Empty);
}
#endif
}
}
this.readOnlyEvents = Array.AsReadOnly(this.events);
}
}
public override void Extract(string outputPath)
{
KStudioSeekableEventStream stream = (KStudioSeekableEventStream)_stream;
IReadOnlyList <KStudioEventHeader> headers = stream.EventHeaders;
int frameCount = (int)stream.EventCount;
double[] audioTiming = new double[frameCount];
const int eventFrameSize = 14432; // Size for a single audio event frame
const int frameSize = 1024; // The size of the actual audio data in an event frame
int dataSize = 0;
int bytePos = 0;
// DEBUG
int missedFrames = 0;
double cumulativeTime = 0;
double lDiff = 0;
KStudioEventHeader lHeader = null;
// END DEBUG
// Hacky counting of total data size
for (int i = 0; i < frameCount; i++)
{
KStudioEventHeader header = headers[i];
int j = eventFrameSize;
double missedTime = 0;
double diff = header.RelativeTime.TotalMilliseconds;
/* DEBUG */
if (lHeader != null)
{
diff = header.RelativeTime.TotalMilliseconds - lHeader.RelativeTime.TotalMilliseconds;
if (diff > lDiff)
{
lDiff = diff;
}
}
lHeader = header;
/* END DEBUG */
do
{
dataSize += frameSize;
j += eventFrameSize - 16;
missedTime += 16;
} while (j <= (int)header.EventDataSize);
missedTime = diff - missedTime;
cumulativeTime += missedTime;
while (cumulativeTime > 16)
{
cumulativeTime -= 16;
missedFrames++;
}
}
// Calculate additional bytes needed
//int numBytes = (int) totalMissedTime / 16 * 1024;
int numBytes = missedFrames * frameSize;
if (numBytes > 0)
{
dataSize += numBytes;
}
// DEBUG
missedFrames = 0;
cumulativeTime = 0;
KStudioEvent lEvent = null;
// END DEBUG
byte[] rawAudio = new byte[dataSize];
for (uint i = 0; i < frameCount; i++)
{
int j = eventFrameSize;
var currEvent = stream.ReadEvent(i);
byte[] rawBuffer = new byte[currEvent.EventDataSize];
int offset = 96;
double diff = currEvent.RelativeTime.TotalMilliseconds;
double missedTime = 0;
if (lEvent != null)
{
diff = currEvent.RelativeTime.TotalMilliseconds - lEvent.RelativeTime.TotalMilliseconds;
if (diff > lDiff)
{
lDiff = diff;
}
}
lEvent = currEvent;
audioTiming[i] = currEvent.RelativeTime.TotalMilliseconds;
currEvent.CopyEventDataToArray(rawBuffer, 0);
do
{
Array.Copy(rawBuffer, offset, rawAudio, bytePos, frameSize);
bytePos += frameSize;
j += eventFrameSize - 16;
offset += eventFrameSize - 16;
missedTime += 16;
} while (j <= (int)currEvent.EventDataSize);
// If there is any missed time, add it here.
missedTime = diff - missedTime;
cumulativeTime += missedTime;
while (cumulativeTime > 16)
{
cumulativeTime -= 16;
missedFrames++;
bytePos += frameSize;
}
// Update progress
OnProgressUpdated(new KinectFileProgressChangedEventArgs {
Progress = (int)((float)(i + 1) / frameCount * 100),
StatusName = Name
});
}
string filepath = outputPath + "/Kinect_Output";
Directory.CreateDirectory(filepath);
File.WriteAllBytes(filepath + "/raw_audio.wav", rawAudio);
// Write timings
Utils.WriteTimingToFile(filepath + "/audio_timing.txt", audioTiming);
}
