public virtual IEnumerable <byte[]> Iterator(int[] apids, Int64 startTime, Int64 stopTime, long skip = 0L)
{
var counter = 0L;
foreach (var p in Iterator())
{
var timestamp = PacketAccessor.Time42(p);
if (timestamp < startTime || timestamp > stopTime)
{
continue;
}
var apid = PacketAccessor.APID(p);
var len = apids.Length;
for (var i = 0; i < len; i++)
{
if (apid == apids[i])
{
if (counter++ >= skip)
{
yield return(p);
}
break;
}
}
}
}
public virtual IEnumerable <byte[]> Iterator(Int64 startTime, Int64 stopTime)
{
return(from p in Iterator()
let timestamp = PacketAccessor.Time42(p)
where timestamp >= startTime && timestamp <= stopTime
select p);
}
public IEnumerable <byte[]> Iterator(int[] apids, Int64 startTime, Int64 stopTime, long skip = 0L)
{
var counter = 0L;
return(from p in Iterator()
let timestamp = PacketAccessor.Time42(p)
let apid = PacketAccessor.APID(p)
where startTime <= timestamp && timestamp <= stopTime && apids.Contains(apid) && counter++ > skip
select p);
//return Iterator();
}
public void CheckPacketLength(byte[] packet, int packetPtr)
{
var len = PacketAccessor.PacketFixedHeaderLength + 1 + PacketAccessor.Length(packet);
//if (len != packetPtr)
// throw new Exception("Packet Length Check Exception");
if (len != packetPtr)
{
Console.Error.WriteLine(@"Packet length check exception: apid={0} timestamp={1}", PacketAccessor.APID(packet),
PacketAccessor.Time42(packet));
}
}
public static IEnumerable <byte[]> Iterator(Stream stream)
{
var buffer = new byte[PacketAccessor.MaximumPacketSize];
using (var reader = new BinaryReader(stream))
{
while (true)
{
// Read the packet header
var ptr = 0;
var toread = 6;
int read;
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
yield break;
}
toread -= read;
ptr += read;
}
// Check whether we've run off the end.
if ((buffer[0] | buffer[1] | buffer[2]) == 0)
{
yield break;
}
var length = PacketAccessor.Length(buffer);
toread = 1 + length;
// Read the rest of the packet
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
yield break;
}
toread -= read;
ptr += read;
}
yield return(buffer);
}
}
}
/// <summary>
/// Used by the iterator with the skip operational parameter. This will cause two initial
/// passes through the file and can be made more efficient.
/// </summary>
/// <param name="apids"></param>
/// <param name="startTime"></param>
/// <returns></returns>
public int ApidMatchesAfterStart(int[] apids, Int64 startTime)
{
if (startTime == _cacheStartTime && _cacheApids == apids)
{
return(_cacheApidMatchesAfterStart);
}
// Cache miss
_cacheApids = apids;
_cacheStartTime = startTime;
if (EndTime42 < startTime)
{
_cacheApidMatchesAfterStart = 0;
return(_cacheApidMatchesAfterStart);
}
if (startTime <= BeginTime42)
{
_cacheApidMatchesAfterStart = ApidCount(apids);
return(_cacheApidMatchesAfterStart);
}
var count = 0;
var max = apids.Count();
foreach (var p in Iterator(startTime, Int64.MaxValue))
{
var apid = PacketAccessor.APID(p);
for (var i = 0; i < max; i++)
{
if (apid == apids[i])
{
count++;
break;
}
}
}
_cacheApidMatchesAfterStart = count;
return(_cacheApidMatchesAfterStart);
}
public IEnumerable <byte[]> Iterator(Int64 startTime, Int64 stopTime)
{
//foreach (byte[] p in Iterator())
//{
// var timestamp = PacketAccessor.Time42(p);
// if (startTime <= timestamp && timestamp <= stopTime) yield return p;
//}
return(from p in Iterator()
let timestamp = PacketAccessor.Time42(p)
where startTime <= timestamp && timestamp <= stopTime
select p);
/*
* Console.WriteLine("startTime={0} stopTime={1}", startTime, stopTime);
* Console.WriteLine("startTime={0} stopTime={1}", TimeUtilities.Time42ToString(startTime), TimeUtilities.Time42ToString(stopTime));
* foreach (byte[] p in Iterator())
* {
* var timestamp = PacketAccessor.Time42(p);
* if (startTime <= timestamp && timestamp <= stopTime)
* yield return p;
* }
*/
}
public override IEnumerable <byte[]> Iterator()
{
var state = State.Idle;
var packet = new byte[65516 + 12]; // add some margin 65536 bytes of data plus 6 of header + 6 of timestamp
var frameCounter = 0;
var packetPtr = 0; // copy pointer into the packet array
var framePtr = 0; // copy pointer into the frame array
foreach (var frame in FrameIterator())
{
var virtualChannel = FrameAccessor.VirtualChannel(frame);
//Console.WriteLine("VirtualChannel={0}", virtualChannel);
if (virtualChannel != VirtualChannel)
{
continue;
}
var frameCount = FrameAccessor.FrameCount(frame);
//Console.WriteLine("FrameCount={0}", frameCounter);
var firstPacketPtr = FrameAccessor.FirstHeaderPointer(frame);
switch (state)
{
case State.Idle:
if (firstPacketPtr == FrameAccessor.FirstHeaderPointerOverflow)
{
break; // No packet starts here
}
framePtr = firstPacketPtr + FrameAccessor.MPDUStart;
state = State.Running;
frameCounter = frameCount;
packetPtr = 0;
goto case State.Iterate;
case State.Running:
frameCounter++;
if (frameCounter != frameCount && !(frameCount == 0 && frameCounter == 16777216))
{
var delta = frameCount - frameCounter;
if (delta < 0)
{
delta += 16777216;
}
if (LogLevel > 0)
{
Console.Error.WriteLine(@"Frame sequence error: expected {0}, saw {1}, {2} frames skipped", frameCounter, frameCount, delta);
}
state = State.Idle;
continue;
}
framePtr = FrameAccessor.MPDUStart;
if (firstPacketPtr == 0)
{
if (packetPtr > 0)
{
CheckPacketLength(packet, packetPtr); // Get rid of this after debugging
yield return(packet);
packetPtr = 0;
}
goto case State.Iterate;
}
if (firstPacketPtr == FrameAccessor.FirstHeaderPointerOverflow)
{
AppendToPacket(frame, FrameAccessor.MPDUPacketZoneLength, packet, ref framePtr,
ref packetPtr);
break;
}
if (firstPacketPtr == 2047)
{
// Idle Frame
break;
}
if (firstPacketPtr > 0)
{
AppendToPacket(frame, firstPacketPtr, packet, ref framePtr, ref packetPtr);
{
CheckPacketLength(packet, packetPtr); // Get rid of this after debugging
yield return(packet);
packetPtr = 0;
}
goto case State.Iterate;
}
break;
case State.Iterate:
{
var flag = true;
while (flag)
{
var gap = FrameAccessor.MPDUEnd - framePtr;
if (gap < PacketAccessor.PacketFixedHeaderLength)
{
// This packet must extend past
for (var i = 0; i < gap; i++)
{
packet[packetPtr++] = frame[framePtr++];
}
flag = false;
continue;
}
for (var i = 0; i < PacketAccessor.PacketFixedHeaderLength; i++)
{
packet[packetPtr++] = frame[framePtr++];
}
var len = PacketAccessor.Length(packet) + 1; // real length is always the field + 1
gap = FrameAccessor.MPDUEnd - framePtr;
if (gap <= 0)
{
flag = false;
continue;
}
if (gap >= len)
{
// it'll fit
AppendToPacket(frame, len, packet, ref framePtr, ref packetPtr);
{
CheckPacketLength(packet, packetPtr); // Get rid of this after debugging
yield return(packet);
packetPtr = 0;
}
flag = (gap > len); // Maybe it fits just barely
continue;
}
// It won't fit
AppendToPacket(frame, gap, packet, ref framePtr, ref packetPtr);
flag = false;
}
}
break;
}
}
// Should there be a yield break here?
}
// This calculates that start, stop and a pseudo-start that's after 2014 (heuristically trying to find
// times when the clock had been jammed).
public void LoadStats()
{
var jamfilter = TimeUtilities.DateTimeToTime42(new DateTime(2014, 1, 1));
//var endFilter = TimeUtilities.DateTimeToTime42(new DateTime(2018, 1, 1));
const long endFilter = 37226859724800L;
try
{
var dict = new Dictionary <int, int>();
var begin = Int64.MaxValue;
var jam = Int64.MaxValue;
var end = Int64.MinValue;
var count = 0;
foreach (var packet in Iterator())
{
count++;
var apid = PacketAccessor.APID(packet);
var timestamp = PacketAccessor.Time42(packet);
if (timestamp == 0L)
{
continue; // ignore CLCW packets, no timestamp
}
if (timestamp < begin)
{
begin = timestamp;
}
if (timestamp < endFilter)
{
if (timestamp > end)
{
end = timestamp;
}
if (timestamp < jam && timestamp > jamfilter)
{
jam = timestamp;
}
}
int prev;
if (dict.TryGetValue(apid, out prev))
{
dict[apid] = prev + 1;
}
else
{
dict.Add(apid, 1);
}
}
if (count > 0)
{
_BeginTime42 = begin;
_EndTime42 = end;
_BeginTime42Jammed = jam == long.MaxValue ? jamfilter : jam;
}
else
{
_BeginTime42 = 0L;
_EndTime42 = 0L;
_BeginTime42Jammed = 0L;
}
_PacketCount = count;
_isFullyReadable = true;
_ApidCount = dict;
_Changed = true;
_BeginString = TimeUtilities.Time42ToString(_BeginTime42);
_EndString = TimeUtilities.Time42ToString(_EndTime42);
_JammedString = TimeUtilities.Time42ToString(_BeginTime42Jammed);
//Console.WriteLine("b={0}\nj={1}\ne={2} f={3}", _BeginString, _JammedString, _EndString, Name);
}
catch (Exception e1)
{
Console.WriteLine(e1);
Console.WriteLine(e1.StackTrace);
_isFullyReadable = false;
}
/* Console.WriteLine(Path);
* var a = _PacketCount;
* var b = Iterator().Count();
* var c = _ApidCount.Sum(pair => pair.Value);
*
* Console.WriteLine("_PacketCount={0}", a);
* Console.WriteLine("Actual={0}", b);
* Console.WriteLine("_ApidCount={0}",c);
* if (a != b || b != c)
* Console.WriteLine("friday");
* //todo*/
}
public static PacketFileType GetFileTypeId(string path)
{
try
{
var ext = System.IO.Path.GetExtension(path);
ext = ext != null?ext.ToLower() : null;
if (".meta".Equals(ext, StringComparison.InvariantCultureIgnoreCase) ||
".dir".Equals(ext, StringComparison.InvariantCultureIgnoreCase) ||
".gz".Equals(ext, StringComparison.InvariantCultureIgnoreCase) ||
".itf".Equals(ext, StringComparison.InvariantCultureIgnoreCase))
{
return(PacketFileType.UNRECOGNIZED_FILE);
}
if (".H".Equals(ext, StringComparison.InvariantCultureIgnoreCase))
{
return(GetFileTypeFromItosMetafile(path));
}
var fname = System.IO.Path.GetFileNameWithoutExtension(path);
var buf = new byte[8];
using (var s = File.OpenRead(path))
{
for (var i = 0; i < 8; i++)
{
buf[i] = (byte)s.ReadByte();
}
}
// CFE_PACKET_FILE
if (buf[0] == 'c' && buf[1] == 'F' && buf[2] == 'E')
{
return(PacketFileType.CFE_PACKET_FILE);
}
if (buf[0] == 'R' && buf[1] == 'A' && buf[2] == 'F' && buf[3] == '1')
{
return(PacketFileType.RAF_FRAME_FILE);
}
// ITOS_FRAME_FILE
if (buf[0] == 26 && buf[1] == 207 && (buf[4] & 248) == 72)
{
return(PacketFileType.ITOS_FRAME_FILE);
}
if (buf[0] == 0 && buf[1] == 0 && buf[2] == 252 && buf[3] == 29)
{
return(PacketFileType.ITOS_FRAME_FILE);
}
//for (var i = 0; i < 8; i++)
// Console.WriteLine(@"buf[{0}]={1}", i, buf[i]);
try
{
var i = 0;
// ITOS_PACKET_FILE
if ((buf[0] & 0xF8) == 0x08 || (buf[0] == 7 && 2001 == PacketAccessor.APID(buf)))
{
foreach (var p in (new ITOSPacketFile(path)).Iterator())
{
//if (!PacketAccessor.IsValidApid(PacketAccessor.APID(p)))
// goto tryAnno12PacketFile;
if (i++ > 30)
{
break;
}
}
return(PacketFileType.ITOS_PACKET_FILE);
}
//tryAnno12PacketFile:
i = 0;
foreach (var p in (new Anno12PacketFile(path)).Iterator())
{
//if (!PacketAccessor.IsValidApid(PacketAccessor.APID(p)))
// goto unrecognizedFile;
if (i++ > 30)
{
break;
}
}
return(PacketFileType.ANNO12_PACKET_FILE);
}
catch (Exception e)
{
Console.WriteLine(@"Error reading telemetry database: {0}", e);
}
}
catch (IOException e)
{
Console.WriteLine(@"Error reading telemetry database: {0}", e);
}
//unrecognizedFile:
return(PacketFileType.UNRECOGNIZED_FILE);
}
public virtual IEnumerable <byte[]> Iterator()
{
var buffer = new byte[PacketAccessor.MaximumPacketSize];
using (var reader = new BinaryReader(File.Open(_Path, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
var readerlen = (int)reader.BaseStream.Length;
long readerpos = 0;
// Skip the header, if any
var headerLength = GetHeaderLength();
for (var i = 0; i < headerLength; i++)
{
reader.ReadByte();
}
readerpos += headerLength;
while (readerpos < readerlen)
{
_Position = readerpos;
// Read the packet header
var ptr = 0;
var toread = 6;
int read;
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
Console.WriteLine(@"Premature EOF: {0}", Path);
yield break;
}
toread -= read;
ptr += read;
}
// Check whether we've run off the end.
if ((buffer[0] | buffer[1] | buffer[2]) == 0)
{
yield break;
}
var length = PacketAccessor.Length(buffer);
toread = 1 + length;
// Read the rest of the packet
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
Console.WriteLine(@"Premature EOF: {0}", Path);
yield break;
}
toread -= read;
ptr += read;
}
//totalLength = length + 7;
//byte[] packet = new byte[totalLength];
//Array.Copy(buffer, packet, totalLength);
//yield return packet;
yield return(buffer);
readerpos += ptr;
}
}
}
public override IEnumerable <byte[]> Iterator()
{
var buffer = new byte[PacketAccessor.MaximumPacketSize]; // Should I allocate the max size here?
var header = new byte[12];
using (var reader = new BinaryReader(File.Open(_Path, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
var readerlen = (int)reader.BaseStream.Length;
long readerpos = 0;
// Skip the header, if any
var headerLength = GetHeaderLength();
for (var i = 0; i < headerLength; i++)
{
reader.ReadByte();
}
readerpos += headerLength;
while (readerpos < readerlen)
{
int read;
// Skip the 12 byte annotation
var ptr = 0;
var toread = 12;
while (toread > 0)
{
read = reader.Read(header, ptr, toread);
if (read == 0)
{
if (ptr != 0)
{
Console.WriteLine(@"Premature EOF: {0}", Path);
}
yield break;
}
toread -= read;
ptr += read;
}
readerpos += 12;
_Position = readerpos;
// Read the packet header
ptr = 0;
toread = 6;
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
Console.WriteLine(@"Premature EOF: {0}", Path);
yield break;
}
toread -= read;
ptr += read;
}
// Check whether we've run off the end.
if (buffer[0] == 0 && buffer[1] == 0 && buffer[2] == 0)
{
yield break;
}
var length = PacketAccessor.Length(buffer);
toread = 1 + length;
// Read the rest of the packet
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
Console.WriteLine(@"Premature EOF: {0}", Path);
yield break;
}
toread -= read;
ptr += read;
}
if ((header[3] & ErrorMask) == 0)
{
yield return(buffer);
}
readerpos += ptr;
}
}
}
public static IEnumerable <byte[]> Iterator(IEnumerable <byte[]> frames)
{
var state = new FrameFileSequence.State[64];
for (var i = 0; i < 63; i++)
{
state[i] = FrameFileSequence.State.Idle;
}
var packet = new byte[64][];
const int bufferSize = 65516 + 12; // add some margin 65536 bytes of data plus 6 of header + 6 of timestamp
var frameCounter = new int[64]; // initial values of 0
var packetPtr = new int[64]; // copy pointer into the packet array
var framePtr = new int[64]; // copy pointer into the frame array
foreach (var frame in frames)
{
var vc = FrameAccessor.VirtualChannel(frame);
if (packet[vc] == null)
{
packet[vc] = new byte[bufferSize];
}
//Console.WriteLine("VirtualChannel={0}", vc);
//if (vc != 1) continue;
var frameCount = FrameAccessor.FrameCount(frame);
//Console.WriteLine("FrameCount={0} State={1}", frameCounter[vc], state[vc]);
var firstPacketPtr = FrameAccessor.FirstHeaderPointer(frame);
switch (state[vc])
{
case FrameFileSequence.State.Idle:
if (firstPacketPtr == FrameAccessor.FirstHeaderPointerOverflow)
{
break; // No packet starts here
}
framePtr[vc] = firstPacketPtr + FrameAccessor.MPDUStart;
state[vc] = FrameFileSequence.State.Running;
frameCounter[vc] = frameCount;
packetPtr[vc] = 0;
goto case FrameFileSequence.State.Iterate;
case FrameFileSequence.State.Running:
frameCounter[vc]++;
if (frameCounter[vc] != frameCount && !(frameCount == 0 && frameCounter[vc] == 16777216))
{
var delta = frameCount - frameCounter[vc];
if (delta < 0)
{
delta += 16777216;
}
//if (LogLevel > 0)
// Console.WriteLine(@"Frame sequence error: expected {0}, saw {1}, {2} frames skipped", frameCounter[vc], frameCount, delta);
state[vc] = FrameFileSequence.State.Idle;
continue;
}
framePtr[vc] = FrameAccessor.MPDUStart;
if (firstPacketPtr == 0)
{
if (packetPtr[vc] > 0)
{
CheckPacketLength(packet[vc], packetPtr[vc]); // Get rid of this after debugging
yield return(packet[vc]);
packetPtr[vc] = 0;
}
goto case FrameFileSequence.State.Iterate;
}
if (firstPacketPtr == FrameAccessor.FirstHeaderPointerOverflow)
{
AppendToPacket(frame, FrameAccessor.MPDUPacketZoneLength, packet[vc], ref framePtr[vc],
ref packetPtr[vc]);
break;
}
if (firstPacketPtr == 2047)
{
// Idle Frame
break;
}
if (firstPacketPtr > 0)
{
AppendToPacket(frame, firstPacketPtr, packet[vc], ref framePtr[vc], ref packetPtr[vc]);
{
CheckPacketLength(packet[vc], packetPtr[vc]); // Get rid of this after debugging
yield return(packet[vc]);
packetPtr[vc] = 0;
}
goto case FrameFileSequence.State.Iterate;
}
break;
case FrameFileSequence.State.Iterate:
{
var flag = true;
var p = packet[vc];
while (flag)
{
var gap = FrameAccessor.MPDUEnd - framePtr[vc];
if (gap < PacketAccessor.PacketFixedHeaderLength)
{
// This packet must extend past
for (var i = 0; i < gap; i++)
{
p[packetPtr[vc]++] = frame[framePtr[vc]++];
}
flag = false;
continue;
}
for (var i = 0; i < PacketAccessor.PacketFixedHeaderLength; i++)
{
p[packetPtr[vc]++] = frame[framePtr[vc]++];
}
var len = PacketAccessor.Length(p) + 1; // real length is always the field + 1
gap = FrameAccessor.MPDUEnd - framePtr[vc];
if (gap <= 0)
{
flag = false;
continue;
}
if (gap >= len)
{
// it'll fit
AppendToPacket(frame, len, p, ref framePtr[vc], ref packetPtr[vc]);
{
CheckPacketLength(p, packetPtr[vc]); // Get rid of this after debugging
yield return(p);
packetPtr[vc] = 0;
}
flag = (gap > len); // Maybe it fits just barely
continue;
}
// It won't fit
AppendToPacket(frame, gap, p, ref framePtr[vc], ref packetPtr[vc]);
flag = false;
}
}
break;
}
}
// Should there be a yield break here?
}
