/// <summary>
/// Filter the packet stream returned by Iterator() to those within a time range and having apids within a given set.
/// Optionally skip the first n packets. (This feature is used by table views when paging.)
/// </summary>
/// <param name="apids"></param>
/// <param name="startTime"></param>
/// <param name="stopTime"></param>
/// <param name="skip"></param>
/// <returns></returns>
public IEnumerable <byte[]> Iterator(int[] apids, long startTime, long stopTime, long skip = 0L)
{
var count = 0;
foreach (var p in Iterator())
{
if (skip >= ++count)
{
continue;
}
var t = PacketAccessor.Time42(p);
if (t < startTime)
{
continue;
}
if (stopTime < t)
{
yield break;
}
var apid = PacketAccessor.APID(p);
if (!apids.Contains(apid))
{
continue;
}
yield return(p);
}
}
public virtual IEnumerable <byte[]> Iterator(Int64 startTime, Int64 stopTime)
{
return(from p in Iterator()
let timestamp = PacketAccessor.Time42(p)
where timestamp >= startTime && timestamp <= stopTime
select p);
}
static void v1()
{
var options = new DbOptions()
.SetCreateIfMissing(true);
using (var db = RocksDb.Open(options, "db1"))
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var count = 0;
foreach (var p in PacketStreamUtilities.PacketsFromFileTree(DataDirectory))
{
var len = PacketAccessor.Length(p);
if (len < 5)
{
continue;
}
var timestamp = PacketAccessor.Time42(p);
var apid = PacketAccessor.APID(p);
var key = new PacketKey {
APID = (short)apid, Timestamp = timestamp
};
var keybuf = KeyToByteArray(key);
db.Put(keybuf, p);
count++;
}
stopwatch.Stop();
Console.WriteLine($"{count} packets; {count / (stopwatch.ElapsedMilliseconds / 1000d)} packets/sec");
}
}
public virtual IEnumerable <byte[]> Iterator(int[] apids, Int64 startTime, Int64 stopTime, long skip = 0L)
{
var counter = 0L;
foreach (var p in Iterator())
{
var timestamp = PacketAccessor.Time42(p);
if (timestamp < startTime || timestamp > stopTime)
{
continue;
}
var apid = PacketAccessor.APID(p);
var len = apids.Length;
for (var i = 0; i < len; i++)
{
if (apid == apids[i])
{
if (counter++ >= skip)
{
yield return(p);
}
break;
}
}
}
}
public void Handle(byte[] p)
{
if (Writer == null)
{
return;
}
var apid = PacketAccessor.APID(p);
_packetCounts[apid]++;
switch (Mode)
{
case PacketPrinterMode.SummaryLine:
WriteSummaryLine(p);
break;
case PacketPrinterMode.PacketContents:
WritePacketContents(p);
break;
case PacketPrinterMode.PacketCount:
WritePacketCount(p);
break;
case PacketPrinterMode.PacketBinary:
WritePacketBinary(p);
break;
}
}
public Double GetDouble()
{
var v = PacketAccessor.GetF12345678(Packet, Offset);
Offset += 8;
return(v);
}
public Single GetSingle()
{
var v = PacketAccessor.GetF1234(Packet, Offset);
Offset += 4;
return(v);
}
public Int64 GetInt64()
{
var v = PacketAccessor.GetI12345678(Packet, Offset);
Offset += 8;
return(v);
}
public Int32 GetInt32()
{
var v = PacketAccessor.GetI1234(Packet, Offset);
Offset += 4;
return(v);
}
public Int16 GetInt16()
{
var v = PacketAccessor.GetI12(Packet, Offset);
Offset += 2;
return(v);
}
public void Run()
{
var s = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
var target = IPAddress.Parse(_host);
var ep = new IPEndPoint(target, _port);
foreach (var packet in PacketFilter.BitsPerSecond(_bitsPerSecond, _source))
{
//Console.WriteLine(@"sending apid {0}", PacketAccessor.APID(packet));
s.SendTo(packet, PacketAccessor.Length(packet) + 7, SocketFlags.None, ep);
//Thread.Sleep(1);
}
}
/// <summary>
/// Guarantee unique byte arrays. Ignore packets too short to have a length field.
/// </summary>
/// <param name="source"></param>
/// <returns></returns>
public static IEnumerable <byte[]> UniqueByteArray(IEnumerable <byte[]> source)
{
foreach (var p in source)
{
var len = p.Length;
if (len < 6) // too short for a length field
{
continue;
}
var tocopy = Math.Min(len, PacketAccessor.Length(p) + 7);
var buf = new byte[tocopy];
Array.Copy(p, buf, tocopy);
yield return(buf);
}
}
/// <summary>
/// Filter the stream of packets returned by Iterator() to those within a time range
/// </summary>
/// <param name="startTime"></param>
/// <param name="stopTime"></param>
/// <returns></returns>
public IEnumerable <byte[]> Iterator(long startTime, long stopTime)
{
foreach (var p in Iterator())
{
var t = PacketAccessor.Time42(p);
if (t < startTime)
{
continue;
}
if (stopTime < t)
{
yield break;
}
yield return(p);
}
}
private void WritePacketContents(byte[] p)
{
if (!_firstLineWritten)
{
Writer.WriteLine(@"Timestamp APID SeqCount PktLen Packet_Name");
_firstLineWritten = true;
}
var apid = PacketAccessor.APID(p);
var seqnum = PacketAccessor.SequenceCount(p);
var pktlen = PacketAccessor.Length(p);
var timestamp = TimeUtilities.SecondaryHeaderToString(PacketAccessor.SecondaryHeader(p));
var packetName = TelemetryDictionary?.GetPacket(apid)?.Name;
if (packetName == null)
{
packetName = String.Empty;
}
Writer.WriteLine(@"{0} {1:####} {2:#####} {3:#####} {4}", timestamp, apid, seqnum, pktlen, packetName);
var pkt = TelemetryDictionary?.GetPacket(apid);
if (pkt == null)
{
return;
}
_decomRows.Clear();
int nameWidth = 0, engWidth = 0, rawWidth = 0;
foreach (var point in pkt.Points)
{
var name = point.Name;
var raw = point.GetRawValue(p).ToString();
var eng = point.GetValue(p).ToString();
nameWidth = Math.Max(nameWidth, name.Length);
engWidth = Math.Max(engWidth, eng.Length);
rawWidth = Math.Max(rawWidth, raw.Length);
_decomRows.Add(new string[] { name, eng, raw });
}
foreach (var row in _decomRows)
{
Console.WriteLine(" " + row[0].PadRight(nameWidth) + " = " + row[1].PadLeft(engWidth) + " (eng), " + row[2].PadLeft(rawWidth) + " (raw)");
}
}
void AddPackets(RocksDb db, short[] apidCounts)
{
var addTimes = new List <long>();
var elapsedStopwatch = new Stopwatch();
var addStopwatch = new Stopwatch();
elapsedStopwatch.Start();
var count = 0;
foreach (var p in PacketStreamUtilities.PacketsFromFileTree(DataDirectory))
{
var len = PacketAccessor.Length(p);
if (len < 5)
{
continue;
}
var timestamp = PacketAccessor.Time42(p);
var apid = PacketAccessor.APID(p);
var key = new PacketKey {
APID = (short)apid, Timestamp = timestamp
};
var keybuf = KeyToByteArray(key);
addStopwatch.Reset();
addStopwatch.Start();
db.Put(keybuf, p);
addStopwatch.Stop();
addTimes.Add(addStopwatch.ElapsedTicks);
apidCounts[apid]++;
count++;
}
elapsedStopwatch.Stop();
Console.WriteLine($"{count} packets; {count / (elapsedStopwatch.ElapsedMilliseconds / 1000d)} packets/sec");
isFinished = true;
addTimes.Sort();
var startAt = Math.Max(0, addTimes.Count - 1000);
for (var i = startAt; i < addTimes.Count; i++)
{
Console.WriteLine((new TimeSpan(addTimes[i])).TotalSeconds);
}
}
private void WriteSummaryLine(byte[] p)
{
if (!_firstLineWritten)
{
Writer.WriteLine(@"Timestamp APID SeqCount PktLen Packet_Name");
_firstLineWritten = true;
}
var apid = PacketAccessor.APID(p);
var seqnum = PacketAccessor.SequenceCount(p);
var pktlen = PacketAccessor.Length(p);
var secondaryHeader = PacketAccessor.SecondaryHeader(p);
var timestamp = TimeUtilities.SecondaryHeaderToString(secondaryHeader);
var packetName = TelemetryDictionary?.GetPacket(apid)?.Name;
if (packetName == null)
{
packetName = String.Empty;
}
Writer.WriteLine(@"{0} {1:####} {2:#####} {3:#####} {4} #x{5:X16}", timestamp, apid, seqnum, pktlen, packetName, secondaryHeader);
}
void v2()
{
var options = new DbOptions()
.SetCreateIfMissing(true);
var addTimes = new List <long>();
using (var db = RocksDb.Open(options, "db1"))
{
var elapsedStopwatch = new Stopwatch();
var addStopwatch = new Stopwatch();
elapsedStopwatch.Start();
var count = 0;
foreach (var p in PacketStreamUtilities.PacketsFromFileTree(DataDirectory))
{
var len = PacketAccessor.Length(p);
if (len < 5)
{
continue;
}
var timestamp = PacketAccessor.Time42(p);
var apid = PacketAccessor.APID(p);
var key = new PacketKey {
APID = (short)apid, Timestamp = timestamp
};
var keybuf = KeyToByteArray(key);
addStopwatch.Reset();
addStopwatch.Start();
db.Put(keybuf, p);
addStopwatch.Stop();
addTimes.Add(addStopwatch.ElapsedTicks);
count++;
}
elapsedStopwatch.Stop();
Console.WriteLine($"{count} packets; {count / (elapsedStopwatch.ElapsedMilliseconds / 1000d)} packets/sec");
}
addTimes.Sort();
foreach (var t in addTimes)
{
Console.WriteLine((new TimeSpan(t)).TotalSeconds);
}
}
public static IEnumerable <byte[]> BitsPerSecond(int bitsPerSecond, IEnumerable <byte[]> source)
{
var stopwatch = new Stopwatch();
var bytesPerMsec = bitsPerSecond / 8000D;
var totalBytes = 0L;
stopwatch.Start();
foreach (var p in source)
{
var lengthInBytes = PacketAccessor.Length(p) + 7;
totalBytes += lengthInBytes;
var targetElapsedMsec = (long)(totalBytes / bytesPerMsec);
var elapsedMsec = stopwatch.ElapsedMilliseconds;
var delta = (int)(targetElapsedMsec - elapsedMsec);
if (delta > 0)
{
Thread.Sleep(delta);
}
yield return(p);
}
}
public void AppendPacket(byte[] buf)
{
var apid = PacketAccessor.APID(buf);
var len = PacketAccessor.Length(buf) + 7;
Debug.Assert(apid < StreamPairs.Length);
if (IgnorePackets[apid])
{
return;
}
var pairs = StreamPairs[apid];
if (pairs == null)
{
var packets = Dictionary.GetPackets(apid);
if (packets.Count == 0)
{
IgnorePackets[apid] = true;
//Console.Error.WriteLine(@"Ignoring apid {0}", apid);
return;
}
pairs = new List <ParallelCSVWriterHelper>(packets.Count);
pairs.AddRange(packets.Select(packet => new ParallelCSVWriterHelper(this, packet)
{
Path = DataPath(packet, FilePrefix),
Buffer = new StringBuilder(BufferMaxSize),
BufferMax = BufferMaxSize,
WriteEngineeringValues = WriteEngineeringValues
}));
// foreach (var pkt in packets)
// Console.Error.WriteLine(@"Adding handler for {0:d4} {1} ", apid, pkt.Name);
StreamPairs[apid] = pairs;
AllPairs.AddRange(pairs);
}
foreach (var pair in pairs)
{
pair.AppendPacket(buf, len);
}
}
public void AppendPacket(byte[] packet, int len)
{
if (!FirstLineHasBeenWritten)
{
WriteFirstLine();
}
if (FillPointer + len >= BufferMax)
{
Flush();
}
var points = Points;
var apid = PacketAccessor.APID(packet);
var timestamp = PacketAccessor.Time42(packet);
if (AddTimestamp)
{
Buffer.Append(TimeUtilities.Time42ToString(timestamp));
Buffer.Append(@", ");
}
if (AddLoganTimestamp)
{
Buffer.Append(TimeUtilities.Time42ToLogan(timestamp));
Buffer.Append(@", ");
}
if (AddStkTimestamp)
{
Buffer.Append(TimeUtilities.Time42ToSTK(timestamp));
Buffer.Append(@", ");
}
if (AddTimestampEt)
{
Buffer.Append(TimeUtilities.Time42ToET(timestamp));
Buffer.Append(@", ");
}
if (AddTimestampSeconds)
{
Buffer.Append(TimeUtilities.Time42ToSeconds(timestamp));
Buffer.Append(@", ");
}
if (AddTimestampSubseconds)
{
Buffer.Append(TimeUtilities.Time42ToSubseconds(timestamp));
Buffer.Append(@", ");
}
if (AddTimestampSubsecondsFloat)
{
Buffer.Append(TimeUtilities.Time42ToSubsecondsFloat(timestamp));
Buffer.Append(@", ");
}
if (AddTimestampLong)
{
Buffer.Append(timestamp);
Buffer.Append(@", ");
}
if (AddApid)
{
Buffer.Append(apid);
Buffer.Append(@", ");
}
for (var i = 0; i < points.Count; i++)
{
var point = points[i];
var v = WriteEngineeringValues ? point.GetValue(packet) : point.GetRawValue(packet);
if (i > 0)
{
Buffer.Append(", ");
}
Buffer.Append(v);
}
Buffer.AppendLine();
}
public void GenerateSeqprintsFromSpecs(IPacketFile source, List <SeqPrintSpec> specs)
{
StreamWriter[] writers = null;
try
{
writers = specs.Select(s => new StreamWriter(s.Filename, true)).ToArray();
// Write header lines
for (var j = 0; j < writers.Count(); j++)
{
for (var header = 0; header < specs[j].Points.Count; header++)
{
if (header > 0)
{
writers[j].Write(',');
}
writers[j].Write(specs[j].Points[header].Point.Name);
}
}
foreach (var w in writers)
{
w.WriteLine();
}
// Set up arrays for persistance
var persistance = specs.Select(spec => new dynamic[spec.Points.Count]).ToArray();
// Set up arrays for apid filtering
var apids = specs.Select(spec =>
{
var set = new HashSet <int>();
foreach (var pspec in spec.Points)
{
set.Add(pspec.Point.APID);
}
return(set.ToArray());
}).ToArray();
var set1 = new HashSet <int>();
foreach (var x in apids.SelectMany(ary => ary))
{
set1.Add(x);
}
var allApids = set1.ToArray();
// Write the seqprint
foreach (var packet in source.Iterator())
{
var apid = PacketAccessor.APID(packet);
if (!allApids.Contains(apid))
{
continue;
}
//var dt = TimeUtilities.Time42ToDateTime(PacketAccessor.Time42(packet));
//var timestamp = dt.ToString("YY-mm-dd HH:mm:ss.mmmm");
for (var i = 0; i < specs.Count; i++)
{
if (!apids[i].Contains(apid))
{
continue;
}
var spec = specs[i];
for (var j = 0; j < spec.Points.Count; j++)
{
var point = spec.Points[j];
dynamic v;
if (point.Point.APID != apid)
{
v = persistance[i][j] ?? "null";
}
else
{
v = point.Raw ? point.Point.GetRawValue(packet) : point.Point.GetValue(packet);
if (point.Function != null)
{
v = point.Function(v);
}
persistance[i][j] = v;
}
writers[i].Write(point.Format, v);
}
writers[i].WriteLine();
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
if (writers != null)
{
foreach (var w in writers)
{
w.Close();
}
}
}
}
private bool IsValidPacket(byte[] p)
{
// Just checks length for now
return(p != null && (p.Length >= 6 && p.Length == PacketAccessor.Length(p) + 7));
}
/// <summary>
/// Read a single packet. This exists so that a try/catch block can be set up (which interacts badly with yields)
/// </summary>
/// <param name="reader"></param>
/// <param name="buffer"></param>
/// <returns></returns>
protected bool ReadPacket(BinaryReader reader, byte[] buffer)
{
try
{
// Read the packet header
var ptr = 0;
var toread = 6;
int read;
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
if (OnPrematureEOF != null)
{
OnPrematureEOF();
}
return(false);
}
toread -= read;
ptr += read;
}
// Check whether we've run off the end. Valid packets can't start with three 0's
//TODO: There's a potential bug here. This only recognizes the end when we get to a header.
//The end of the previous packet might be missing.
if ((buffer[0] | buffer[1] | buffer[2]) == 0)
{
if (OnFoundZeros != null)
{
OnFoundZeros();
}
return(false);
}
var length = PacketAccessor.Length(buffer);
toread = 1 + length;
// Read the rest of the packet
while (toread > 0)
{
read = reader.Read(buffer, ptr, toread);
if (read == 0)
{
if (OnPrematureEOF != null)
{
OnPrematureEOF();
}
return(false);
}
toread -= read;
ptr += read;
}
Position += ptr;
return(true);
}
catch (Exception e)
{
if (OnError != null)
{
OnError(e);
}
return(false);
}
}
