static void v1() { var options = new DbOptions() .SetCreateIfMissing(true); using (var db = RocksDb.Open(options, "db1")) { var stopwatch = new Stopwatch(); stopwatch.Start(); var count = 0; foreach (var p in PacketStreamUtilities.PacketsFromFileTree(DataDirectory)) { var len = PacketAccessor.Length(p); if (len < 5) { continue; } var timestamp = PacketAccessor.Time42(p); var apid = PacketAccessor.APID(p); var key = new PacketKey { APID = (short)apid, Timestamp = timestamp }; var keybuf = KeyToByteArray(key); db.Put(keybuf, p); count++; } stopwatch.Stop(); Console.WriteLine($"{count} packets; {count / (stopwatch.ElapsedMilliseconds / 1000d)} packets/sec"); } }
static byte[] KeyToByteArray(PacketKey key) { var r = new byte[10]; var apid = key.APID; r[0] = (byte)(apid >> 8 & 0xFF); r[1] = (byte)(apid & 0xFF); var timestamp = key.Timestamp; r[2] = (byte)(timestamp >> 56 & 0xFF); r[3] = (byte)(timestamp >> 48 & 0xFF); r[4] = (byte)(timestamp >> 40 & 0xFF); r[5] = (byte)(timestamp >> 32 & 0xFF); r[6] = (byte)(timestamp >> 24 & 0xFF); r[7] = (byte)(timestamp >> 16 & 0xFF); r[8] = (byte)(timestamp >> 8 & 0xFF); r[9] = (byte)(timestamp >> 0 & 0xFF); return(r); }
void AddPackets(RocksDb db, short[] apidCounts) { var addTimes = new List <long>(); var elapsedStopwatch = new Stopwatch(); var addStopwatch = new Stopwatch(); elapsedStopwatch.Start(); var count = 0; foreach (var p in PacketStreamUtilities.PacketsFromFileTree(DataDirectory)) { var len = PacketAccessor.Length(p); if (len < 5) { continue; } var timestamp = PacketAccessor.Time42(p); var apid = PacketAccessor.APID(p); var key = new PacketKey { APID = (short)apid, Timestamp = timestamp }; var keybuf = KeyToByteArray(key); addStopwatch.Reset(); addStopwatch.Start(); db.Put(keybuf, p); addStopwatch.Stop(); addTimes.Add(addStopwatch.ElapsedTicks); apidCounts[apid]++; count++; } elapsedStopwatch.Stop(); Console.WriteLine($"{count} packets; {count / (elapsedStopwatch.ElapsedMilliseconds / 1000d)} packets/sec"); isFinished = true; addTimes.Sort(); var startAt = Math.Max(0, addTimes.Count - 1000); for (var i = startAt; i < addTimes.Count; i++) { Console.WriteLine((new TimeSpan(addTimes[i])).TotalSeconds); } }
void v2() { var options = new DbOptions() .SetCreateIfMissing(true); var addTimes = new List <long>(); using (var db = RocksDb.Open(options, "db1")) { var elapsedStopwatch = new Stopwatch(); var addStopwatch = new Stopwatch(); elapsedStopwatch.Start(); var count = 0; foreach (var p in PacketStreamUtilities.PacketsFromFileTree(DataDirectory)) { var len = PacketAccessor.Length(p); if (len < 5) { continue; } var timestamp = PacketAccessor.Time42(p); var apid = PacketAccessor.APID(p); var key = new PacketKey { APID = (short)apid, Timestamp = timestamp }; var keybuf = KeyToByteArray(key); addStopwatch.Reset(); addStopwatch.Start(); db.Put(keybuf, p); addStopwatch.Stop(); addTimes.Add(addStopwatch.ElapsedTicks); count++; } elapsedStopwatch.Stop(); Console.WriteLine($"{count} packets; {count / (elapsedStopwatch.ElapsedMilliseconds / 1000d)} packets/sec"); } addTimes.Sort(); foreach (var t in addTimes) { Console.WriteLine((new TimeSpan(t)).TotalSeconds); } }