public async Task TestTimeWindowOperator()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap");
var sw = new Stopwatch();
sw.Start();
var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket);
var wins = observable.TimeSpanWindow(t => t.Ticks, TimeSpan.FromSeconds(60)).Select(o => o.GroupBy(f => f.Packet.GetFlowKey()));
var windowNumber = 0;
var totalPackets = 0;
await wins.ForEachAsync(async win =>
{
Console.Write($"Window {++windowNumber}: ");
await win.ForEachAsync(async flow =>
{
var packets = 0;
var octets = 0;
var firstSeen = long.MaxValue;
var lastSeen = long.MinValue;
await flow.ForEachAsync(packet =>
{
totalPackets++;
packets++;
octets += packet.Packet.TotalPacketLength;
firstSeen = Math.Min(firstSeen, packet.Ticks);
lastSeen = Math.Max(lastSeen, packet.Ticks);
});
Console.Write(".");
//Console.WriteLine($" Flow {flow.Key}: firstSeen={new DateTime(firstSeen)}, duration={new TimeSpan(lastSeen - firstSeen)}, packets={packets}, octets={octets}");
});
Console.WriteLine($"Packets = {totalPackets}");
});
}
public async Task TestTimeWindowConversationProcessor()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap");
var sw = new Stopwatch();
sw.Start();
var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacketAndKey);
var windows = observable.TimeSpanWindow(t => t.Ticks, TimeSpan.FromSeconds(60));
var windowCount = 0;
var totalPackets = 0;
await windows.Do(_ => windowCount++).ForEachAsync(async window =>
{
var flowProcessor = new NetFlowProcessor();
await window.Do(_ => totalPackets++).ForEachAsync(p => flowProcessor.OnNext(p));
// Get the results:
Console.WriteLine($"# Window {windowCount}");
Console.WriteLine($"Flows = {flowProcessor.Count}, Packets = {totalPackets}");
Console.WriteLine();
Console.WriteLine("| Date first seen | Duration | Proto | Src IP Addr:Port | Dst IP Addr:Port | Packets | Bytes |");
Console.WriteLine("| --------------- | -------- | ----- | ---------------- | ---------------- | ------- | ----- |");
foreach (var flow in flowProcessor.GetConversations(flowProcessor.GetConversationKey))
{
Console.WriteLine($"| {new DateTime(flow.Value.Value.FirstSeen)} | {new TimeSpan(flow.Value.Value.LastSeen - flow.Value.Value.FirstSeen)} | {flow.Key.FlowKey.ProtocolType} | {flow.Key.FlowKey.SourceIpAddress}:{flow.Key.FlowKey.SourcePort} | {flow.Key.FlowKey.DestinationIpAddress}:{flow.Key.FlowKey.DestinationPort} | {flow.Value.Value.Packets} | {flow.Value.Value.Octets} |");
}
Console.WriteLine();
});
}
public async Task TestCountWindowOperator()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap");
var sw = new Stopwatch();
sw.Start();
var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket);
var wins = observable.Window(100).Select(o => o.GroupBy(f => f.Packet.GetFlowKey()));
var windowNumber = 0;
await wins.ForEachAsync(async win =>
{
Console.WriteLine($"Window {++windowNumber}");
await win.ForEachAsync(async flow =>
{
var packets = 0;
var octets = 0;
var firstSeen = long.MaxValue;
var lastSeen = long.MinValue;
await flow.ForEachAsync(packet =>
{
packets++;
octets += packet.Packet.TotalPacketLength;
firstSeen = Math.Min(firstSeen, packet.Ticks);
lastSeen = Math.Max(lastSeen, packet.Ticks);
});
Console.WriteLine($" {flow.Key}:firstSeen={firstSeen}, lastSeen={lastSeen}, packets={packets}, octets={octets}");
});
});
}
public async Task ExportPacketsWithFlowKey()
{
var packetCount = 0;
var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket).Select(x => (x.Ticks, x.Packet, x.Packet.GetFlowKey()));
await packets.ForEachAsync(packet =>
{
packetCount++;
});
}
public async Task ExportPackets()
{
var packetCount = 0;
var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket);
await packets.ForEachAsync(packet =>
{
packetCount++;
});
}
public async Task TestBasicTrill()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-16.pcap");
var sw = new Stopwatch();
sw.Start();
var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket);
var streamable = observable.ToTemporalStreamable(f => f.Ticks);
}
public async Task ExportConversations()
{
var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket);
var conversations = packets.GroupConversations(f => f.Packet.GetFlowKey(), f => GetConversationKey(f));
var conversationCount = 0;
await conversations.ApplyFlowProcessor(ConversationProcessor).ForEachAsync(_ =>
{
conversationCount++;
});
}
public async Task ExportFlows()
{
var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket);
var flows = packets.GroupFlows(f => f.Packet.GetFlowKey());
var flowCount = 0;
await flows.ApplyFlowProcessor(FlowProcessorFunc).ForEachAsync(_ =>
{
flowCount++;
});
}
public async Task ExportWindowedPackets()
{
var packetCount = 0;
var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket);
var windows = packets.TimeSpanWindow(t => t.Ticks, TimeSpan.FromMinutes(5));
await windows.ForEachAsync(async window =>
{
await window.ForEachAsync(_ =>
{
packetCount++;
});
});
}
public void Setup()
{
_conversationTable = FasterConversationTable.Create($"{dataset}.{loadTableRun}", 3000000);
using (var loader = _conversationTable.GetStreamer())
using (var pcapReader = new SharpPcapReader(dataset))
{
while (pcapReader.GetNextFrame(out var rawFrame))
{
loader.AddFrame(rawFrame);
}
loader.Close();
}
}
public async Task ExportWindowedFlowsByProcessor()
{
var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacketAndKey);
var windows = packets.TimeSpanWindow(t => t.Ticks, TimeSpan.FromMinutes(5));
var windowCount = 0;
var totalPackets = 0;
await windows.ForEachAsync(async window =>
{
windowCount++;
var flowProcessor = new NetFlowProcessor();
await window.Do(_ => totalPackets++).ForEachAsync(p => flowProcessor.OnNext(p));
Console.WriteLine($"Window = {windowCount}, Flows = {flowProcessor.Count}, Packets = {totalPackets}");
});
}
public async Task ExportWindowedFlows()
{
var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket);
var windows = packets.TimeSpanWindow(t => t.Ticks, TimeSpan.FromMinutes(5)).Select(packets => packets.GroupFlows(f => f.Packet.GetFlowKey()));
var windowCount = 0;
var flowCount = 0;
await windows.ForEachAsync(async window =>
{
windowCount++;
await window.ApplyFlowProcessor(FlowProcessorFunc).ForEachAsync(_ =>
{
flowCount++;
});
});
}
public async Task TestCreateFrameTable()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap");
var sw = new Stopwatch();
var dbPath = Path.GetFullPath(@"c:\temp\0001\");
if (Directory.Exists(dbPath))
{
Directory.Delete(dbPath, true);
}
using var flowTable = FasterFrameTable.Create(dbPath, framesCapacity: 1700000);
var frameNumber = 0;
sw.Restart();
using (var loader = flowTable.GetStreamer())
using (var pcapReader = new SharpPcapReader(pcapPath))
{
while (pcapReader.GetNextFrame(out var rawFrame))
{
frameNumber++;
loader.AddFrame(rawFrame);
}
loader.Close();
}
Console.WriteLine($"--- LOADED --- [{sw.Elapsed}]");
flowTable.SaveChanges();
Console.WriteLine($"--- COMMITED --- [{sw.Elapsed}]");
sw.Restart();
var observable = flowTable.GetObservable(TestHelperFunctions.FrameProcessor);
var packetCount = 0;
var flows = new Dictionary <FlowKey, int>();
void UpdatePacket(Packet p)
{
packetCount++;
var fk = p.GetFlowKey();
flows.TryGetValue(fk, out var count);
flows[fk] = count++;
}
await observable.ForEachAsync(UpdatePacket);
Console.WriteLine($"- Packets = {packetCount} [{sw.Elapsed}]");
Console.WriteLine($"- Flows = {flows.Count}");
}
public void TestCreateTable()
{
var sw = new Stopwatch();
var pcapPath = Path.GetFullPath(@"data\PCAP\modbus.pcap");
var dbPath = Path.GetFullPath(@"c:\temp\0001\");
if (Directory.Exists(dbPath))
{
Directory.Delete(dbPath, true);
}
var flowTable = FasterConversationTable.Create(dbPath, framesCapacity: 1700000);
var frameNumber = 0;
sw.Restart();
using (var loader = flowTable.GetStreamer())
using (var pcapReader = new SharpPcapReader(pcapPath))
{
while (pcapReader.GetNextFrame(out var rawFrame))
{
frameNumber++;
loader.AddFrame(rawFrame);
}
loader.Close();
}
Console.WriteLine($"--- LOADED --- [{sw.Elapsed}]");
sw.Restart();
Console.WriteLine($"Convs= {flowTable.ConversationsCount} [{sw.Elapsed}]");
sw.Restart();
Console.WriteLine($"Frames= {flowTable.FramesCount} /{frameNumber} [{sw.Elapsed}]");
sw.Restart();
flowTable.SaveChanges();
Console.WriteLine($"--- COMMITED --- [{sw.Elapsed}]");
sw.Restart();
Console.WriteLine($"Frames= {flowTable.FramesCount} / {frameNumber} [{sw.Elapsed}]");
flowTable.Dispose();
}
public async Task TestApplyFlowProcessor2()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap");
var sw = new Stopwatch();
sw.Start();
var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket);
var windows = observable
.TimeSpanWindow(t => t.Ticks, TimeSpan.FromSeconds(60))
.Select(packets => packets.GroupFlowsDictionary(packet => packet.Packet.GetFlowKey()));
var windowNumber = 0;
var totalFlows = 0;
await windows.ForEachAsync(async win =>
{
Console.WriteLine($"Window {++windowNumber}: ");
await win.ApplyFlowProcessor(FlowProcessorFunc).Do(_ => totalFlows++).ForEachAsync(flowStr =>
{
Console.WriteLine(flowStr);
});
});
Console.WriteLine($"All done, flows = {totalFlows} [{sw.Elapsed}]");
}
public async Task TestObservableFromReader()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap");
var sw = new Stopwatch();
sw.Start();
var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket);
var packetCount = 0;
var flows = new Dictionary <FlowKey, int>();
void UpdatePacket(Packet p)
{
packetCount++;
var fk = p.GetFlowKey();
flows.TryGetValue(fk, out var count);
flows[fk] = count++;
}
await observable.ForEachAsync(f => UpdatePacket(f.Packet));
Console.WriteLine($"- Packets = {packetCount} [{sw.Elapsed}]");
Console.WriteLine($"- Flows = {flows.Count}");
}
public async Task TestCreateConversations()
{
var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap");
var sw = new Stopwatch();
sw.Start();
var source = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket);
// get windows of flows:
var windows = source
.TimeSpanWindow(packet => packet.Ticks, TimeSpan.FromSeconds(60))
.Select(window => window.GroupConversations(packet => packet.Packet.GetFlowKey(), flowKey => GetConversationKey(flowKey)));
var windowNumber = 0;
await windows.ForEachAsync(async win =>
{
Console.WriteLine($"Window {++windowNumber}: ");
await win.ApplyFlowProcessor(ConversationProcessor).ForEachAsync(flowStr =>
{
Console.WriteLine(flowStr);
});
});
Console.WriteLine($"All done [{sw.Elapsed}]");
}
