public async Task TestTimeWindowOperator() { var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap"); var sw = new Stopwatch(); sw.Start(); var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket); var wins = observable.TimeSpanWindow(t => t.Ticks, TimeSpan.FromSeconds(60)).Select(o => o.GroupBy(f => f.Packet.GetFlowKey())); var windowNumber = 0; var totalPackets = 0; await wins.ForEachAsync(async win => { Console.Write($"Window {++windowNumber}: "); await win.ForEachAsync(async flow => { var packets = 0; var octets = 0; var firstSeen = long.MaxValue; var lastSeen = long.MinValue; await flow.ForEachAsync(packet => { totalPackets++; packets++; octets += packet.Packet.TotalPacketLength; firstSeen = Math.Min(firstSeen, packet.Ticks); lastSeen = Math.Max(lastSeen, packet.Ticks); }); Console.Write("."); //Console.WriteLine($" Flow {flow.Key}: firstSeen={new DateTime(firstSeen)}, duration={new TimeSpan(lastSeen - firstSeen)}, packets={packets}, octets={octets}"); }); Console.WriteLine($"Packets = {totalPackets}"); }); }
public async Task TestTimeWindowConversationProcessor() { var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap"); var sw = new Stopwatch(); sw.Start(); var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacketAndKey); var windows = observable.TimeSpanWindow(t => t.Ticks, TimeSpan.FromSeconds(60)); var windowCount = 0; var totalPackets = 0; await windows.Do(_ => windowCount++).ForEachAsync(async window => { var flowProcessor = new NetFlowProcessor(); await window.Do(_ => totalPackets++).ForEachAsync(p => flowProcessor.OnNext(p)); // Get the results: Console.WriteLine($"# Window {windowCount}"); Console.WriteLine($"Flows = {flowProcessor.Count}, Packets = {totalPackets}"); Console.WriteLine(); Console.WriteLine("| Date first seen | Duration | Proto | Src IP Addr:Port | Dst IP Addr:Port | Packets | Bytes |"); Console.WriteLine("| --------------- | -------- | ----- | ---------------- | ---------------- | ------- | ----- |"); foreach (var flow in flowProcessor.GetConversations(flowProcessor.GetConversationKey)) { Console.WriteLine($"| {new DateTime(flow.Value.Value.FirstSeen)} | {new TimeSpan(flow.Value.Value.LastSeen - flow.Value.Value.FirstSeen)} | {flow.Key.FlowKey.ProtocolType} | {flow.Key.FlowKey.SourceIpAddress}:{flow.Key.FlowKey.SourcePort} | {flow.Key.FlowKey.DestinationIpAddress}:{flow.Key.FlowKey.DestinationPort} | {flow.Value.Value.Packets} | {flow.Value.Value.Octets} |"); } Console.WriteLine(); }); }
public async Task TestCountWindowOperator() { var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap"); var sw = new Stopwatch(); sw.Start(); var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket); var wins = observable.Window(100).Select(o => o.GroupBy(f => f.Packet.GetFlowKey())); var windowNumber = 0; await wins.ForEachAsync(async win => { Console.WriteLine($"Window {++windowNumber}"); await win.ForEachAsync(async flow => { var packets = 0; var octets = 0; var firstSeen = long.MaxValue; var lastSeen = long.MinValue; await flow.ForEachAsync(packet => { packets++; octets += packet.Packet.TotalPacketLength; firstSeen = Math.Min(firstSeen, packet.Ticks); lastSeen = Math.Max(lastSeen, packet.Ticks); }); Console.WriteLine($" {flow.Key}:firstSeen={firstSeen}, lastSeen={lastSeen}, packets={packets}, octets={octets}"); }); }); }
public async Task ExportPacketsWithFlowKey() { var packetCount = 0; var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket).Select(x => (x.Ticks, x.Packet, x.Packet.GetFlowKey())); await packets.ForEachAsync(packet => { packetCount++; }); }
public async Task ExportPackets() { var packetCount = 0; var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket); await packets.ForEachAsync(packet => { packetCount++; }); }
public async Task TestBasicTrill() { var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-16.pcap"); var sw = new Stopwatch(); sw.Start(); var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket); var streamable = observable.ToTemporalStreamable(f => f.Ticks); }
public async Task ExportFlows() { var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket); var flows = packets.GroupFlows(f => f.Packet.GetFlowKey()); var flowCount = 0; await flows.ApplyFlowProcessor(FlowProcessorFunc).ForEachAsync(_ => { flowCount++; }); }
public async Task ExportConversations() { var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket); var conversations = packets.GroupConversations(f => f.Packet.GetFlowKey(), f => GetConversationKey(f)); var conversationCount = 0; await conversations.ApplyFlowProcessor(ConversationProcessor).ForEachAsync(_ => { conversationCount++; }); }
public async Task ExportWindowedPackets() { var packetCount = 0; var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket); var windows = packets.TimeSpanWindow(t => t.Ticks, TimeSpan.FromMinutes(5)); await windows.ForEachAsync(async window => { await window.ForEachAsync(_ => { packetCount++; }); }); }
public async Task ExportWindowedFlowsByProcessor() { var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacketAndKey); var windows = packets.TimeSpanWindow(t => t.Ticks, TimeSpan.FromMinutes(5)); var windowCount = 0; var totalPackets = 0; await windows.ForEachAsync(async window => { windowCount++; var flowProcessor = new NetFlowProcessor(); await window.Do(_ => totalPackets++).ForEachAsync(p => flowProcessor.OnNext(p)); Console.WriteLine($"Window = {windowCount}, Flows = {flowProcessor.Count}, Packets = {totalPackets}"); }); }
public async Task ExportWindowedFlows() { var packets = SharpPcapReader.CreateObservable(dataset).Select(GetPacket); var windows = packets.TimeSpanWindow(t => t.Ticks, TimeSpan.FromMinutes(5)).Select(packets => packets.GroupFlows(f => f.Packet.GetFlowKey())); var windowCount = 0; var flowCount = 0; await windows.ForEachAsync(async window => { windowCount++; await window.ApplyFlowProcessor(FlowProcessorFunc).ForEachAsync(_ => { flowCount++; }); }); }
public async Task TestApplyFlowProcessor2() { var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap"); var sw = new Stopwatch(); sw.Start(); var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket); var windows = observable .TimeSpanWindow(t => t.Ticks, TimeSpan.FromSeconds(60)) .Select(packets => packets.GroupFlowsDictionary(packet => packet.Packet.GetFlowKey())); var windowNumber = 0; var totalFlows = 0; await windows.ForEachAsync(async win => { Console.WriteLine($"Window {++windowNumber}: "); await win.ApplyFlowProcessor(FlowProcessorFunc).Do(_ => totalFlows++).ForEachAsync(flowStr => { Console.WriteLine(flowStr); }); }); Console.WriteLine($"All done, flows = {totalFlows} [{sw.Elapsed}]"); }
public async Task TestObservableFromReader() { var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap"); var sw = new Stopwatch(); sw.Start(); var observable = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket); var packetCount = 0; var flows = new Dictionary <FlowKey, int>(); void UpdatePacket(Packet p) { packetCount++; var fk = p.GetFlowKey(); flows.TryGetValue(fk, out var count); flows[fk] = count++; } await observable.ForEachAsync(f => UpdatePacket(f.Packet)); Console.WriteLine($"- Packets = {packetCount} [{sw.Elapsed}]"); Console.WriteLine($"- Flows = {flows.Count}"); }
public async Task TestCreateConversations() { var pcapPath = Path.Combine(TestEnvironment.DataPath, "testbed-32.pcap"); var sw = new Stopwatch(); sw.Start(); var source = SharpPcapReader.CreateObservable(pcapPath).Select(TestHelperFunctions.GetPacket); // get windows of flows: var windows = source .TimeSpanWindow(packet => packet.Ticks, TimeSpan.FromSeconds(60)) .Select(window => window.GroupConversations(packet => packet.Packet.GetFlowKey(), flowKey => GetConversationKey(flowKey))); var windowNumber = 0; await windows.ForEachAsync(async win => { Console.WriteLine($"Window {++windowNumber}: "); await win.ApplyFlowProcessor(ConversationProcessor).ForEachAsync(flowStr => { Console.WriteLine(flowStr); }); }); Console.WriteLine($"All done [{sw.Elapsed}]"); }