private IEnumerable <LabeledPacketData> PrepareSinglePacketData(string[] packetFiles, string[] maliciousIps)
{
Stopwatch stopwatch = new Stopwatch();
Console.WriteLine("=============== Preparing Single Packet Data ===============");
Console.WriteLine();
Console.WriteLine($"Start data prep using {packetFiles.Length} packet files, {maliciousIps.Length} malicious Ips.");
Console.WriteLine($"Files Being Processed:");
foreach (var file in packetFiles)
{
Console.WriteLine($"\t{file}");
}
stopwatch.Start();
var ipv4Packets = packetFiles
.SelectMany(x => Pcap.FromFile(x).Packets)
.Where(x => x.Body is EthernetFrame && ((EthernetFrame)x.Body).EtherType == EthernetFrame.EtherTypeEnum.Ipv4)
.Select(x => (TimeStamp: x.TsSec, Packet: ((Ipv4Packet)((EthernetFrame)x.Body).Body)))
.ToArray();
var largestPacketSize = ipv4Packets.Max(x => x.Packet.M_RawBody.Length);
Console.WriteLine($"Largest Packet Size: {largestPacketSize} bytes.");
var packets = ipv4Packets
.Select(x =>
{
var labeledPacket = new LabeledPacketData()
{
SrcIp = x.Packet.SrcIpAddrStr,
IsMalicious = maliciousIps.Contains(x.Packet.SrcIpAddrStr),
PacketLength = x.Packet.M_RawBody.Length,
PacketBody = new byte[largestPacketSize]
};
Array.Copy(x.Packet.M_RawBody, labeledPacket.PacketBody, x.Packet.M_RawBody.Length);
return(labeledPacket);
});
stopwatch.Stop();
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("=============== End of data prep ===============");
return(packets);
}
private IEnumerable <SummaryPacketData> PrepareSummaryData(string[] packetFiles, int[] includedPorts, string[] maliciousIps, int windowSize = 5)
{
Stopwatch stopwatch = new Stopwatch();
Console.WriteLine("=============== Preparing Summary Window Packet Data ===============");
Console.WriteLine();
Console.WriteLine($"Start data prep using {packetFiles.Length} packet files, {includedPorts.Length} included ports, {maliciousIps?.Length ?? 0} malicious Ips and window size of {windowSize} seconds.");
Console.WriteLine($"Files Being Processed:");
foreach (var file in packetFiles)
{
Console.WriteLine($"\t{file}");
}
stopwatch.Start();
var ipv4Packets = packetFiles
.SelectMany(x => Pcap.FromFile(x).Packets)
.Where(x => x.Body is EthernetFrame && ((EthernetFrame)x.Body).EtherType == EthernetFrame.EtherTypeEnum.Ipv4)
.Select(x => (TimeStamp: x.TsSec, Packet: ((Ipv4Packet)((EthernetFrame)x.Body).Body)))
.ToArray();
Console.WriteLine($"Done Extracting Packets for cap files. Found {ipv4Packets.Length} IPv4 packets");
var dataTimeSpan = (Min : ipv4Packets.Select(x => x.TimeStamp).Min(), Max : ipv4Packets.Select(x => x.TimeStamp).Max());
var totalWindows = Math.Ceiling((float)((dataTimeSpan.Max - dataTimeSpan.Min) / windowSize));
Console.WriteLine($"Date time range for packets is {dataTimeSpan.Min} - {dataTimeSpan.Max} ({totalWindows} total window chunks)");
var uniqueIps = ipv4Packets
.SelectMany(x => new[] { x.Packet.SrcIpAddrStr, x.Packet.DstIpAddrStr })
.Distinct();
Console.WriteLine($"{uniqueIps.Count()} unique IP's found in the data. Starting packet summarization");
var data = ipv4Packets
.GroupBy((x => (x.TimeStamp - dataTimeSpan.Min - ((x.TimeStamp - dataTimeSpan.Min) % windowSize)) / windowSize))
.AsParallel()
.SelectMany(window =>
window
.SelectMany(x => new[] { x.Packet.SrcIpAddrStr, x.Packet.DstIpAddrStr })
.Distinct()
.AsParallel()
.Select(ip =>
{
var packetData = window.Where(x => x.Packet.DstIpAddrStr == ip || x.Packet.SrcIpAddrStr == ip);
var portData = includedPorts.SelectMany(x =>
{
var portPackets = packetData.Where(y => Convert.ToInt32(y.Packet.Protocol) == x);
return(portPackets.Any() ? new[]
{
(float)x,
((float)portPackets.Average(p => p.Packet.TotalLength)),
(float)portPackets.Where(p => p.Packet.DstIpAddrStr == ip).Count(),
(float)portPackets.Where(p => p.Packet.SrcIpAddrStr == ip).Count()
} : new[] { 0.0f, 0.0f, 0.0f, 0.0f });
}).ToArray();
return((packetData.Count() == 0 || portData.Sum() == 0) ? null : new SummaryPacketData()
{
WindowSize = windowSize,
Ip = ip,
IsMalicious = maliciousIps != null ? maliciousIps.Contains(ip) : false,
PortData = portData
});
})
)
.Where(x => x != null)
.ToArray();
if (maliciousIps != null)
{
var malIps = data.Where(x => x.IsMalicious).Count();
Console.WriteLine($"Data summarization done, Labels found. {malIps} packet windows out of {data.Count()} labels as malicious.");
}
stopwatch.Stop();
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("=============== End of data prep ===============");
return(data);
}