private IFlowTracker <FlowData> TrackFlows(ICache <FrameKey, FrameData> frameCache, ProgressRecord progressRecord)
{
var flowTracker = new FlowTracker(new FrameKeyProvider());
try
{
progressRecord.TotalFrames = frameCache.GetLocalSize();
Progress?.Report(progressRecord);
var framesCount = 0;
foreach (var frame in frameCache.GetLocalEntries())
{
flowTracker.ProcessFrame(frame.Value);
if (++framesCount % ProgressFrameBatch == 0)
{
progressRecord.CompletedFrames += ProgressFrameBatch;
Progress?.Report(progressRecord);
}
}
progressRecord.CompletedFrames += framesCount % ProgressFrameBatch;
Progress?.Report(progressRecord);
}
catch (Exception e)
{
Console.Error.WriteLine(e);
}
return(flowTracker);
}
private IFlowTracker <FlowData> TrackFlowsUsingQuery(ProgressRecord progressRecord)
{
var flowTracker = new FlowTracker(new FrameKeyProvider());
try
{
var cache = CacheFactory.GetOrCreateFrameCache(m_ignite, FrameCacheName);
var query = new ScanQuery <FrameKey, FrameData>();
progressRecord.TotalFrames = cache.GetLocalSize();
Progress?.Report(progressRecord);
var framesCount = 0;
foreach (var cursor in cache.Query(query))
{
flowTracker.ProcessFrame(cursor.Value);
if (++framesCount % ProgressFrameBatch == 0)
{
progressRecord.CompletedFrames += ProgressFrameBatch;
Progress?.Report(progressRecord);
}
}
progressRecord.CompletedFrames += framesCount % ProgressFrameBatch;
Progress?.Report(progressRecord);
}
catch (Exception e)
{
Console.Error.WriteLine(e);
}
return(flowTracker);
}
public FrameStreamVisitor(FlowTracker flowTracker)
{
m_flowTracker = flowTracker;
}
private int Process(string trainingFolder)
{
// training folder contains pcaps for each protocol. Each pcap has name corresponding to the protocol.
var statClassifier = new Statistical.StatisticalClassifier();
var portClassifier = new PortBased.PortBasedClassifier();
portClassifier.LoadConfiguration(null);
foreach (var capFile in Directory.EnumerateFiles(trainingFolder, "*.cap"))
{
using (var errorLog = new StreamWriter(File.Create(Path.ChangeExtension(capFile, "errors"))))
{
var protocolName = Path.GetFileNameWithoutExtension(capFile).ToLowerInvariant();
// read pcap, compute flows, merge to biflows, and train the classifier:
var device = new CaptureFileReaderDevice(capFile);
// Open the device for capturing
var flowTracker = new FlowTracker(new FrameKeyProvider());
device.Open();
RawCapture packet = null;
var frameIndex = 0;
while ((packet = device.GetNextPacket()) != null)
{
frameIndex++;
try
{
var frame = new Frame
{
LinkLayer = (LinkLayerType)packet.LinkLayerType,
Timestamp = ToUnixTimeMilliseconds(packet.Timeval),
Data = packet.Data
};
flowTracker.ProcessFrame(frame);
}
catch (Exception e)
{
errorLog.WriteLine($"[#{frameIndex}] {e.Message}");
}
}
var flowTable = flowTracker.FlowTable.Select(f => KeyValuePair.Create(f.Key, new FlowRecord <Statistical.FlowStatisticalVector> {
Flow = f.Value
}));
Conversation <FlowRecord <Statistical.FlowStatisticalVector> > getConversation(KeyValuePair <FlowKey, FlowRecord <Statistical.FlowStatisticalVector> > upflow)
{
var downflow = flowTracker.FlowTable.GetValueOrDefault(upflow.Key.SwapEndpoints());
return(new Conversation <FlowRecord <Statistical.FlowStatisticalVector> >
{
ConversationKey = upflow.Key,
Upflow = upflow.Value,
Downflow = new FlowRecord <Statistical.FlowStatisticalVector> {
Flow = downflow
}
});
}
// HACK: What if there is a single flow conversation, where src.port < dst.port ?
var conversations = flowTable.Where(f => f.Key.SourcePort > f.Key.DestinationPort).Select(f => getConversation(f)).ToList();
// TRAIN CLASSIFIERS FOR THE RECOGNIZED CONVERSATIONS:
foreach (var conversation in conversations)
{
// use port based classifier to get rough information on the flow in the pcap:
var protocol = portClassifier.Match(conversation);
Console.WriteLine($"{conversation.ConversationKey.ToString()} | {protocol.ProtocolName} ({protocol.Similarity})");
statClassifier.Train(protocolName, conversation);
}
device.Close();
}
}
return(0);
}