private async Task ProcessFile(IIgnite client, FileInfo fileInfo, FastPcapFileReaderDevice device)
{
device.Open();
var frameKeyProvider = new FrameKeyProvider();
var cache = CacheFactory.GetOrCreateCache <string, Artifact>(client, FrameCacheName ?? fileInfo.Name);
var flowTracker = new PacketFlowTracker(new FrameKeyProvider());
using (var dataStreamer = client.GetDataStreamer <string, Artifact>(cache.Name))
{
dataStreamer.AllowOverwrite = true;
dataStreamer.Receiver = new PacketFlowVisitor(new MergePacketFlowProcessor());
var cacheStoreTask = Task.CompletedTask;
var currentChunkBytes = 0;
var currentChunkNumber = 0;
RawCapture rawCapture = null;
while ((rawCapture = device.GetNextPacket()) != null)
{
currentChunkBytes += rawCapture.Data.Length + 4 * sizeof(int);
var frame = new FrameData
{
LinkLayer = (LinkLayerType)rawCapture.LinkLayerType,
Timestamp = rawCapture.Timeval.ToUnixTimeMilliseconds(),
Data = rawCapture.Data
};
flowTracker.ProcessFrame(frame);
if (flowTracker.TotalFrameCount == ChunkSize)
{
OnChunkLoaded(currentChunkNumber, currentChunkBytes);
cacheStoreTask = cacheStoreTask.ContinueWith(StreamData(dataStreamer, flowTracker.FlowTable, currentChunkNumber, currentChunkBytes));
flowTracker.Reset();
}
}
OnChunkLoaded(currentChunkNumber, currentChunkBytes);
cacheStoreTask = cacheStoreTask.ContinueWith(StreamData(dataStreamer, flowTracker.FlowTable, currentChunkNumber, currentChunkBytes));
await cacheStoreTask;
dataStreamer.Flush();
}
Console.WriteLine($"Stored flows: {cache.GetSize()}");
device.Close();
}
public override async Task Invoke()
{
var cfg = GetIgniteConfiguration();
cfg.PeerAssemblyLoadingMode = PeerAssemblyLoadingMode.CurrentAppDomain;
cfg.ClientMode = true;
using (var ignite = Ignition.Start(cfg))
{
ignite.GetCompute().Broadcast(new NotificationAction("Start streaming..."));
foreach (var fileInfo in SourceFiles)
{
OnFileOpened(fileInfo);
using (var device = new FastPcapFileReaderDevice(fileInfo.FullName))
{
await ProcessFile(ignite, fileInfo, device);
}
OnFileCompleted(fileInfo);
}
ignite.GetCompute().Broadcast(new NotificationAction("Done."));
}
}
