public async Task Invoke()
{
var cfg = new IgniteConfiguration
{
JvmOptions = new[] { //"-Xms256m",
$"-Xmx{maxOnHeap}m",
"-XX:+AlwaysPreTouch",
"-XX:+UseG1GC",
"-XX:+ScavengeBeforeFullGC",
"-XX:+DisableExplicitGC",
$"-XX:MaxDirectMemorySize={maxOffHeap}m"
},
DiscoverySpi = new Apache.Ignite.Core.Discovery.Tcp.TcpDiscoverySpi
{
IpFinder = new TcpDiscoveryStaticIpFinder
{
Endpoints = new[] { $"{ClusterNode.Address}:{(ClusterNode.Port != 0 ? ClusterNode.Port : DEFAULT_PORT)}" }
},
}
};
Ignition.ClientMode = true;
using (var client = Ignition.Start(cfg))
{
foreach (var fileInfo in SourceFiles)
{
OnFileOpen?.Invoke(this, fileInfo);
using (var device = new FastPcapFileReaderDevice(fileInfo.FullName))
{
await ProcessFile(client, fileInfo, device);
}
OnFileCompleted?.Invoke(this, fileInfo);
}
}
}
private async Task ProcessFile(IIgnite client, FileInfo fileInfo, FastPcapFileReaderDevice device)
{
device.Open();
var frameKeyProvider = new FrameKeyProvider();
var cache = CacheFactory.GetOrCreateFrameCache(client, FrameCacheName ?? fileInfo.Name);
using (var dataStreamer = client.GetDataStreamer <FrameKey, FrameData>(cache.Name))
{
var frameIndex = 0;
var frameArray = new KeyValuePair <FrameKey, FrameData> [ChunkSize];
var cacheStoreTask = Task.CompletedTask;
var currentChunkBytes = 0;
var currentChunkNumber = 0;
RawCapture rawCapture = null;
while ((rawCapture = device.GetNextPacket()) != null)
{
currentChunkBytes += rawCapture.Data.Length + 4 * sizeof(int);
var frame = new FrameData
{
LinkLayer = (LinkLayerType)rawCapture.LinkLayerType,
Timestamp = rawCapture.Timeval.ToUnixTimeMilliseconds(),
Data = rawCapture.Data
};
var frameKey = new FrameKey(frameIndex, frameKeyProvider.GetKeyHash(frame));
frameArray[frameIndex % ChunkSize] = KeyValuePair.Create(frameKey, frame);
// Is CHUNK full?
if (frameIndex % ChunkSize == ChunkSize - 1)
{
OnChunkLoaded?.Invoke(this, currentChunkNumber, currentChunkBytes);
cacheStoreTask = cacheStoreTask.ContinueWith(CreateStoreAction(dataStreamer, frameArray, ChunkSize, currentChunkNumber, currentChunkBytes));
currentChunkNumber++;
currentChunkBytes = 0;
}
frameIndex++;
}
OnChunkLoaded?.Invoke(this, currentChunkNumber, currentChunkBytes);
cacheStoreTask = cacheStoreTask.ContinueWith(CreateStoreAction(dataStreamer, frameArray, frameIndex % ChunkSize, currentChunkNumber, currentChunkBytes));
await cacheStoreTask;
dataStreamer.Flush();
//dataStreamer.Close(false); // HACK: causes Exception in JVM.DLL
}
device.Close();
}
public override async Task Invoke()
{
var cfg = GetIgniteConfiguration();
Ignition.ClientMode = true;
using (var client = Ignition.Start(cfg))
{
foreach (var fileInfo in SourceFiles)
{
OnFileOpened(fileInfo);
using (var device = new FastPcapFileReaderDevice(fileInfo.FullName))
{
await ProcessFile(client, fileInfo, device);
}
OnFileCompleted(fileInfo);
}
}
}
private async Task ProcessFile(IIgnite client, FileInfo fileInfo, FastPcapFileReaderDevice device)
{
device.Open();
var frameKeyProvider = new FrameKeyProvider();
var cache = CacheFactory.GetOrCreateFrameCache(client, FrameCacheName ?? fileInfo.Name);
RawCapture rawCapture = null;
var frameIndex = 0;
var currentChunkBytes = 0;
var currentChunkNumber = 0;
while ((rawCapture = device.GetNextPacket()) != null)
{
currentChunkBytes += rawCapture.Data.Length + 4 * sizeof(int);
var frame = new FrameData
{
LinkLayer = (LinkLayerType)rawCapture.LinkLayerType,
Timestamp = rawCapture.Timeval.ToUnixTimeMilliseconds(),
Data = rawCapture.Data
};
var frameKey = new FrameKey(frameIndex, frameKeyProvider.GetKeyHash(frame));
var storedFrame = await cache.GetAsync(frameKey);
if (storedFrame == null)
{
OnErrorFrameOccured(fileInfo, frameIndex, null);
}
if (storedFrame != null && frame.Timestamp != storedFrame.Timestamp)
{
OnErrorFrameOccured(fileInfo, frameIndex, storedFrame);
}
if (frameIndex % ChunkSize == ChunkSize - 1)
{
OnChunkLoaded(currentChunkNumber, currentChunkBytes);
OnChunkStored(currentChunkNumber, currentChunkBytes);
currentChunkNumber++;
currentChunkBytes = 0;
}
frameIndex++;
}
device.Close();
}
public async Task Invoke()
{
var cfg = new IgniteClientConfiguration
{
Host = ClusterNode.Address.ToString(),
Port = ClusterNode.Port != 0 ? ClusterNode.Port : DEFAULT_PORT,
};
using (var client = Ignition.StartClient(cfg))
{
foreach (var fileInfo in SourceFiles)
{
OnFileOpen?.Invoke(this, fileInfo);
using (var device = new FastPcapFileReaderDevice(fileInfo.FullName))
{
await ProcessFile(client, fileInfo, device);
}
OnFileCompleted?.Invoke(this, fileInfo);
}
}
}
