private void ReadFile()
{
var fileInfo = new FileInfo(fileName);
var offset = 0L;
while (offset < fileInfo.Length)
{
var size = Math.Min(fileInfo.Length - offset, batchSize);
var viewStream = memoryMappedFile.CreateViewStream(offset, size);
var chunk = new Chunk(offset, viewStream);
offset += viewStream.Length;
producingBag.Add(chunk);
}
producingBag.CompleteAdding();
}
private void Start()
{
producingBag = producer.StartProducing();
consumer.StartConsuming(consumingBag);
for (var i = 0; i < transformConcurrency; i++)
{
var task = new Task(() =>
{
while (producingBag.TryTake(out var chunk))
{
consumingBag.Add(mapper(chunk));
}
});
transformTasks.Add(task);
threadPool.RunTask(task);
}
}
private void ReadFile()
{
const int chunkLengthSize = sizeof(int);
const int initialOffsetSize = sizeof(long);
var fileInfo = new FileInfo(fileName);
using var memoryMappedFile = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open, null);
var offset = fileHeaderSize;
while (offset < fileInfo.Length)
{
using var tmpStream = memoryMappedFile.CreateViewStream(offset, chunkLengthSize);
var chunkLength = tmpStream.ReadInt32();
var viewStream =
memoryMappedFile.CreateViewStream(offset + chunkLengthSize, chunkLength + initialOffsetSize);
offset += viewStream.Length + 4;
bag.Add(viewStream);
}
bag.CompleteAdding();
}
