private void ReadFile() { var fileInfo = new FileInfo(fileName); var offset = 0L; while (offset < fileInfo.Length) { var size = Math.Min(fileInfo.Length - offset, batchSize); var viewStream = memoryMappedFile.CreateViewStream(offset, size); var chunk = new Chunk(offset, viewStream); offset += viewStream.Length; producingBag.Add(chunk); } producingBag.CompleteAdding(); }
private void Start() { producingBag = producer.StartProducing(); consumer.StartConsuming(consumingBag); for (var i = 0; i < transformConcurrency; i++) { var task = new Task(() => { while (producingBag.TryTake(out var chunk)) { consumingBag.Add(mapper(chunk)); } }); transformTasks.Add(task); threadPool.RunTask(task); } }
private void ReadFile() { const int chunkLengthSize = sizeof(int); const int initialOffsetSize = sizeof(long); var fileInfo = new FileInfo(fileName); using var memoryMappedFile = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open, null); var offset = fileHeaderSize; while (offset < fileInfo.Length) { using var tmpStream = memoryMappedFile.CreateViewStream(offset, chunkLengthSize); var chunkLength = tmpStream.ReadInt32(); var viewStream = memoryMappedFile.CreateViewStream(offset + chunkLengthSize, chunkLength + initialOffsetSize); offset += viewStream.Length + 4; bag.Add(viewStream); } bag.CompleteAdding(); }