public async Task Sort() { Console.WriteLine($"[{DateTime.Now}] Read file {Path.GetFileName(_sourceFileName)}"); Stopwatch stopwatch = Stopwatch.StartNew(); var fileSize = new FileInfo(_sourceFileName).Length; var maxDegreeOfParallelism = _sortConfig.MaxDegreeOfParallelismPerLevel; var partitionSize = EstimateSizeOfPartition(_sortConfig.ApproximateMemoryLimit, maxDegreeOfParallelism); if (Directory.Exists(_splitsFolder)) { Directory.Delete(_splitsFolder, true); } Directory.CreateDirectory(_splitsFolder); if (Directory.Exists(_mergeFolder)) { Directory.Delete(_mergeFolder, true); } Directory.CreateDirectory(_mergeFolder); var batchReader = new BufferBlock <BatchInfo>(new DataflowBlockOptions { BoundedCapacity = (int)(maxDegreeOfParallelism * 0.1 + 1) }); var sorterBlock = new TransformBlock <BatchInfo, BatchInfo>(l => PartitionSorter(l), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, BoundedCapacity = maxDegreeOfParallelism }); var batchSaverBlock = new TransformBlock <BatchInfo, string>(l => PartitionSaver(l), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, BoundedCapacity = maxDegreeOfParallelism }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; batchReader.LinkTo(sorterBlock, linkOptions); sorterBlock.LinkTo(batchSaverBlock, linkOptions); var partitioner = Partitioner(_sourceFileName, partitionSize, batchReader); var filesForMerge = await batchSaverBlock.ReceiveAllData(); await partitioner; await batchSaverBlock.Completion; Console.WriteLine($"[{DateTime.Now}] File splitted. Elapsed: {stopwatch.Elapsed}"); BatchBlock <string> sortedBufferBlock = null; TransformBlock <IEnumerable <string>, string> mergerBlock = null; while (filesForMerge.Count > 1) { var estimatedCacheSize = EstimateSizeOfCache(_sortConfig.ApproximateMemoryLimit, filesForMerge.Count, _sortConfig.MaxDegreeOfParallelismPerLevel); var batchBlockSize = filesForMerge.Count / _sortConfig.MaxDegreeOfParallelismPerLevel; if (batchBlockSize == 1) { batchBlockSize = 2; } if (batchBlockSize == 0) { batchBlockSize = _sortConfig.MaxDegreeOfParallelismPerLevel; } sortedBufferBlock = new BatchBlock <string>(batchBlockSize); mergerBlock = new TransformBlock <IEnumerable <string>, string>(l => Merge(l, estimatedCacheSize), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, BoundedCapacity = maxDegreeOfParallelism }); sortedBufferBlock.LinkTo(mergerBlock, linkOptions); await sortedBufferBlock.SendAllData(filesForMerge); filesForMerge = await mergerBlock.ReceiveAllData(); await mergerBlock.Completion; } if (filesForMerge.Count == 0) { Console.WriteLine("Somthing went wrong. Result file is not created"); } else { File.Move(filesForMerge.ElementAt(0), _destFileName, true); Console.WriteLine($"[{DateTime.Now}] Order complete. Result file: {_destFileName}. Elapsed: {stopwatch.Elapsed}"); } }