public async Task Sort()
{
Console.WriteLine($"[{DateTime.Now}] Read file {Path.GetFileName(_sourceFileName)}");
Stopwatch stopwatch = Stopwatch.StartNew();
var fileSize = new FileInfo(_sourceFileName).Length;
var maxDegreeOfParallelism = _sortConfig.MaxDegreeOfParallelismPerLevel;
var partitionSize = EstimateSizeOfPartition(_sortConfig.ApproximateMemoryLimit, maxDegreeOfParallelism);
if (Directory.Exists(_splitsFolder))
{
Directory.Delete(_splitsFolder, true);
}
Directory.CreateDirectory(_splitsFolder);
if (Directory.Exists(_mergeFolder))
{
Directory.Delete(_mergeFolder, true);
}
Directory.CreateDirectory(_mergeFolder);
var batchReader = new BufferBlock <BatchInfo>(new DataflowBlockOptions {
BoundedCapacity = (int)(maxDegreeOfParallelism * 0.1 + 1)
});
var sorterBlock = new TransformBlock <BatchInfo, BatchInfo>(l => PartitionSorter(l),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism, BoundedCapacity = maxDegreeOfParallelism
});
var batchSaverBlock = new TransformBlock <BatchInfo, string>(l => PartitionSaver(l),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism, BoundedCapacity = maxDegreeOfParallelism
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
batchReader.LinkTo(sorterBlock, linkOptions);
sorterBlock.LinkTo(batchSaverBlock, linkOptions);
var partitioner = Partitioner(_sourceFileName, partitionSize, batchReader);
var filesForMerge = await batchSaverBlock.ReceiveAllData();
await partitioner;
await batchSaverBlock.Completion;
Console.WriteLine($"[{DateTime.Now}] File splitted. Elapsed: {stopwatch.Elapsed}");
BatchBlock <string> sortedBufferBlock = null;
TransformBlock <IEnumerable <string>, string> mergerBlock = null;
while (filesForMerge.Count > 1)
{
var estimatedCacheSize = EstimateSizeOfCache(_sortConfig.ApproximateMemoryLimit, filesForMerge.Count, _sortConfig.MaxDegreeOfParallelismPerLevel);
var batchBlockSize = filesForMerge.Count / _sortConfig.MaxDegreeOfParallelismPerLevel;
if (batchBlockSize == 1)
{
batchBlockSize = 2;
}
if (batchBlockSize == 0)
{
batchBlockSize = _sortConfig.MaxDegreeOfParallelismPerLevel;
}
sortedBufferBlock = new BatchBlock <string>(batchBlockSize);
mergerBlock = new TransformBlock <IEnumerable <string>, string>(l => Merge(l, estimatedCacheSize),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism, BoundedCapacity = maxDegreeOfParallelism
});
sortedBufferBlock.LinkTo(mergerBlock, linkOptions);
await sortedBufferBlock.SendAllData(filesForMerge);
filesForMerge = await mergerBlock.ReceiveAllData();
await mergerBlock.Completion;
}
if (filesForMerge.Count == 0)
{
Console.WriteLine("Somthing went wrong. Result file is not created");
}
else
{
File.Move(filesForMerge.ElementAt(0), _destFileName, true);
Console.WriteLine($"[{DateTime.Now}] Order complete. Result file: {_destFileName}. Elapsed: {stopwatch.Elapsed}");
}
}