private async Task MergeSortFiles(IList <string> files, long totalLines, string outputFile, Comparison <string> linesEqualityComparer)
{
var maxQueueRecords = Settings.MaxQueueRecords;
var sortedChunks = new SortedDictionary <string, List <AutoFileQueue> >(_comparer);
var totalSize = 0L;
foreach (var file in files)
{
totalSize += new FileInfo(file).Length;
var reader = OpenForAsyncTextRead(file);
var autoQueue = new AutoFileQueue(reader, maxQueueRecords);
if (autoQueue.Any())
{
AddToQueue(sortedChunks, autoQueue);
}
else
{
autoQueue.Dispose();
Debug.Assert(false, "Empty queue loaded from file. This should never happen!");
}
}
using (new AutoStopwatch("Merge sort input", _inputFileLength))
using (new AutoStopwatch("Merge sort compressed files ", totalSize))
using (var sw = new StreamWriter(OpenForAsyncWrite(outputFile)))
{
var progress = 0;
var totalWork = totalLines;
var writerTask = Task.CompletedTask;
while (sortedChunks.Any())
{
Console.Write("{0:f2}% \r", (100.0 * progress) / totalWork);
var lines = NWayMerge(sortedChunks);
await writerTask;
writerTask = Task.Run(async() =>
{
foreach (var line in lines)
{
await sw.WriteLineAsync(line);
}
});
progress += lines.Count;
}
await writerTask;
}
}
private static void AddToQueue(IDictionary <string, List <AutoFileQueue> > sortedChunks, AutoFileQueue queue)
{
var newTop = queue.Peek();
if (sortedChunks.ContainsKey(newTop))
{
sortedChunks[newTop].Add(queue);
}
else
{
sortedChunks.Add(newTop, new List <AutoFileQueue> {
queue
});
}
}