private static async Task MergeRuns(string runsFolder)
{
var outputFile = "Test_Merged_Runs.txt";
var directoryFiles = new DirectoryInfo(runsFolder).GetFiles();
var streams = directoryFiles.Select(file =>
{
var fileStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
var iterator = new FileLineIterator(
fileStream,
mergeBufferSizePerRun: 1024 * 1024
);
return(fileStream, iterator);
});
var targetFileName = Path.Combine(runsFolder, "..", outputFile);
File.Delete(targetFileName);
await StreamMerger.Merge(streams.Select(s => s.Item2), targetFileName, TestUtils.Logger);
foreach (var stream in streams)
{
stream.iterator.Dispose();
stream.fileStream.Dispose();
}
}
private async Task MergeSortedRuns(ExternalRunRepository runRepository, string targetFileName)
{
_logger($"Started merge on {runRepository.Count} runs");
using var runs = runRepository.GetStoredRunsAsIterableRecords(
// fileStream => new FileLineIterator(fileStream, _mergeBufferSizePerRun)
fileStream => new DoublyBufferedLineIterator(fileStream, _mergeBufferSizePerRun)
);
await StreamMerger.Merge(
runs : runs,
targetFileName : targetFileName,
logger : _logger);
}
public void MergeNonEmptyList()
{
var streamParts = StreamPart.Create(StreamHelper.GetStream(SizeOf64KByte + 1));
var streamMerger = new StreamMerger();
int count = 0;
streamMerger.StreamAvailable += stream =>
{
count++;
Assert.That(stream.Length, Is.EqualTo(SizeOf64KByte + 1));
};
streamMerger.Merge(streamParts);
Assert.That(count, Is.EqualTo(1));
}
public void MergeEmptyList()
{
var streamParts = new List<IStreamPart>();
var streamMerger = new StreamMerger();
int count = 0;
streamMerger.StreamAvailable += stream =>
{
count++;
Assert.That(stream.Length, Is.EqualTo(0));
};
streamMerger.Merge(streamParts);
Assert.That(count, Is.EqualTo(1));
}
public void MergeNonEmptyList()
{
var streamParts = StreamPart.Create(StreamHelper.GetStream(SizeOf64KByte + 1));
var streamMerger = new StreamMerger();
int count = 0;
streamMerger.StreamAvailable += stream =>
{
count++;
Assert.That(stream.Length, Is.EqualTo(SizeOf64KByte + 1));
};
streamMerger.Merge(streamParts);
Assert.That(count, Is.EqualTo(1));
}
public void MergeEmptyList()
{
var streamParts = new List <IStreamPart>();
var streamMerger = new StreamMerger();
int count = 0;
streamMerger.StreamAvailable += stream =>
{
count++;
Assert.That(stream.Length, Is.EqualTo(0));
};
streamMerger.Merge(streamParts);
Assert.That(count, Is.EqualTo(1));
}
public async Task Test_Read_And_Sort_M100_Runs()
{
var testFile = @"f:\atesttask\M099.txt";
var testFile_Sorted = testFile + ".sorted_test";
using var fileStream = new FileStream(testFile, FileMode.Open, FileAccess.Read);
var iterator = CreateIterator(fileStream);
await StreamMerger.Merge(
runs : new[] { iterator },
targetFileName : testFile_Sorted,
logger : TestUtils.Logger
);
var sourceFileInfo = new FileInfo(testFile);
var targetFileInfo = new FileInfo(testFile_Sorted);
targetFileInfo.Length.ShouldBe(sourceFileInfo.Length);
File.Delete(testFile_Sorted);
}
