private static async Task MergeRuns(string runsFolder) { var outputFile = "Test_Merged_Runs.txt"; var directoryFiles = new DirectoryInfo(runsFolder).GetFiles(); var streams = directoryFiles.Select(file => { var fileStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); var iterator = new FileLineIterator( fileStream, mergeBufferSizePerRun: 1024 * 1024 ); return(fileStream, iterator); }); var targetFileName = Path.Combine(runsFolder, "..", outputFile); File.Delete(targetFileName); await StreamMerger.Merge(streams.Select(s => s.Item2), targetFileName, TestUtils.Logger); foreach (var stream in streams) { stream.iterator.Dispose(); stream.fileStream.Dispose(); } }
private async Task MergeSortedRuns(ExternalRunRepository runRepository, string targetFileName) { _logger($"Started merge on {runRepository.Count} runs"); using var runs = runRepository.GetStoredRunsAsIterableRecords( // fileStream => new FileLineIterator(fileStream, _mergeBufferSizePerRun) fileStream => new DoublyBufferedLineIterator(fileStream, _mergeBufferSizePerRun) ); await StreamMerger.Merge( runs : runs, targetFileName : targetFileName, logger : _logger); }
public void MergeNonEmptyList() { var streamParts = StreamPart.Create(StreamHelper.GetStream(SizeOf64KByte + 1)); var streamMerger = new StreamMerger(); int count = 0; streamMerger.StreamAvailable += stream => { count++; Assert.That(stream.Length, Is.EqualTo(SizeOf64KByte + 1)); }; streamMerger.Merge(streamParts); Assert.That(count, Is.EqualTo(1)); }
public void MergeEmptyList() { var streamParts = new List<IStreamPart>(); var streamMerger = new StreamMerger(); int count = 0; streamMerger.StreamAvailable += stream => { count++; Assert.That(stream.Length, Is.EqualTo(0)); }; streamMerger.Merge(streamParts); Assert.That(count, Is.EqualTo(1)); }
public void MergeEmptyList() { var streamParts = new List <IStreamPart>(); var streamMerger = new StreamMerger(); int count = 0; streamMerger.StreamAvailable += stream => { count++; Assert.That(stream.Length, Is.EqualTo(0)); }; streamMerger.Merge(streamParts); Assert.That(count, Is.EqualTo(1)); }
public async Task Test_Read_And_Sort_M100_Runs() { var testFile = @"f:\atesttask\M099.txt"; var testFile_Sorted = testFile + ".sorted_test"; using var fileStream = new FileStream(testFile, FileMode.Open, FileAccess.Read); var iterator = CreateIterator(fileStream); await StreamMerger.Merge( runs : new[] { iterator }, targetFileName : testFile_Sorted, logger : TestUtils.Logger ); var sourceFileInfo = new FileInfo(testFile); var targetFileInfo = new FileInfo(testFile_Sorted); targetFileInfo.Length.ShouldBe(sourceFileInfo.Length); File.Delete(testFile_Sorted); }