/// <see cref="IIntegerFileInfoCollector.GetIntegerFileInfo(string, int)"/> public IntegerFileInfo GetIntegerFileInfo(string filePath, int chunkSize) { Debug.Assert(filePath != null); Debug.Assert(chunkSize > 0); IntegerFileInfo fileInfo = new IntegerFileInfo() { NumOfChunks = 0, NumOfIntegers = 0 }; //Open a text reader to the file using (StreamReader fileReader = fileIO.CreateFileStreamReader(filePath)) { while (!fileReader.EndOfStream) { //Read each line to count the number of lines in the file fileReader.ReadLine(); fileInfo.NumOfIntegers++; } } //Now that we have the number of integers, calculate the number of chunks fileInfo.NumOfChunks = (int)Math.Ceiling(Decimal.Divide(fileInfo.NumOfIntegers, chunkSize)); return(fileInfo); }
/// <summary> /// Merges an input file group into a single output file /// </summary> /// <param name="fileGroup">The names of the input files to be merged</param> /// <param name="outputDirectory">The directory the output file is to be written in</param> /// <param name="fileTemplate">The template for the output file</param> /// <param name="fileGroupNumber">The file group number</param> /// <param name="updateProgress">A method that will be called to update chunk file creation progress. The /// number of integers that have been merged so far in the file group will be passed to this method /// whenever an integer is merged.</param> /// <returns>The path of the output file that was written</returns> private string MergeInputFileGroup(List <string> fileGroup, string outputDirectory, string fileTemplate, int fileGroupNumber, Action <int> updateProgress = null) { //Create the input stream readers List <StreamReader> inputStreamReaders = fileGroup .Select(file => fileIO.CreateFileStreamReader(file)) .ToList(); //Create the output file and output file stream writer string outputFilePath = Path.Combine(outputDirectory, string.Format(fileTemplate, fileGroupNumber)); using (Stream outputFileStream = fileIO.CreateFile(outputFilePath)) using (StreamWriter outputStreamWriter = new StreamWriter(outputFileStream)) { //Merge the input streams into the output stream integerStreamMerger.MergeIntegerStreams(inputStreamReaders, outputStreamWriter, updateProgress); } //Close the input stream readers inputStreamReaders.ForEach(inputStreamReader => inputStreamReader.Close()); return(outputFilePath); }
/// <summary> /// Creates sorted chunks files from an input file /// </summary> /// <param name="inputFile">The input file from which the integers are to be read</param> /// <param name="chunkSize">The chunk size to use when chunking the input</param> /// <param name="outputDirectory">The output directory where the intermediate files are to be written</param> /// /// <returns>The names of the chunk files that were created</returns> private static List <string> CreateSortedChunkFiles(string inputFile, int chunkSize, string outputDirectory, string chunkFileTemplate, Action <int> updateChunkProgress) { IFileIO fileIO = serviceProvider.GetService <IFileIO>(); //Create the chunk stream generator from the input file IChunkStreamCreator chunkStreamCreator = serviceProvider.GetService <IChunkStreamCreator>(); StreamReader inputStreamReader = fileIO.CreateFileStreamReader(inputFile); //Create the chunk generator and add a transformation that sorts each chunk as it is emitted IEnumerable <List <int> > sortedChunkGenerator = chunkStreamCreator .CreateIntegerChunkGenerator(inputStreamReader, chunkSize) .Select(chunk => chunk.OrderBy(integer => integer).ToList()); //Create the chunk files IChunkFileCreator chunkFileCreator = serviceProvider.GetService <IChunkFileCreator>(); List <string> chunkFiles = chunkFileCreator.CreateChunkFiles(sortedChunkGenerator, chunkFileTemplate, outputDirectory, updateChunkProgress); return(chunkFiles); }