public TimeSpan Process(string peopleJsonFilePath, string targetFilePath, string errorsFilePath) { var peopleCount = _fileLinesCounter.Count(peopleJsonFilePath); var sw = Stopwatch.StartNew(); var dataPool = new DataPool(); IList <Data> data; if (Settings.SplitReadingIntoTwoSteps) { Events.Write.BlockEnter("ReadLines", DATA_ID); using (var peopleJsonStream = File.OpenText(peopleJsonFilePath)) { data = _streamLinesReader.Read(peopleJsonStream, peopleCount) .Select(x => { var d = dataPool.Rent(); d.PersonJson = x; return(d); }) .ToList(); } Events.Write.BlockExit("ReadLines", DATA_ID, 0); Console.WriteLine("Lines read."); Events.Write.BlockEnter("ParseData", DATA_ID); data.ForEach(_dataParser.Parse); Events.Write.BlockExit("ParseData", DATA_ID, 0); Console.WriteLine("Data parsed."); } else { Events.Write.BlockEnter("ReadData", DATA_ID); using (var peopleJsonStream = File.OpenText(peopleJsonFilePath)) { data = _dataReader.Read(peopleJsonStream, peopleCount, dataPool).ToList(); } Events.Write.BlockExit("ReadData", DATA_ID, 0); Console.WriteLine("Data read."); } Events.Write.BlockEnter("Validate", DATA_ID); if (Settings.ProcessInParallel) { Parallel.ForEach(data.Where(x => x.IsValid), new ParallelOptions { MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism }, _personValidator.Validate); } else { data.Where(x => x.IsValid).ForEach(_personValidator.Validate); } Events.Write.BlockExit("Validate", DATA_ID, 0); Console.WriteLine("Data validated."); Events.Write.BlockEnter("ComputeFields", DATA_ID); if (Settings.ProcessInParallel) { Parallel.ForEach(data.Where(x => x.IsValid), new ParallelOptions { MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism }, _personFieldsComputer.Compute); } else { data.Where(x => x.IsValid).ForEach(_personFieldsComputer.Compute); } Events.Write.BlockExit("ComputeFields", DATA_ID, 0); Console.WriteLine("Fields computed."); for (var i = 1; i <= Settings.ExtraProcessingBlocksCount; i++) { Events.Write.BlockEnter($"ExtraProcessing {i}", DATA_ID); if (Settings.ProcessInParallel) { Parallel.ForEach(data, new ParallelOptions { MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism }, _ => ComputationsHelper.PerformTimeConsumingOperation()); } else { data.ForEach(_ => ComputationsHelper.PerformTimeConsumingOperation()); } Events.Write.BlockExit($"ExtraProcessing {i}", DATA_ID, 0); Console.WriteLine($"ExtraProcessing {i} done."); } Events.Write.BlockEnter("WriteData", DATA_ID); using (var writer = new StreamWriter(targetFilePath)) { data.ForEach(x => _dataWriter.Write(writer, x)); } Events.Write.BlockExit("WriteData", DATA_ID, 0); Console.WriteLine("Data written."); Events.Write.BlockEnter("DisposeData", DATA_ID); data.ForEach(dataPool.Return); Events.Write.BlockExit("DisposeData", DATA_ID, 0); Console.WriteLine("Data disposed."); return(sw.Elapsed); }
public IEnumerable <Data> Read(StreamReader peopleJsonStream, int peopleToRead, DataPool dataPool) { for (var i = 0; i < peopleToRead && !peopleJsonStream.EndOfStream; i++) { var line = peopleJsonStream.ReadLine(); var data = dataPool.Rent(); data.PersonJson = line; _dataParser.Parse(data); yield return(data); } }
public DataPool() { Current = this; }