예제 #1
0
        public TimeSpan Process(string peopleJsonFilePath,
                                string targetFilePath,
                                string errorsFilePath)
        {
            var peopleCount = _fileLinesCounter.Count(peopleJsonFilePath);

            var sw = Stopwatch.StartNew();

            var          dataPool = new DataPool();
            IList <Data> data;

            if (Settings.SplitReadingIntoTwoSteps)
            {
                Events.Write.BlockEnter("ReadLines", DATA_ID);

                using (var peopleJsonStream = File.OpenText(peopleJsonFilePath))
                {
                    data = _streamLinesReader.Read(peopleJsonStream, peopleCount)
                           .Select(x =>
                    {
                        var d        = dataPool.Rent();
                        d.PersonJson = x;

                        return(d);
                    })
                           .ToList();
                }

                Events.Write.BlockExit("ReadLines", DATA_ID, 0);
                Console.WriteLine("Lines read.");

                Events.Write.BlockEnter("ParseData", DATA_ID);

                data.ForEach(_dataParser.Parse);

                Events.Write.BlockExit("ParseData", DATA_ID, 0);
                Console.WriteLine("Data parsed.");
            }
            else
            {
                Events.Write.BlockEnter("ReadData", DATA_ID);

                using (var peopleJsonStream = File.OpenText(peopleJsonFilePath))
                {
                    data = _dataReader.Read(peopleJsonStream, peopleCount, dataPool).ToList();
                }

                Events.Write.BlockExit("ReadData", DATA_ID, 0);
                Console.WriteLine("Data read.");
            }

            Events.Write.BlockEnter("Validate", DATA_ID);

            if (Settings.ProcessInParallel)
            {
                Parallel.ForEach(data.Where(x => x.IsValid),
                                 new ParallelOptions {
                    MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism
                },
                                 _personValidator.Validate);
            }
            else
            {
                data.Where(x => x.IsValid).ForEach(_personValidator.Validate);
            }

            Events.Write.BlockExit("Validate", DATA_ID, 0);
            Console.WriteLine("Data validated.");

            Events.Write.BlockEnter("ComputeFields", DATA_ID);

            if (Settings.ProcessInParallel)
            {
                Parallel.ForEach(data.Where(x => x.IsValid),
                                 new ParallelOptions {
                    MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism
                },
                                 _personFieldsComputer.Compute);
            }
            else
            {
                data.Where(x => x.IsValid).ForEach(_personFieldsComputer.Compute);
            }

            Events.Write.BlockExit("ComputeFields", DATA_ID, 0);
            Console.WriteLine("Fields computed.");

            for (var i = 1; i <= Settings.ExtraProcessingBlocksCount; i++)
            {
                Events.Write.BlockEnter($"ExtraProcessing {i}", DATA_ID);

                if (Settings.ProcessInParallel)
                {
                    Parallel.ForEach(data,
                                     new ParallelOptions {
                        MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism
                    },
                                     _ => ComputationsHelper.PerformTimeConsumingOperation());
                }
                else
                {
                    data.ForEach(_ => ComputationsHelper.PerformTimeConsumingOperation());
                }

                Events.Write.BlockExit($"ExtraProcessing {i}", DATA_ID, 0);
                Console.WriteLine($"ExtraProcessing {i} done.");
            }

            Events.Write.BlockEnter("WriteData", DATA_ID);

            using (var writer = new StreamWriter(targetFilePath))
            {
                data.ForEach(x => _dataWriter.Write(writer, x));
            }

            Events.Write.BlockExit("WriteData", DATA_ID, 0);
            Console.WriteLine("Data written.");

            Events.Write.BlockEnter("DisposeData", DATA_ID);
            data.ForEach(dataPool.Return);
            Events.Write.BlockExit("DisposeData", DATA_ID, 0);
            Console.WriteLine("Data disposed.");

            return(sw.Elapsed);
        }
예제 #2
0
        public IEnumerable <Data> Read(StreamReader peopleJsonStream, int peopleToRead, DataPool dataPool)
        {
            for (var i = 0; i < peopleToRead && !peopleJsonStream.EndOfStream; i++)
            {
                var line = peopleJsonStream.ReadLine();

                var data = dataPool.Rent();
                data.PersonJson = line;
                _dataParser.Parse(data);

                yield return(data);
            }
        }
예제 #3
0
 public DataPool()
 {
     Current = this;
 }