public void OnPost() { var payees = payeeRepository.GetAll(); var input = new StringBuilder(); using (var reader = new StreamReader(FormFile.OpenReadStream(), Encoding.GetEncoding("ISO-8859-1"))) { while (reader.Peek() >= 0) { input.AppendLine(reader.ReadLine()); } } var inputRecords = MyConverter.GetInputRecords(input.ToString()); var outputRecords = new List <Output>(); foreach (var record in inputRecords) { outputRecords.Add(MyConverter.Convert(record, payees)); } InputRecords.AddRange(inputRecords); OutputRecords.AddRange(outputRecords); Output = MyConverter.GetOutputString(outputRecords); }
protected virtual StageResult Transform(int?recordBatchSize = null, int?recordLimit = null, Dictionary <string, string> options = null) { if (!ParallelExecution || InputRecords.Count < 100 || ((RecordLimitSize > 0) && (RecordLimitSize < 100))) { using (Operation transformOp = Begin("Transforming {0} records using sequential execution", InputRecords.Count)) { for (int i = 0; i < InputRecords.Count; i++) { OutputRecords.Add(TransformInputToOutput(this, WriterOptions, InputRecords[i])); if ((i + 1) % 1000 == 0) { Info("Transformed range {0} to {1} of {2} records...", (i + 1) - 1000, i + 1, InputRecords.Count); } if ((RecordLimitSize > 0) && (i + 1 == RecordLimitSize)) { Info("Stopping transformation at record limit {0}.", i + 1); transformOp.Complete(); break; } } transformOp.Complete(); } } else { int limit = RecordLimitSize > 0 ? RecordLimitSize <= InputRecords.Count ? RecordLimitSize : InputRecords.Count : InputRecords.Count; using (Operation transformOp = Begin("Transforming {0} records using parallel execution", limit)) { ConcurrentDictionary <int, TRecord> concurrentOutputDictionary = new ConcurrentDictionary <int, TRecord>(); Parallel.For(0, limit, (i, loop) => { TRecord output = TransformInputToOutput(this, WriterOptions, InputRecords[i]); concurrentOutputDictionary.TryAdd(i, output); if ((i + 1) % 1000 == 0) { Info("Transformed range {0} to {1} of {2} records...", (i + 1) - 1000, i + 1, InputRecords.Count); } }); OutputRecords = concurrentOutputDictionary.Values.ToList(); transformOp.Complete(); } } Info("Transformed {0} records with maximum {1} features to {2}.", OutputRecords.Count, OutputRecords.Max(r => r.Features.Count), OutputFileName); return(StageResult.SUCCESS); }