public DataFlow(ImporterConfiguration configuration) { try { var stopwatch = new Stopwatch(); stopwatch.Start(); Logger.GetLogger().SetLogginLevel(Logger.LogLevel.Debug); if (configuration.Files.TryGetValue("Error_Output", out string errorPath)) { ErrorOutput.SetOutputTarget(File.Open(errorPath, FileMode.Create, FileAccess.Write, FileShare.Read)); } this.LoadDictionaries(configuration.Files, configuration.Readers).Wait(); this.writers = configuration.Writers.Where(x => !x.Disabled) .Select( x => { if (configuration.Files.TryGetValue(x.Name, out string filePath)) { return(FileWriter.GetFileWriter(File.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.Read), x)); } throw new ArgumentOutOfRangeException($"There is no file path defined for \"{x.Name}\""); }) .ToList(); var reader = configuration.Readers.FirstOrDefault(x => !x.Disabled && !DataDictionary.GetDictionaryNames().Contains(x.Name)); if (reader != null) { if (configuration.Files.TryGetValue(reader.Name, out string readerFilePath)) { if (!File.Exists(readerFilePath)) { throw new FileNotFoundException($"File \"{readerFilePath}\" not found"); } using (var fileReader = FileReader.GetFileReader(File.Open(readerFilePath, FileMode.Open, FileAccess.Read, FileShare.Read), reader)) { var cancellationTokenSource = new CancellationTokenSource(); var tasks = new Task[Environment.ProcessorCount]; for (var i = 0; i < Environment.ProcessorCount; i++) { tasks[i] = Task.Run(() => this.WriterTask(cancellationTokenSource.Token)); } foreach (var record in fileReader.ReadData()) { if (this.buffer.Count > MAX_BUFFER_SIZE) { Logger.GetLogger().DebugAsync("Reached record buffer limit"); Thread.Sleep(50); } this.buffer.Add(record); } cancellationTokenSource.Cancel(); Task.WaitAll(tasks); Logger.GetLogger().InfoAsync($"Successfully saved {this.successfulRecordCount} records"); Logger.GetLogger().InfoAsync($"Failed to process {this.exceptionRecordCount} records"); } } } stopwatch.Stop(); Logger.GetLogger() .InfoAsync($"Finished data processing in {stopwatch.Elapsed.Hours:00}:{stopwatch.Elapsed.Minutes:00}:{stopwatch.Elapsed.Seconds:00}.{stopwatch.Elapsed.Milliseconds}"); } finally { ErrorOutput.Flush(); } }