private static void ProcessLocalWeatherData(string csvFilePath) { if (log.IsInfoEnabled) { log.Info($"Processing File: {csvFilePath}"); } // Construct the Batch Processor: var processor = new LocalWeatherDataBatchProcessor(ConnectionString); // Access to the List of Parsers: var batches = Parsers // Use the LocalWeatherData Parser: .LocalWeatherDataParser // Read the File: .ReadFromFile(csvFilePath, Encoding.UTF8, 1) // Get the Valid Results: .Where(x => x.IsValid) // And get the populated Entities: .Select(x => x.Result) // Convert into the Sql Data Model: .Select(x => Converters.Converters.Convert(x)) // Sequential Evaluation: .AsEnumerable() // Batch in 80000 Entities / or wait 1 Second: .Batch(80000); foreach (var batch in batches) { processor.Write(batch); } }
private static void ProcessLocalWeatherData(string[] csvFiles) { var processor = new LocalWeatherDataBatchProcessor(ConnectionString); csvFiles .AsParallel() .WithDegreeOfParallelism(4) .ForAll(file => { log.Info($"Processing File: {file}"); // Access to the List of Parsers: var batches = Parsers // Use the LocalWeatherData Parser: .LocalWeatherDataParser // Read the File: .ReadFromFile(file, Encoding.UTF8, 1) // Get the Valid Results: .Where(x => x.IsValid) // And get the populated Entities: .Select(x => x.Result) // Convert into the Sql Data Model: .Select(x => LocalWeatherDataConverter.Convert(x)) // Batch: .Batch(80000); foreach (var batch in batches) { processor.Write(batch); } }); }
private static void ProcessLocalWeatherData(string csvFilePath) { if (log.IsInfoEnabled) { log.Info($"Processing File: {csvFilePath}"); } // Construct the Batch Processor: var processor = new LocalWeatherDataBatchProcessor(ConnectionString, Database); // Access to the List of Parsers: var batches = Parsers // Use the LocalWeatherData Parser: .LocalWeatherDataParser // Read the File: .ReadFromFile(csvFilePath, Encoding.UTF8, 1) // Get the Valid Results: .Where(x => x.IsValid) // And get the populated Entities: .Select(x => x.Result) // Let's stay safe! Stop parallelism here: .AsEnumerable() // Evaluate: .Batch(10000) // Convert each Batch into a LineProtocolPayload: .Select(measurements => LocalWeatherDataConverter.Convert(measurements)); foreach (var batch in batches) { try { var result = processor.WriteAsync(batch).GetAwaiter().GetResult(); // Log all unsuccessful writes, but do not quit execution: if (!result.Success) { if (log.IsErrorEnabled) { log.Error(result.ErrorMessage); } } } catch (Exception e) { // Some Pokemon Exception Handling here. I am seeing TaskCanceledExceptions with the // InfluxDB .NET Client. At the same time I do not want to quit execution, because // some batches fail: if (log.IsErrorEnabled) { log.Error(e, "Error occured writing InfluxDB Payload"); } } } }