private static void ProcessLocalWeatherData(string csvFilePath)
{
if (log.IsInfoEnabled)
{
log.Info($"Processing File: {csvFilePath}");
}
// Construct the Batch Processor:
var processor = new LocalWeatherDataBatchProcessor(ConnectionString);
// Access to the List of Parsers:
var batches = Parsers
// Use the LocalWeatherData Parser:
.LocalWeatherDataParser
// Read the File:
.ReadFromFile(csvFilePath, Encoding.UTF8, 1)
// Get the Valid Results:
.Where(x => x.IsValid)
// And get the populated Entities:
.Select(x => x.Result)
// Convert into the Sql Data Model:
.Select(x => Converters.Converters.Convert(x))
// Sequential Evaluation:
.AsEnumerable()
// Batch in 80000 Entities / or wait 1 Second:
.Batch(80000);
foreach (var batch in batches)
{
processor.Write(batch);
}
}
private static void ProcessLocalWeatherData(string[] csvFiles)
{
var processor = new LocalWeatherDataBatchProcessor(ConnectionString);
csvFiles
.AsParallel()
.WithDegreeOfParallelism(4)
.ForAll(file =>
{
log.Info($"Processing File: {file}");
// Access to the List of Parsers:
var batches = Parsers
// Use the LocalWeatherData Parser:
.LocalWeatherDataParser
// Read the File:
.ReadFromFile(file, Encoding.UTF8, 1)
// Get the Valid Results:
.Where(x => x.IsValid)
// And get the populated Entities:
.Select(x => x.Result)
// Convert into the Sql Data Model:
.Select(x => LocalWeatherDataConverter.Convert(x))
// Batch:
.Batch(80000);
foreach (var batch in batches)
{
processor.Write(batch);
}
});
}
private static void ProcessLocalWeatherData(string csvFilePath)
{
if (log.IsInfoEnabled)
{
log.Info($"Processing File: {csvFilePath}");
}
// Construct the Batch Processor:
var processor = new LocalWeatherDataBatchProcessor(ConnectionString, Database);
// Access to the List of Parsers:
var batches = Parsers
// Use the LocalWeatherData Parser:
.LocalWeatherDataParser
// Read the File:
.ReadFromFile(csvFilePath, Encoding.UTF8, 1)
// Get the Valid Results:
.Where(x => x.IsValid)
// And get the populated Entities:
.Select(x => x.Result)
// Let's stay safe! Stop parallelism here:
.AsEnumerable()
// Evaluate:
.Batch(10000)
// Convert each Batch into a LineProtocolPayload:
.Select(measurements => LocalWeatherDataConverter.Convert(measurements));
foreach (var batch in batches)
{
try
{
var result = processor.WriteAsync(batch).GetAwaiter().GetResult();
// Log all unsuccessful writes, but do not quit execution:
if (!result.Success)
{
if (log.IsErrorEnabled)
{
log.Error(result.ErrorMessage);
}
}
}
catch (Exception e)
{
// Some Pokemon Exception Handling here. I am seeing TaskCanceledExceptions with the
// InfluxDB .NET Client. At the same time I do not want to quit execution, because
// some batches fail:
if (log.IsErrorEnabled)
{
log.Error(e, "Error occured writing InfluxDB Payload");
}
}
}
}
