private static void ProcessStationData(string csvFilePath) { if (log.IsInfoEnabled) { log.Info($"Processing File: {csvFilePath}"); } // Construct the Batch Processor: var processor = new StationBatchProcessor(ConnectionString); // Access to the List of Parsers: var batches = Parsers // Use the Station Parser: .StationParser // Read the File: .ReadFromFile(csvFilePath, Encoding.UTF8, 2) // Get the Valid Results: .Where(x => x.IsValid) // And get the populated Entities: .Select(x => x.Result) // If there is no WBAN, do not process the record: .Where(x => !string.IsNullOrWhiteSpace(x.Identifier)) // Convert into the Sql Data Model: .Select(x => Converters.Converters.Convert(x)) // Sequential: .AsEnumerable() // Batch: .Batch(80000); // Finally write them with the Batch Writer: foreach (var batch in batches) { processor.Write(batch); } }
private static void ProcessStationData(string csvFilePath) { log.Info($"Processing File: {csvFilePath}"); var batches = Parsers .StationParser .ReadFromFile(csvFilePath, Encoding.UTF8, 2) .Where(x => x.IsValid) .Select(x => x.Result) .Select(x => LocalWeatherDataConverter.Convert(x)) .Batch(500); // Construct the Batch Processor: var processor = new StationBatchProcessor(ConnectionString); foreach (var batch in batches) { // Finally write them with the Batch Writer: processor.Write(batch); } }