public static void Main(string[] args) { var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json", true, true) .Build(); var csvFlightFiles = configuration.GetSection("Files").Get <string[]>(); // Initialize the Client Settings (Connection String, ...): var settings = new DGraphClientSettings(configuration); // Create the Client: var client = new DGraphClient(settings); // Initialize the Schema: client.CreateSchema(); // Import all hourly weather data from 2014: foreach (var csvFlightStatisticsFile in csvFlightFiles) { ProcessFlights(client, csvFlightStatisticsFile); } }
private static void ProcessFlights(DGraphClient client, string csvFilePath) { // Create the Converter: var converter = new FlightConverter(); // Access to the List of Parsers: Parsers // Use the Flights Parser: .FlightStatisticsParser // Read the File: .ReadFromFile(csvFilePath, Encoding.UTF8) // As an Observable: .ToObservable() // Batch Entities by Time / Count: .Buffer(TimeSpan.FromSeconds(1), 1000) // And subscribe to the Batch synchronously (we don't want to handle too much backpressure here): .Subscribe(records => { var validRecords = records // Get the Valid Results: .Where(x => x.IsValid) // And get the populated Entities: .Select(x => x.Result) // Group by WBAN, Date and Time to avoid duplicates for this batch: .GroupBy(x => new { x.UniqueCarrier, x.FlightNumber, x.FlightDate }) // If there are duplicates then make a guess and select the first one: .Select(x => x.First()) // Convert into the DGraph Data Model: .Select(x => converter.Convert(x)) // Evaluate: .ToList(); // Finally write them with the Batch Writer: client.Mutate(validRecords); }); }