static async Task Main(string[] args) { Console.CancelKeyPress += (sender, eventArgs) => { Console.WriteLine("Cancel event triggered"); Environment.Exit(-1); }; string storageConnectionString = Environment.GetEnvironmentVariable("CLIMATE_COMPARISON_STORAGE_ACCOUNT"); var storageAccount = CloudStorageAccount.Parse(storageConnectionString); var tableClient = storageAccount.CreateCloudTableClient(); var progressRepository = new ProgressRepository(tableClient); int start = await progressRepository.Get(OPERATION) ?? 0; Console.WriteLine($"Starting from {start}"); var repository = new PrecipitationRepository(tableClient); string sqlConnectionString = Environment.GetEnvironmentVariable("CLIMATE_COMPARISON_ConnectionStrings__ConnectionString"); using (var sqlConnection = new SqlConnection(sqlConnectionString)) { sqlConnection.Open(); for (; ;) { var placeIds = GetPlaceIds(start, PLACES_BATCH_SIZE, sqlConnection).ToList(); if (placeIds.Count == 0) { break; } foreach (int id in placeIds) { double[] precipitation; try { precipitation = GetPrecipitation(id, sqlConnection).ToArray(); } catch (SqlException sqlException) when(sqlException.ErrorCode == -2146232060) { Console.WriteLine($"can't get precipitation for {id}"); Console.WriteLine(sqlException); continue; } await repository.Save(id, precipitation); Console.WriteLine($"Processed {id}"); start = id; } await progressRepository.Set(OPERATION, start); Console.WriteLine($"Saved progress {start}"); } } }
static async Task Main(string[] args) { Console.CancelKeyPress += (sender, eventArgs) => { Console.WriteLine("Cancel event triggered"); Environment.Exit(-1); }; string storageConnectionString = Environment.GetEnvironmentVariable("CLIMATE_COMPARISON_STORAGE_ACCOUNT"); var storageAccount = CloudStorageAccount.Parse(storageConnectionString); var tableClient = storageAccount.CreateCloudTableClient(); var placesTable = tableClient.GetTableReference("places"); var progressRepository = new ProgressRepository(tableClient); int start = await progressRepository.Get(OPERATION) ?? 0; Console.WriteLine($"Starting from {start}"); string sqlConnectionString = Environment.GetEnvironmentVariable("CLIMATE_ConnectionStrings__ConnectionString"); using (var sqlConnection = new SqlConnection(sqlConnectionString)) { sqlConnection.Open(); for (; ;) { var places = GetPlaces(start, PLACES_BATCH_SIZE, sqlConnection).ToList(); if (places.Count == 0) { break; } foreach (var place in places) { bool insertedRows = false; var allNames = place.AltNames.Split(",").Union(new[] { place.Name }) .Select(SanitizeName) .Select(it => it.ToLowerInvariant()) .Where(it => !string.IsNullOrWhiteSpace(it)) .Distinct(); var tasks = allNames.Select(async altName => { var placeEntity = new PlaceEntity { PartitionKey = altName, RowKey = place.Id.ToString(), Id = place.Id.ToString(), Name = place.Name, AltName = altName, Latitude = place.Latitude, Longitude = place.Longitude, CountryCode = place.CountryCode, Population = place.Population }; var getOperation = TableOperation.Retrieve <PlaceEntity>(placeEntity.PartitionKey, placeEntity.RowKey); var existingRecord = await placesTable.ExecuteAsync(getOperation); if (existingRecord.Result == null) { try { var insertOperation = TableOperation.Insert(placeEntity); await placesTable.ExecuteAsync(insertOperation); insertedRows = true; } catch (Exception exception) { Console.WriteLine($"Cannot insert {place.Name} {altName}: {exception}"); } } }); await Task.WhenAll(tasks); if (insertedRows) { Console.WriteLine($"{place.Name} imported"); } else { Console.WriteLine($"Skipped {place.Name}"); } start = place.Id; } await progressRepository.Set(OPERATION, start); Console.WriteLine($"Saved progress {start}"); } } Console.WriteLine("Done"); }