static async Task Main(string[] args)
{
Console.CancelKeyPress += (sender, eventArgs) =>
{
Console.WriteLine("Cancel event triggered");
Environment.Exit(-1);
};
string storageConnectionString = Environment.GetEnvironmentVariable("CLIMATE_COMPARISON_STORAGE_ACCOUNT");
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
var progressRepository = new ProgressRepository(tableClient);
int start = await progressRepository.Get(OPERATION) ?? 0;
Console.WriteLine($"Starting from {start}");
var repository = new PrecipitationRepository(tableClient);
string sqlConnectionString = Environment.GetEnvironmentVariable("CLIMATE_COMPARISON_ConnectionStrings__ConnectionString");
using (var sqlConnection = new SqlConnection(sqlConnectionString))
{
sqlConnection.Open();
for (; ;)
{
var placeIds = GetPlaceIds(start, PLACES_BATCH_SIZE, sqlConnection).ToList();
if (placeIds.Count == 0)
{
break;
}
foreach (int id in placeIds)
{
double[] precipitation;
try
{
precipitation = GetPrecipitation(id, sqlConnection).ToArray();
}
catch (SqlException sqlException) when(sqlException.ErrorCode == -2146232060)
{
Console.WriteLine($"can't get precipitation for {id}");
Console.WriteLine(sqlException);
continue;
}
await repository.Save(id, precipitation);
Console.WriteLine($"Processed {id}");
start = id;
}
await progressRepository.Set(OPERATION, start);
Console.WriteLine($"Saved progress {start}");
}
}
}
static async Task Main(string[] args)
{
Console.CancelKeyPress += (sender, eventArgs) =>
{
Console.WriteLine("Cancel event triggered");
Environment.Exit(-1);
};
string storageConnectionString = Environment.GetEnvironmentVariable("CLIMATE_COMPARISON_STORAGE_ACCOUNT");
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
var placesTable = tableClient.GetTableReference("places");
var progressRepository = new ProgressRepository(tableClient);
int start = await progressRepository.Get(OPERATION) ?? 0;
Console.WriteLine($"Starting from {start}");
string sqlConnectionString = Environment.GetEnvironmentVariable("CLIMATE_ConnectionStrings__ConnectionString");
using (var sqlConnection = new SqlConnection(sqlConnectionString))
{
sqlConnection.Open();
for (; ;)
{
var places = GetPlaces(start, PLACES_BATCH_SIZE, sqlConnection).ToList();
if (places.Count == 0)
{
break;
}
foreach (var place in places)
{
bool insertedRows = false;
var allNames = place.AltNames.Split(",").Union(new[] { place.Name })
.Select(SanitizeName)
.Select(it => it.ToLowerInvariant())
.Where(it => !string.IsNullOrWhiteSpace(it))
.Distinct();
var tasks = allNames.Select(async altName =>
{
var placeEntity = new PlaceEntity
{
PartitionKey = altName,
RowKey = place.Id.ToString(),
Id = place.Id.ToString(),
Name = place.Name,
AltName = altName,
Latitude = place.Latitude,
Longitude = place.Longitude,
CountryCode = place.CountryCode,
Population = place.Population
};
var getOperation = TableOperation.Retrieve <PlaceEntity>(placeEntity.PartitionKey, placeEntity.RowKey);
var existingRecord = await placesTable.ExecuteAsync(getOperation);
if (existingRecord.Result == null)
{
try
{
var insertOperation = TableOperation.Insert(placeEntity);
await placesTable.ExecuteAsync(insertOperation);
insertedRows = true;
}
catch (Exception exception)
{
Console.WriteLine($"Cannot insert {place.Name} {altName}: {exception}");
}
}
});
await Task.WhenAll(tasks);
if (insertedRows)
{
Console.WriteLine($"{place.Name} imported");
}
else
{
Console.WriteLine($"Skipped {place.Name}");
}
start = place.Id;
}
await progressRepository.Set(OPERATION, start);
Console.WriteLine($"Saved progress {start}");
}
}
Console.WriteLine("Done");
}
