public async Task <EuroStatTable> createEuroStatTable(EuroStatTable table)
{
await db.EuroStatTable.AddAsync(table);
db.SaveChanges();
return(table);
}
// TODO: Change no internet connection handling
/// <summary>
/// Fetches and stores all data from a specific Eurostat dataset.
/// </summary>
/// <param name="table">Representation of the dataset. Has information needed to fetch.</param>
/// <returns></returns>
private async Task FetchAndStore(EuroStatTable table)
{
List <Nace> naces = await databaseStore.getAllNaces();
int iterationCount = GetFetchIterationsCount(naces);
for (int i = 0; i < iterationCount; i++)
{
string url = GetEuroStatURL(table, i, naces, StartYear, EndYear);
try
{
HttpResponseMessage response = await client.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string jsonString = response.Content.ReadAsStringAsync().Result;
List <NaceRegionData> EurostatNRData = await JSONConverter.Convert(jsonString, table.attributeName);
await databaseStore.addNaceRegionData(EurostatNRData);
}
else
{
await handleStatusCodeNotSuccess((int)response.StatusCode, response, table, url);
}
}
catch (System.Net.Http.HttpRequestException e)
{
_logger.LogWarning("No internet connection, check your network configuration and try again" + "\nError trace: " + e);
throw new Exception();
}
}
}
/// <summary>
/// Generates an Eurostat URL for fetching.
/// </summary>
/// <param name="table">Representation of the dataset. Has information needed to fetch.</param>
/// <param name="index">Index to find the first nace to ask for in this fetch.</param>
/// <param name="naces">All naces to fetch information on from Eurostat.</param>
/// <returns></returns>
public String GetEuroStatURL(EuroStatTable table, int index, List <Nace> naces, int StartYear, int EndYear)
{
return(euroStatApiEndpoint + table.tableCode
+ '?' + StaticFilters
+ GetNaceFilters(index, naces)
+ GetTimeFilters(StartYear, EndYear)
+ '&' + table.filters);
}
public async Task <List <NaceWithHasData> > getAllNacesWithHasData(int regionId, int tableId)
{
EuroStatTable table = await db.EuroStatTable.FirstAsync(table => table.tableId == tableId);
string tableName = table.attributeName;
System.Reflection.PropertyInfo prop = typeof(NaceRegionData).GetProperty(tableName);
return(db_wp.Nace.ToList().Select(nace => new NaceWithHasData
{
naceId = nace.naceId,
naceName = nace.naceName,
naceCode = nace.naceCode,
hasData = db.NaceRegionData.Where(nrd => nrd.naceId == nace.naceId && nrd.regionId == regionId)
.ToList()
.Find(nrd => prop.GetValue(nrd) != null) != null
}
).ToList());
}
/// <summary>
/// Based on status code and response message the following cases of unsuccessfull fetches are handled:
/// * Eurostat API is unavailable
/// * Asking for data that does not exist
/// * Other error responses are handled collectively by logging the error message
/// </summary>
/// <param name="statusCode">Status Code received from Eurostat</param>
/// <param name="response">Object fetched from Eurostat</param>
/// <param name="table">Representation of the dataset. Has information needed to fetch.</param>
/// <param name="url">The URL for which the fetch was made.</param>
/// <returns></returns>
private async Task handleStatusCodeNotSuccess(int statusCode, HttpResponseMessage response, EuroStatTable table, string url)
{
int timeoutCounter = 0;
while (statusCode == 503 && timeoutCounter < 10)
{
response = await client.GetAsync(url);
System.Threading.Thread.Sleep(100);
timeoutCounter++;
if (timeoutCounter >= 10)
{
_logger.LogWarning("Warning: Service unavailable for fetching data from eurostat on URL: " + url);
}
}
if (GetErrorMessage(response).Equals("Dataset contains no data. One or more filtering elements (query parameters) are probably invalid."))
{
_logger.LogInformation(message: "Dataset contains no data. Nothing was stored from the fetch to: " + url);
}
else if (statusCode >= 400 && !(statusCode == 503 && timeoutCounter >= 10))
{
_logger.LogWarning("For dataset: " + table.attributeName + ". ERROR: " + GetErrorMessage(response));
}
}
