/// <summary>
/// ReduceGeojsonFeatures
/// </summary>
/// <param name="sourceGeojson"></param>
/// <param name="reductionFactor"></param>
private void ReduceGeojsonFeatures(ref GeoJsonDto sourceGeojson, float reductionFactor = AggregatorService.DefaultCoordinatesReductionFactor)
{
//--- Process daily file: reduce the number of features
var totalInitialCoordinatesCount = 0;
var totalReducedCoordinatesCount = 0;
foreach (var f in sourceGeojson.features)
{
// Reduce the number of coordinates per feature
totalInitialCoordinatesCount += f.geometry.coordinates.Count;
var reducedCoordiantes = f.geometry.coordinates.Where((_, i) => i % reductionFactor == 0).ToList();
f.geometry.coordinates = reducedCoordiantes;
totalReducedCoordinatesCount += reducedCoordiantes.Count;
}
_logger.LogInformation($"Reduced Coordinates: {totalReducedCoordinatesCount} = {totalInitialCoordinatesCount} / {reductionFactor}");
}
private void CropGeoJsonFeatures(ref GeoJsonDto sourceGeojson)
{
//--- Process daily file: reduce the number of features
var totalInitialCoordinatesCount = 0;
var totalReducedCoordinatesCount = 0;
foreach (var f in sourceGeojson.features)
{
// Reduce the number of coordinates per feature
totalInitialCoordinatesCount += f.geometry.coordinates.Count;
var reducedCoordiantes = f.geometry.coordinates.Where((c, i) => IsInsideBoundingBox(c[0], c[1])).ToList();
f.geometry.coordinates = reducedCoordiantes;
totalReducedCoordinatesCount += reducedCoordiantes.Count;
}
_logger.LogInformation($"Remove out of the box coordinates: {totalReducedCoordinatesCount} = {totalInitialCoordinatesCount}");
}
/// <summary>
/// Run
/// Default Aggregation entry point: process backlog and integrated reduced daily files into yearly cumulative tracks file.
/// </summary>
/// <param name="keepBacklog">Set to true to preserve backlog (files will not be deleted once processed)</param>
/// <returns></returns>
public async Task Run(float?reductionFactor = AggregatorService.DefaultCoordinatesReductionFactor,
bool keepBacklog = false)
{
_logger.LogInformation("######### [AggregatorService] Building yearly cumulative track ... ###########");
var appliedReductionFactor = reductionFactor.HasValue ? reductionFactor.Value : AggregatorService.DefaultCoordinatesReductionFactor;
// --- List files to be processed in backlog ---
var backlogFilesList = _storageService.GetFilenameList("backlog");
backlogFilesList = backlogFilesList.Where(f => Path.GetExtension(f) == ".zip").ToList(); // Keep zip files only
if (backlogFilesList.Count > 0)
{
// --- Get yearly geojson
_logger.LogInformation($"Downloading zip file from bucket: {_configurationService.TraceAggregatorBucketName} / {YearlyTracksZipFilename}");
// --- Get yearly geojson
GeoJsonDto yearGeojson = null;
List <string> yearFlightIdList = null; // FlightIds in the yearly aggregation
try
{
var yearlyGeojsonFile = await _zipStorageService.DownloadZipedFileAsStringAsync(YearlyTracksZipFilename);
yearGeojson = JsonHelper.Deserialize <GeoJsonDto>(yearlyGeojsonFile);
yearFlightIdList = yearGeojson.features.Select(f => f.properties.flightId).ToList();
}
catch (Google.GoogleApiException e)
{
_logger.LogWarning($"[AggregatorService]: {e.Message}");
yearGeojson = new GeoJsonFeatureCollectionDto();
yearFlightIdList = new List <string>();
}
// -- Get yearly metadata
TracksMetadataDto metadataDto = null;
try
{
var metadatajson = await _storageService.DownloadObjectFromBucketAsStringAsync(YearlyTtrackMetadataFilename);
metadataDto = JsonHelper.Deserialize <TracksMetadataDto>(metadatajson);
}
catch (Google.GoogleApiException e)
{
_logger.LogWarning($"[AggregatorService]: {e.Message}");
metadataDto = new TracksMetadataDto();
}
// --- Process files ---
var currentFileCount = 0;
var totalFileCount = backlogFilesList.Count;
var addedFeatureCount = 0;
foreach (var filename in backlogFilesList)
{
// --- Get daily geojson
_logger.LogInformation($"{currentFileCount}/{totalFileCount} Integrating file into yearly tracks: {filename}");
currentFileCount++;
var tracksForDayAsJson = await _zipStorageService.DownloadZipedFileAsStringAsync(filename);
var dayGeojson = JsonHelper.Deserialize <GeoJsonDto>(tracksForDayAsJson);
// Get rid of features that are already present in the yearly aggregation
//var newFeatures = dayGeojson.features.Where(f => !yearFlightIdList.Contains(f.properties.flightId)).ToList();
var newFeatures = dayGeojson.features;
if (newFeatures.Count > 0)
{
_logger.LogInformation($"Adding new feature to the aggregated file. New features count= {newFeatures.Count}");
dayGeojson.features = newFeatures;
addedFeatureCount += newFeatures.Count;
//--- Reduce number of features
ReduceGeojsonFeatures(ref dayGeojson, appliedReductionFactor);
yearGeojson.features.AddRange(dayGeojson.features); // Add the features to the yearly aggregation
// --- Crop to eliminate points outside of the bounding box ---
CropGeoJsonFeatures(ref dayGeojson);
}
// --- Delete the processed file from the backlog
if (!keepBacklog)
{
await _storageService.DeleteFileAsync(filename);
}
}
if (addedFeatureCount > 0)
{
// --- Update metadata ---
metadataDto.ScriptEndTime = DateTime.UtcNow;
metadataDto.TargetDate = DateTime.UtcNow;
metadataDto.FlightsCount += addedFeatureCount;
metadataDto.ProcessedFlightsCount += addedFeatureCount;
// --- Store yearly geojson file with new feature added ---
_logger.LogInformation("Storing new yearly tracemap into bucket ...");
var yearlyGeojsonText = JsonHelper.Serialize(yearGeojson);
await _zipStorageService.UploadStringToZipFileAsync(yearlyGeojsonText, YearlyTracksFilename, YearlyTracksZipFilename); // Store into a GCP bucket: geojson
// --- Store updated metadata ---
_logger.LogInformation("Storing updated metadata file into bucket ...");
var metadataText = JsonHelper.Serialize(metadataDto);
await _storageService.UploadToBucketAsync(YearlyTtrackMetadataFilename, metadataText);
}
else
{
_logger.LogInformation("No new feature to add.");
}
_logger.LogInformation("######### [AggregatorService] ######### Done !");
}
else
{
_logger.LogInformation("######### [AggregatorService] ######### No file to process in the backlog. Done !");
}
}
