public Task <string> AggregateDataAsync(UserIdService userIdService,
string databaseFolderPath,
AggregationSetup setup,
CancellationToken cancellationToken)
{
if (setup.Begin != setup.Begin.Date)
{
throw new ValidationException("The begin parameter must have no time component.");
}
if (setup.End != setup.End.Date)
{
throw new ValidationException("The end parameter must have no time component.");
}
return(Task.Run(() =>
{
// log
var message = $"User '{userIdService.GetUserId()}' aggregates data: {setup.Begin.ToISO8601()} to {setup.End.ToISO8601()} ... ";
_logger.LogInformation(message);
try
{
var progress = (IProgress <ProgressUpdatedEventArgs>) this.Progress;
var instructions = AggregationService.ComputeInstructions(setup, _databaseManager.State, _logger);
var days = (setup.End - setup.Begin).TotalDays;
var totalDays = instructions.Count() * days;
var i = 0;
foreach (var instruction in instructions)
{
var projectId = instruction.Container.Id;
for (int j = 0; j < days; j++)
{
cancellationToken.ThrowIfCancellationRequested();
var currentDay = setup.Begin.AddDays(j);
var progressMessage = $"Processing project '{projectId}': {currentDay.ToString("yyyy-MM-dd")}";
var progressValue = (i * days + j) / totalDays;
var eventArgs = new ProgressUpdatedEventArgs(progressValue, progressMessage);
progress.Report(eventArgs);
this.AggregateProject(userIdService.User, databaseFolderPath, projectId, currentDay, setup, instruction, cancellationToken);
}
i++;
}
}
catch (Exception ex)
{
_logger.LogError(ex.GetFullMessage());
throw;
}
_logger.LogInformation($"{message} Done.");
return string.Empty;
}, cancellationToken));
}
public static List <AggregationInstruction> ComputeInstructions(AggregationSetup setup, DatabaseManagerState state, ILogger logger)
{
var projectIds = setup.Aggregations
.Select(aggregation => aggregation.ProjectId)
.Distinct().ToList();
return(projectIds.Select(projectId =>
{
var container = state.Database.ProjectContainers.FirstOrDefault(container => container.Id == projectId);
if (container is null)
{
return null;
}
var dataReaderRegistrations = container
.Project
.Channels
.SelectMany(channel => channel.Datasets.Select(dataset => dataset.Registration))
.Distinct()
.Where(registration => registration != state.AggregationRegistration)
.ToList();
return new AggregationInstruction(container, dataReaderRegistrations.ToDictionary(registration => registration, registration =>
{
// find aggregations for project ID
var potentialAggregations = setup.Aggregations
.Where(parameters => parameters.ProjectId == container.Project.Id)
.ToList();
// create channel to aggregations map
var aggregationChannels = container.Project.Channels
// find all channels for current reader registration
.Where(channel => channel.Datasets.Any(dataset => dataset.Registration == registration))
// find all aggregations for current channel
.Select(channel =>
{
var channelMeta = container.ProjectMeta.Channels
.First(current => current.Id == channel.Id);
return new AggregationChannel()
{
Channel = channel,
Aggregations = potentialAggregations.Where(current => AggregationService.ApplyAggregationFilter(channel, channelMeta, current.Filters, logger)).ToList()
};
})
// take all channels with aggregations
.Where(aggregationChannel => aggregationChannel.Aggregations.Any());
return aggregationChannels.ToList();
}));
}).Where(instruction => instruction != null).ToList());
}
public ActionResult <AggregationJob> CreateAggregationJob(AggregationSetup setup)
{
if (_databaseManager.Database == null)
{
return(this.StatusCode(503, "The database has not been loaded yet."));
}
setup.Begin = setup.Begin.ToUniversalTime();
setup.End = setup.End.ToUniversalTime();
// security check
if (!this.User.HasClaim(Claims.IS_ADMIN, "true"))
{
return(this.Unauthorized($"The current user is not authorized to create an aggregation job."));
}
//
var job = new AggregationJob()
{
Owner = this.User.Identity.Name,
Setup = setup
};
var aggregationService = _serviceProvider.GetRequiredService <AggregationService>();
try
{
var jobControl = _aggregationJobService.AddJob(job, aggregationService.Progress, (jobControl, cts) =>
{
var userIdService = _serviceProvider.GetRequiredService <UserIdService>();
var task = aggregationService.AggregateDataAsync(
userIdService,
_options.DataBaseFolderPath,
setup,
cts.Token);
return(task);
});
return(this.Accepted($"{this.GetBasePath()}{this.Request.Path}/{jobControl.Job.Id}/status", jobControl.Job));
}
catch (ValidationException ex)
{
return(this.UnprocessableEntity(ex.GetFullMessage(includeStackTrace: false)));
}
}
private void AggregateProject(ClaimsPrincipal user,
string databaseFolderPath,
string projectId,
DateTime date,
AggregationSetup setup,
AggregationInstruction instruction,
CancellationToken cancellationToken)
{
foreach (var(registration, aggregationChannels) in instruction.DataReaderToAggregationsMap)
{
using var dataReader = _databaseManager.GetDataReader(user, registration);
// find reader configurations
foreach (var configuration in setup.ReaderConfigurations
.Where(configuration => configuration.ProjectId == projectId))
{
var tmpRegistration = new DataReaderRegistration()
{
RootPath = configuration.DataReaderRootPath,
DataReaderId = configuration.DataReaderId
};
if (dataReader.Registration.Equals(tmpRegistration))
{
dataReader.OptionalParameters = configuration.Parameters;
break;
}
}
// get files
if (!dataReader.IsDataOfDayAvailable(projectId, date))
{
return;
}
// project
var container = _databaseManager.Database.ProjectContainers.FirstOrDefault(container => container.Id == projectId);
if (container == null)
{
throw new Exception($"The requested project '{projectId}' could not be found.");
}
var targetDirectoryPath = Path.Combine(databaseFolderPath, "DATA", WebUtility.UrlEncode(container.Id), date.ToString("yyyy-MM"), date.ToString("dd"));
// for each channel
foreach (var aggregationChannel in aggregationChannels)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var dataset = aggregationChannel.Channel.Datasets.First();
NexusUtilities.InvokeGenericMethod(this, nameof(this.OrchestrateAggregation),
BindingFlags.Instance | BindingFlags.NonPublic,
NexusUtilities.GetTypeFromNexusDataType(dataset.DataType),
new object[]
{
targetDirectoryPath,
dataReader,
dataset,
aggregationChannel.Aggregations,
date,
setup.Force,
cancellationToken
});
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex.GetFullMessage());
}
}
}
}
