public Task <string> AggregateDataAsync(UserIdService userIdService, string databaseFolderPath, AggregationSetup setup, CancellationToken cancellationToken) { if (setup.Begin != setup.Begin.Date) { throw new ValidationException("The begin parameter must have no time component."); } if (setup.End != setup.End.Date) { throw new ValidationException("The end parameter must have no time component."); } return(Task.Run(() => { // log var message = $"User '{userIdService.GetUserId()}' aggregates data: {setup.Begin.ToISO8601()} to {setup.End.ToISO8601()} ... "; _logger.LogInformation(message); try { var progress = (IProgress <ProgressUpdatedEventArgs>) this.Progress; var instructions = AggregationService.ComputeInstructions(setup, _databaseManager.State, _logger); var days = (setup.End - setup.Begin).TotalDays; var totalDays = instructions.Count() * days; var i = 0; foreach (var instruction in instructions) { var projectId = instruction.Container.Id; for (int j = 0; j < days; j++) { cancellationToken.ThrowIfCancellationRequested(); var currentDay = setup.Begin.AddDays(j); var progressMessage = $"Processing project '{projectId}': {currentDay.ToString("yyyy-MM-dd")}"; var progressValue = (i * days + j) / totalDays; var eventArgs = new ProgressUpdatedEventArgs(progressValue, progressMessage); progress.Report(eventArgs); this.AggregateProject(userIdService.User, databaseFolderPath, projectId, currentDay, setup, instruction, cancellationToken); } i++; } } catch (Exception ex) { _logger.LogError(ex.GetFullMessage()); throw; } _logger.LogInformation($"{message} Done."); return string.Empty; }, cancellationToken)); }
public static List <AggregationInstruction> ComputeInstructions(AggregationSetup setup, DatabaseManagerState state, ILogger logger) { var projectIds = setup.Aggregations .Select(aggregation => aggregation.ProjectId) .Distinct().ToList(); return(projectIds.Select(projectId => { var container = state.Database.ProjectContainers.FirstOrDefault(container => container.Id == projectId); if (container is null) { return null; } var dataReaderRegistrations = container .Project .Channels .SelectMany(channel => channel.Datasets.Select(dataset => dataset.Registration)) .Distinct() .Where(registration => registration != state.AggregationRegistration) .ToList(); return new AggregationInstruction(container, dataReaderRegistrations.ToDictionary(registration => registration, registration => { // find aggregations for project ID var potentialAggregations = setup.Aggregations .Where(parameters => parameters.ProjectId == container.Project.Id) .ToList(); // create channel to aggregations map var aggregationChannels = container.Project.Channels // find all channels for current reader registration .Where(channel => channel.Datasets.Any(dataset => dataset.Registration == registration)) // find all aggregations for current channel .Select(channel => { var channelMeta = container.ProjectMeta.Channels .First(current => current.Id == channel.Id); return new AggregationChannel() { Channel = channel, Aggregations = potentialAggregations.Where(current => AggregationService.ApplyAggregationFilter(channel, channelMeta, current.Filters, logger)).ToList() }; }) // take all channels with aggregations .Where(aggregationChannel => aggregationChannel.Aggregations.Any()); return aggregationChannels.ToList(); })); }).Where(instruction => instruction != null).ToList()); }
public ActionResult <AggregationJob> CreateAggregationJob(AggregationSetup setup) { if (_databaseManager.Database == null) { return(this.StatusCode(503, "The database has not been loaded yet.")); } setup.Begin = setup.Begin.ToUniversalTime(); setup.End = setup.End.ToUniversalTime(); // security check if (!this.User.HasClaim(Claims.IS_ADMIN, "true")) { return(this.Unauthorized($"The current user is not authorized to create an aggregation job.")); } // var job = new AggregationJob() { Owner = this.User.Identity.Name, Setup = setup }; var aggregationService = _serviceProvider.GetRequiredService <AggregationService>(); try { var jobControl = _aggregationJobService.AddJob(job, aggregationService.Progress, (jobControl, cts) => { var userIdService = _serviceProvider.GetRequiredService <UserIdService>(); var task = aggregationService.AggregateDataAsync( userIdService, _options.DataBaseFolderPath, setup, cts.Token); return(task); }); return(this.Accepted($"{this.GetBasePath()}{this.Request.Path}/{jobControl.Job.Id}/status", jobControl.Job)); } catch (ValidationException ex) { return(this.UnprocessableEntity(ex.GetFullMessage(includeStackTrace: false))); } }
private void AggregateProject(ClaimsPrincipal user, string databaseFolderPath, string projectId, DateTime date, AggregationSetup setup, AggregationInstruction instruction, CancellationToken cancellationToken) { foreach (var(registration, aggregationChannels) in instruction.DataReaderToAggregationsMap) { using var dataReader = _databaseManager.GetDataReader(user, registration); // find reader configurations foreach (var configuration in setup.ReaderConfigurations .Where(configuration => configuration.ProjectId == projectId)) { var tmpRegistration = new DataReaderRegistration() { RootPath = configuration.DataReaderRootPath, DataReaderId = configuration.DataReaderId }; if (dataReader.Registration.Equals(tmpRegistration)) { dataReader.OptionalParameters = configuration.Parameters; break; } } // get files if (!dataReader.IsDataOfDayAvailable(projectId, date)) { return; } // project var container = _databaseManager.Database.ProjectContainers.FirstOrDefault(container => container.Id == projectId); if (container == null) { throw new Exception($"The requested project '{projectId}' could not be found."); } var targetDirectoryPath = Path.Combine(databaseFolderPath, "DATA", WebUtility.UrlEncode(container.Id), date.ToString("yyyy-MM"), date.ToString("dd")); // for each channel foreach (var aggregationChannel in aggregationChannels) { cancellationToken.ThrowIfCancellationRequested(); try { var dataset = aggregationChannel.Channel.Datasets.First(); NexusUtilities.InvokeGenericMethod(this, nameof(this.OrchestrateAggregation), BindingFlags.Instance | BindingFlags.NonPublic, NexusUtilities.GetTypeFromNexusDataType(dataset.DataType), new object[] { targetDirectoryPath, dataReader, dataset, aggregationChannel.Aggregations, date, setup.Force, cancellationToken }); } catch (TaskCanceledException) { throw; } catch (Exception ex) { _logger.LogError(ex.GetFullMessage()); } } } }