public Task StartNewJobAsync(DataAcquirerJobConfig jobConfig) { lock (_dictionaryLock) { var jobId = jobConfig.JobId; if (_isStopping) { _logger.TrackWarning( "StartNewJob", "Could not start job, because the component is stopping", new { jobId = jobId }); return(Task.CompletedTask); } if (_runningJobsRecords.ContainsKey(jobId)) { _logger.TrackWarning( "StartNewJob", "Job is with this id already running", new { jobId = jobId }); return(Task.CompletedTask); } _logger.TrackInfo( "StartNewJob", "Config recieved", new { config = jobConfig }); var cancellationTokenSource = new CancellationTokenSource(); var downloadingTask = RunJobAsync(jobConfig, cancellationTokenSource.Token). ContinueWith(async r => { try { await r; } catch (TaskCanceledException) { } _runningJobsRecords.Remove(jobId, out _); _logger.TrackInfo( "StartNewJob", "Job removed", new { jobId = jobId.ToString() }); }); var jobManagerJobRecord = new JobManagerJobRecord { JobId = jobConfig.JobId, JobTask = downloadingTask, CancellationTokenSource = cancellationTokenSource }; _runningJobsRecords.TryAdd(jobManagerJobRecord.JobId, jobManagerJobRecord); return(Task.CompletedTask); } }
private async Task <List <ComponentModel> > PushAnalyserJobConfig( string storageChannelName, JobConfigUpdateCommand jobConfigUpdateCommand) { var analysers = new List <ComponentModel>(); foreach (var analyser in jobConfigUpdateCommand.DataAnalysers) { var analyserComponent = await _componentRegistry .GetComponentByIdAsync(analyser); if (analyserComponent == null) { _logger.TrackWarning( "ComponentConfig", $"Analyser {analyser} was not registered"); } else { analysers.Add(analyserComponent); } } var notification = new AnalyserConfigUpdateNotification() { JobId = jobConfigUpdateCommand.JobId, Attributes = new Dictionary <string, string>(), OutputMessageBrokerChannels = new[] { storageChannelName }, }; var configUpdateTasks = analysers.Select(async analyserCmp => { _logger.TrackInfo( "ComponentConfig", $"Config pushed to: {analyserCmp}, updateChannelName: {analyserCmp.UpdateChannelName}", new { config = notification }); await _componentConfigUpdateNotifier.NotifyComponentAsync( analyserCmp.UpdateChannelName, notification); var componentConfig = new JobComponentConfig { ComponentId = analyserCmp.ComponentId, Attributes = analyserCmp.Attributes, JobId = jobConfigUpdateCommand.JobId, OutputMessageBrokerChannels = notification.OutputMessageBrokerChannels }; await _componentRegistry.InsertJobComponentConfigAsync(componentConfig); }); await Task.WhenAll(configUpdateTasks); return(analysers); }
public Task SaveAsync(Guid jobId, DataAcquirerJobConfig jobConfig) { if (_jobConfigs.TryAdd(jobId, jobConfig)) { _eventTracker.TrackWarning( "AcquirerJobsAlreadyExists", "The job for acquirer already exists", new { jobId = jobId, config = jobConfig }); } return(Task.CompletedTask); }
public async IAsyncEnumerable <DataAcquirerPost> GetPostsAsync( DataAcquirerInputModel acquirerInputModel, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var minio = new MinioClient( _endpoint, _accessKey, _secret); var attributes = acquirerInputModel.Attributes; var bucketName = attributes.GetValue(_bucketElementName, null); var objectName = attributes.GetValue(_objectElementName, null); var mappingName = attributes.GetValue(_mappingElementName, null); var atts = await GetMappingAttributesAsync(minio, bucketName, mappingName); if (atts == null || string.IsNullOrEmpty(atts.DataFormat) || atts.MappingAttributes == null) { throw new InvalidOperationException("Invalid config"); } try { await minio.StatObjectAsync(bucketName, objectName); } catch (MinioException) { //_logger.TrackError( // "CustomDataAcquirer", // $"Object '{bucketName}-{objectName}' does not exist", // new // { // bucketName, // objectName, // exception = e // }); throw new InvalidOperationException($"Mapping object {mappingName} does not exits"); } var reader = _customStreamReaderFactory.Create(atts); using (var cts = new CancellationTokenSource()) { var listeningTask = Task.Run(async() => { await minio.GetObjectAsync(bucketName, objectName, reader.StartPopulating) .ConfigureAwait(false); }); while (!reader.IsCompleted) { if (reader.TryGetPost(out var post)) { var validation = PostValidator.ValidatePost(post); if (validation.IsSuccessful) { yield return(post); } else { _logger.TrackWarning( "CustomStaticData", "Invalid post encountered", new { errorMessage = validation.ErrorMessage, post }); } } else { await Task.Delay(TimeSpan.FromSeconds(1)); } } cts.Cancel(); try { await listeningTask; } catch (TaskCanceledException) { } } }