public Task StartNewJobAsync(DataAcquirerJobConfig jobConfig)
{
lock (_dictionaryLock)
{
var jobId = jobConfig.JobId;
if (_isStopping)
{
_logger.TrackWarning(
"StartNewJob",
"Could not start job, because the component is stopping",
new { jobId = jobId });
return(Task.CompletedTask);
}
if (_runningJobsRecords.ContainsKey(jobId))
{
_logger.TrackWarning(
"StartNewJob",
"Job is with this id already running",
new { jobId = jobId });
return(Task.CompletedTask);
}
_logger.TrackInfo(
"StartNewJob",
"Config recieved",
new { config = jobConfig });
var cancellationTokenSource = new CancellationTokenSource();
var downloadingTask = RunJobAsync(jobConfig, cancellationTokenSource.Token).
ContinueWith(async r =>
{
try
{
await r;
}
catch (TaskCanceledException) { }
_runningJobsRecords.Remove(jobId, out _);
_logger.TrackInfo(
"StartNewJob",
"Job removed",
new { jobId = jobId.ToString() });
});
var jobManagerJobRecord = new JobManagerJobRecord
{
JobId = jobConfig.JobId,
JobTask = downloadingTask,
CancellationTokenSource = cancellationTokenSource
};
_runningJobsRecords.TryAdd(jobManagerJobRecord.JobId, jobManagerJobRecord);
return(Task.CompletedTask);
}
}
private async Task <List <ComponentModel> > PushAnalyserJobConfig(
string storageChannelName,
JobConfigUpdateCommand jobConfigUpdateCommand)
{
var analysers = new List <ComponentModel>();
foreach (var analyser in jobConfigUpdateCommand.DataAnalysers)
{
var analyserComponent = await _componentRegistry
.GetComponentByIdAsync(analyser);
if (analyserComponent == null)
{
_logger.TrackWarning(
"ComponentConfig",
$"Analyser {analyser} was not registered");
}
else
{
analysers.Add(analyserComponent);
}
}
var notification = new AnalyserConfigUpdateNotification()
{
JobId = jobConfigUpdateCommand.JobId,
Attributes = new Dictionary <string, string>(),
OutputMessageBrokerChannels = new[] { storageChannelName },
};
var configUpdateTasks = analysers.Select(async analyserCmp =>
{
_logger.TrackInfo(
"ComponentConfig",
$"Config pushed to: {analyserCmp}, updateChannelName: {analyserCmp.UpdateChannelName}",
new
{
config = notification
});
await _componentConfigUpdateNotifier.NotifyComponentAsync(
analyserCmp.UpdateChannelName,
notification);
var componentConfig = new JobComponentConfig
{
ComponentId = analyserCmp.ComponentId,
Attributes = analyserCmp.Attributes,
JobId = jobConfigUpdateCommand.JobId,
OutputMessageBrokerChannels = notification.OutputMessageBrokerChannels
};
await _componentRegistry.InsertJobComponentConfigAsync(componentConfig);
});
await Task.WhenAll(configUpdateTasks);
return(analysers);
}
public Task SaveAsync(Guid jobId, DataAcquirerJobConfig jobConfig)
{
if (_jobConfigs.TryAdd(jobId, jobConfig))
{
_eventTracker.TrackWarning(
"AcquirerJobsAlreadyExists",
"The job for acquirer already exists",
new
{
jobId = jobId,
config = jobConfig
});
}
return(Task.CompletedTask);
}
public async IAsyncEnumerable <DataAcquirerPost> GetPostsAsync(
DataAcquirerInputModel acquirerInputModel,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var minio = new MinioClient(
_endpoint,
_accessKey,
_secret);
var attributes = acquirerInputModel.Attributes;
var bucketName = attributes.GetValue(_bucketElementName, null);
var objectName = attributes.GetValue(_objectElementName, null);
var mappingName = attributes.GetValue(_mappingElementName, null);
var atts = await GetMappingAttributesAsync(minio, bucketName, mappingName);
if (atts == null ||
string.IsNullOrEmpty(atts.DataFormat) ||
atts.MappingAttributes == null)
{
throw new InvalidOperationException("Invalid config");
}
try
{
await minio.StatObjectAsync(bucketName, objectName);
}
catch (MinioException)
{
//_logger.TrackError(
// "CustomDataAcquirer",
// $"Object '{bucketName}-{objectName}' does not exist",
// new
// {
// bucketName,
// objectName,
// exception = e
// });
throw new InvalidOperationException($"Mapping object {mappingName} does not exits");
}
var reader = _customStreamReaderFactory.Create(atts);
using (var cts = new CancellationTokenSource())
{
var listeningTask = Task.Run(async() =>
{
await minio.GetObjectAsync(bucketName, objectName, reader.StartPopulating)
.ConfigureAwait(false);
});
while (!reader.IsCompleted)
{
if (reader.TryGetPost(out var post))
{
var validation = PostValidator.ValidatePost(post);
if (validation.IsSuccessful)
{
yield return(post);
}
else
{
_logger.TrackWarning(
"CustomStaticData",
"Invalid post encountered",
new
{
errorMessage = validation.ErrorMessage,
post
});
}
}
else
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
}
cts.Cancel();
try
{
await listeningTask;
}
catch (TaskCanceledException) { }
}
}
