private async Task RunJobAsync(DataAcquirerJobConfig jobConfig,
CancellationToken cancellationToken)
{
try
{
var translate = jobConfig.Attributes.TryGetValue("Translate", out string value)
//&& value.ToLower() == "true"
;
// TODO validate job config
if (!jobConfig.Attributes.ContainsKey("TopicQuery"))
{
_logger.TrackError(
"StartNewJob",
"TopicQuery attribute is not present. Job did not start",
new { jobId = jobConfig.JobId });
return;
}
string queryLanguage = null;
if (jobConfig.Attributes.TryGetValue("Language", out var desiredLanguage))
{
queryLanguage = desiredLanguage;
}
await _dataAcquirerJobStorage.SaveAsync(jobConfig.JobId, jobConfig);
var batchSize = 100;
var dataAcquirerInputModel = DataAcquirerInputModel.FromValues(
jobConfig.JobId,
jobConfig.Attributes["TopicQuery"],
queryLanguage,
new DataAcquirerAttributes(jobConfig.Attributes),
batchSize
);
var batch = _acquirer.GetPostsAsync(
dataAcquirerInputModel,
cancellationToken);
_logger.TrackInfo("MessageTracking", "Starting");
await ProcessBatch(jobConfig, dataAcquirerInputModel, batch, translate);
}
catch (TaskCanceledException) { }
catch (Exception e)
{
_runningJobsRecords.Remove(jobConfig.JobId);
_logger.TrackError(
"RunJob",
"Job encountered an error and stopped.",
new
{
jobId = jobConfig.JobId,
exception = e
});
}
}
