private async Task RunJobAsync(DataAcquirerJobConfig jobConfig,
CancellationToken cancellationToken)
{
try
{
var translate = jobConfig.Attributes.TryGetValue("Translate", out string value)
//&& value.ToLower() == "true"
;
// TODO validate job config
if (!jobConfig.Attributes.ContainsKey("TopicQuery"))
{
_logger.TrackError(
"StartNewJob",
"TopicQuery attribute is not present. Job did not start",
new { jobId = jobConfig.JobId });
return;
}
string queryLanguage = null;
if (jobConfig.Attributes.TryGetValue("Language", out var desiredLanguage))
{
queryLanguage = desiredLanguage;
}
await _dataAcquirerJobStorage.SaveAsync(jobConfig.JobId, jobConfig);
var batchSize = 100;
var dataAcquirerInputModel = DataAcquirerInputModel.FromValues(
jobConfig.JobId,
jobConfig.Attributes["TopicQuery"],
queryLanguage,
new DataAcquirerAttributes(jobConfig.Attributes),
batchSize
);
var batch = _acquirer.GetPostsAsync(
dataAcquirerInputModel,
cancellationToken);
_logger.TrackInfo("MessageTracking", "Starting");
await ProcessBatch(jobConfig, dataAcquirerInputModel, batch, translate);
}
catch (TaskCanceledException) { }
catch (Exception e)
{
_runningJobsRecords.Remove(jobConfig.JobId);
_logger.TrackError(
"RunJob",
"Job encountered an error and stopped.",
new
{
jobId = jobConfig.JobId,
exception = e
});
}
}
public async Task <ITwitterContext> GetContextAsync(
TwitterCredentials credentials)
{
credentials = credentials ?? throw new ArgumentNullException(nameof(credentials));
var props = new[]
{
credentials.AccessToken,
credentials.AccessTokenSecret,
credentials.ConsumerKey,
credentials.ConsumerSecret
};
if (props.Any(r => string.IsNullOrEmpty(r)))
{
throw new ArgumentException("Credentials contain null value", nameof(credentials));
}
var key = string.Join('+', props);
if (_contextPerUser.TryGetValue(key, out var context))
{
return(context);
}
try
{
var newContext = CreateContext(credentials);
try
{
await newContext.GetStatusBatchAsync(
"test",
1,
null);
}
catch (TwitterQueryException e) when(e.ErrorCode == 88)
{
// this is ok
}
_contextPerUser.TryAdd(key, newContext);
return(newContext);
}
catch (Exception e)
{
_eventTracker.TrackError(
"TwitterContext",
"Encountered error while creating context",
new
{
exception = e
});
throw;
}
}
private async Task ProcessJobConfigAsync(string configJson)
{
DataAcquirerJobConfig jobConfig;
try
{
jobConfig = JsonConvert.DeserializeObject <DataAcquirerJobConfig>(configJson);
}
catch (JsonException jre)
{
_logger.TrackError(
"StartNewJob",
"Could not parse job config",
new
{
config = configJson,
exception = jre
});
throw new InvalidOperationException($"Could not parse job config {jre.Message}");
}
switch (jobConfig.Command)
{
case JobCommand.Start:
try
{
await _jobManager.StartNewJobAsync(jobConfig);
}
catch (JobException e)
{
_logger.TrackError(
"StartNewJob",
"Job failed to start",
new
{
jobId = jobConfig.JobId,
exception = e
});
}
break;
case JobCommand.Stop:
await _jobManager.StopJobAsync(jobConfig.JobId);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public async Task <ActionResult> Produce([FromBody] ProduceRequest request)
{
_eventTracker.TrackInfo("JmsDebugProducing", "produced message");
try
{
var kafkaOptions = Options.Create(new KafkaOptions {
ServerAddress = request.ServerAddress
});
var taskOptions =
Options.Create(new TaskOptions {
ConsumeTaskTopic = "n/a", ProduceTopic = request.KafkaTopic
});
var kafka = new KafkaResultProducer(kafkaOptions, taskOptions, _kafkaLogger);
var message = new Message()
{
Key = "key",
Value = request.Message
};
await kafka.ProduceAsync(message);
}
catch (Exception e)
{
_eventTracker.TrackError(
"JmsDebugProducing",
$"Error while producing: {e.Message}",
new { exception = e });
}
return(Ok());
}
public async Task <SubscribedComponentResultModel> SubscribeComponentAsync(
ComponentModel componentRegistrationModel)
{
try
{
await _componentRegistry.AddOrUpdateAsync(componentRegistrationModel);
return(SubscribedComponentResultModel.Successful());
}
catch (Exception e)
{
string error = $"Subscription failed due to: {e.Message}";
_logger.TrackError("SubscribeComponent", error, new { componentRegistrationModel, exception = e });
return(SubscribedComponentResultModel.Failed(error));
}
}
private async Task <MappingAttributesRoot> GetMappingAttributesAsync(
MinioClient minio,
string bucketName,
string mappingName)
{
try
{
await minio.StatObjectAsync(bucketName, mappingName);
}
catch (MinioException e)
{
_logger.TrackError(
"CustomDataAcquirer",
$"Missing mapping object '{bucketName}-{mappingName}'",
new
{
bucketName,
mappingName,
exception = e
});
return(null);
}
try
{
return(await ParseMappingAttributesAsync(minio, bucketName, mappingName));
}
catch (JsonException e)
{
_logger.TrackError(
"CustomDataAcquirer",
$"could not parse json from '{bucketName}-{mappingName}'",
new
{
bucketName,
mappingName,
exception = e
});
return(null);
}
}
public async Task <ActionResult <AggregationAnalysisResponse> > GetJobAnalysisAggregation([FromRoute] Guid jobId, [FromBody] GetAggregationAnalysisRequest analysisRequest)
{
_eventTracker.TrackInfo("GetJobAnalysisAggregation", $"User '{User.Identity.Name}' requested aggregation analysis", analysisRequest);
if (!await _authorizationService.IsUserAuthorizedToSeeJob(User.Identity.Name, jobId))
{
_eventTracker.TrackInfo("GetJobAnalysisAggregation", $"User '{User.Identity.Name}' is not authorized to see job '{jobId}'");
return(Unauthorized());
}
try
{
var analysisResult = await _getAnalysisService.GetAggregationAnalysis(jobId, analysisRequest.AnalyserId,
analysisRequest.AnalysisProperty);
var analysisResponse = AggregationAnalysisResponse.FromModel(analysisResult);
return(Ok(analysisResponse));
}
catch (GetAnalysisService.GetAnalysisException e)
{
_eventTracker.TrackError("GetAggregationAnalysis", e.Message, analysisRequest);
return(Ok(AggregationAnalysisResponse.Empty()));
}
}
public Task <RedditClient> GetContextAsync(RedditCredentials credentials)
{
credentials = credentials ?? throw new ArgumentNullException(nameof(credentials));
var props = new[]
{
credentials.AppId,
credentials.ApiSecret,
credentials.RefreshToken
};
if (props.Any(r => string.IsNullOrEmpty(r)))
{
throw new ArgumentException("Credentials field contains null value", nameof(credentials));
}
var key = string.Join('+', props);
if (_redditClients.TryGetValue(key, out var context))
{
return(Task.FromResult(context));
}
try
{
var redditContext = new RedditClient(
appId: credentials.AppId,
appSecret: credentials.ApiSecret,
refreshToken: credentials.RefreshToken);
//var searchInput = new SearchGetSearchInput(q: "foo bar", count: 5);
//var s = redditContext.Search(searchInput);
_redditClients.TryAdd(key, redditContext);
return(Task.FromResult(redditContext));
}
catch (Exception e)
{
_eventTracker.TrackError(
"RedditContext",
"Encountered error while creating context",
new
{
exception = e
});
throw;
}
}
private async IAsyncEnumerable <IList <Status> > QueryPastPostAsync(
ITwitterContext context,
TwitterQueryInput acquirerInputModel)
{
var searchTerm = acquirerInputModel.SearchTerm;
var language = acquirerInputModel.Language ?? "";
var maxId = acquirerInputModel.MaxId;
var sinceId = acquirerInputModel.SinceId;
var batchSize = acquirerInputModel.BatchSize;
while (true)
{
var batch = new List <Status>();
try
{
_logger.TrackInfo(
"QueryData",
"Downloading data");
var statuses = await context.GetStatusBatchAsync(
searchTerm,
batchSize,
language,
maxId : maxId,
sinceId : sinceId);
batch = statuses
.ToList();
maxId = statuses.Any()
? Math.Min(statuses.Min(status => status.StatusID) - 1, acquirerInputModel.MaxId)
: acquirerInputModel.MaxId;
}
catch (TwitterQueryException e) when(e.ErrorCode == 88)
{
Console.WriteLine("Rate limit exceeded - waiting");
await Task.Delay(_options.RateLimitExceededWaitDelay);
continue;
}
catch (Exception e)
{
_logger.TrackError(
"QueryData",
"Unexpected error",
new
{
exception = e,
jobId = _jobId,
input = acquirerInputModel
});
await Task.Delay(_options.ErrorEncounteredWaitDelay);
continue;
}
if (batch.Count == 0)
{
yield break;
}
yield return(batch);
}
}