public async Task DoAsync()
{
var bucketName = "example-datasets";
var objectName = "tweets_tiny_no_headers.csv";
objectName = "tweets_tiny.csv";
objectName = "tweets.csv";
var mappingName = "tweets.csv.mapping";
objectName = "tweets.apple.json";
mappingName = "tweets.json.mapping";
var attributesDict = new Dictionary <string, string> {
{ "bucketName", bucketName },
{ "objectName", objectName },
{ "mappingName", mappingName }
};
var attributes = new DataAcquirerAttributes(attributesDict);
var guid = Guid.NewGuid();
var daInput = new DataAcquirerInputModel(
guid,
null,
null,
attributes,
0
);
var posts = _dataAcquirer.GetPostsAsync(daInput, CancellationToken.None);
await foreach (var item in posts)
{
Console.WriteLine(item.Text);
}
}
private async Task RunJobAsync(DataAcquirerJobConfig jobConfig,
CancellationToken cancellationToken)
{
try
{
var translate = jobConfig.Attributes.TryGetValue("Translate", out string value)
//&& value.ToLower() == "true"
;
// TODO validate job config
if (!jobConfig.Attributes.ContainsKey("TopicQuery"))
{
_logger.TrackError(
"StartNewJob",
"TopicQuery attribute is not present. Job did not start",
new { jobId = jobConfig.JobId });
return;
}
string queryLanguage = null;
if (jobConfig.Attributes.TryGetValue("Language", out var desiredLanguage))
{
queryLanguage = desiredLanguage;
}
await _dataAcquirerJobStorage.SaveAsync(jobConfig.JobId, jobConfig);
var batchSize = 100;
var dataAcquirerInputModel = DataAcquirerInputModel.FromValues(
jobConfig.JobId,
jobConfig.Attributes["TopicQuery"],
queryLanguage,
new DataAcquirerAttributes(jobConfig.Attributes),
batchSize
);
var batch = _acquirer.GetPostsAsync(
dataAcquirerInputModel,
cancellationToken);
_logger.TrackInfo("MessageTracking", "Starting");
await ProcessBatch(jobConfig, dataAcquirerInputModel, batch, translate);
}
catch (TaskCanceledException) { }
catch (Exception e)
{
_runningJobsRecords.Remove(jobConfig.JobId);
_logger.TrackError(
"RunJob",
"Job encountered an error and stopped.",
new
{
jobId = jobConfig.JobId,
exception = e
});
}
}
