public async Task DoAsync() { var bucketName = "example-datasets"; var objectName = "tweets_tiny_no_headers.csv"; objectName = "tweets_tiny.csv"; objectName = "tweets.csv"; var mappingName = "tweets.csv.mapping"; objectName = "tweets.apple.json"; mappingName = "tweets.json.mapping"; var attributesDict = new Dictionary <string, string> { { "bucketName", bucketName }, { "objectName", objectName }, { "mappingName", mappingName } }; var attributes = new DataAcquirerAttributes(attributesDict); var guid = Guid.NewGuid(); var daInput = new DataAcquirerInputModel( guid, null, null, attributes, 0 ); var posts = _dataAcquirer.GetPostsAsync(daInput, CancellationToken.None); await foreach (var item in posts) { Console.WriteLine(item.Text); } }
private async Task RunJobAsync(DataAcquirerJobConfig jobConfig, CancellationToken cancellationToken) { try { var translate = jobConfig.Attributes.TryGetValue("Translate", out string value) //&& value.ToLower() == "true" ; // TODO validate job config if (!jobConfig.Attributes.ContainsKey("TopicQuery")) { _logger.TrackError( "StartNewJob", "TopicQuery attribute is not present. Job did not start", new { jobId = jobConfig.JobId }); return; } string queryLanguage = null; if (jobConfig.Attributes.TryGetValue("Language", out var desiredLanguage)) { queryLanguage = desiredLanguage; } await _dataAcquirerJobStorage.SaveAsync(jobConfig.JobId, jobConfig); var batchSize = 100; var dataAcquirerInputModel = DataAcquirerInputModel.FromValues( jobConfig.JobId, jobConfig.Attributes["TopicQuery"], queryLanguage, new DataAcquirerAttributes(jobConfig.Attributes), batchSize ); var batch = _acquirer.GetPostsAsync( dataAcquirerInputModel, cancellationToken); _logger.TrackInfo("MessageTracking", "Starting"); await ProcessBatch(jobConfig, dataAcquirerInputModel, batch, translate); } catch (TaskCanceledException) { } catch (Exception e) { _runningJobsRecords.Remove(jobConfig.JobId); _logger.TrackError( "RunJob", "Job encountered an error and stopped.", new { jobId = jobConfig.JobId, exception = e }); } }