private async Task SendRecordToOutputs(string[] outputChannels,
MessageBrokerMessage messageBrokerMessage)
{
_logger.TrackStatistics(
"SendingData",
new { channels = outputChannels });
foreach (var outputChannel in outputChannels)
{
await _producer.ProduceAsync(outputChannel,
messageBrokerMessage);
}
}
public async IAsyncEnumerable <DataAcquirerPost> CreateBatchPostEnumerator(
ITwitterContext twitterContext,
TwitterMetadata defaultMetadata)
{
var includeRetweets = false;
var metadata = await _metadataContext.GetOrCreateAsync(defaultMetadata);
_logger.TrackInfo("QueryData", "Twitter - using metadata",
new
{
metadata,
jobId = _jobId
});
var query = defaultMetadata.Query;
while (true)
{
var twitterInputModel = new TwitterQueryInput(
query,
metadata.Language,
metadata.MaxId,
metadata.SinceId,
metadata.BatchSize);
var batches = QueryPastPostAsync(
twitterContext,
twitterInputModel);
await foreach (var batch in batches)
{
if (!batch.Any())
{
break;
}
foreach (var post in batch)
{
metadata.MaxId = Math.Min(post.StatusID - 1, metadata.MaxId);
metadata.SinceId = Math.Max(post.StatusID, metadata.SinceId);
if (post.CreatedAt > _mostRecentPost)
{
_mostRecentPost = post.CreatedAt;
}
var isRetweet = post.RetweetedStatus?.StatusID > 0;
if (isRetweet && !includeRetweets)
{
continue;
}
yield return(FromStatus(post, query));
}
_foundPosts += batch.Count;
_logger.TrackStatistics("QueryDataStatistics",
new
{
_foundPosts,
_mostRecentPost,
query = metadata.Query,
language = metadata.Language,
jobId = _jobId
});
await _metadataContext.UpdateAsync(metadata);
}
metadata.MaxId = ulong.MaxValue;
await _metadataContext.UpdateAsync(metadata);
_logger.TrackInfo("QueryData", $"Waiting {_options.NoPostWaitDelay} for new posts");
await Task.Delay(_options.NoPostWaitDelay);
}
}