private async Task CollectAnswers()
{
var keys = _settingsProvider.ApiKeys;
var par = _settingsProvider.Parallelism;
var allParents = await _repository.GetCommentsWithAnswersAsync();
var parentIdParts = allParents.Partition(par);
var answersCounter = new SyncCounter();
var updatesCounter = new SyncCounter();
var parentsCountDown = new SyncCounter(allParents.Count);
var tasks = parentIdParts.Select(parents => GetAnswersFromComments(parents, keys.Next(), answersCounter, updatesCounter, parentsCountDown)).ToList();
while (!_ct.IsCancellationRequested)
{
var runningTasks = tasks.Count(t => !t.IsCompleted);
_logger.LogInformation($"answers received: {answersCounter.Read()}, parents left: {parentsCountDown.Read()}, db updates: {updatesCounter.Read()}, running tasks: {runningTasks}");
if (runningTasks <= 0)
{
break;
}
await tasks.WaitOneOrTimeout(4000, _ct);
}
foreach (var task in tasks)
{
await task;
}
_logger.LogInformation($"comments db updates: {updatesCounter.Read()}");
}
private async Task CollectComments()
{
var keys = _settingsProvider.ApiKeys;
var par = _settingsProvider.Parallelism;
var videoIdsWithComments = _repository.GetAllVideoIdsWithComments();
var videoIdParts = videoIdsWithComments.Partition(par);
var commentsCounter = new SyncCounter();
var updatesCounter = new SyncCounter();
var videoCountDown = new SyncCounter(videoIdsWithComments.Count);
var tasks = videoIdParts.Select(videoIds => GetCommentsFromVideoIds(videoIds, keys.Next(), commentsCounter, updatesCounter, videoCountDown)).ToList();
while (!_ct.IsCancellationRequested)
{
var runningTasks = tasks.Count(t => !t.IsCompleted);
_logger.LogInformation($"comments received: {commentsCounter.Read()}, videos left: {videoCountDown.Read()}, db updates: {updatesCounter.Read()}, running tasks: {runningTasks}");
if (runningTasks == 0)
{
break;
}
await tasks.WaitOneOrTimeout(4000, ct : _ct);
}
foreach (var task in tasks)
{
await task;
}
_logger.LogInformation($"comments db updates: {updatesCounter.Read()}");
}
private async Task CollectVideoDetails()
{
var keys = _settingsProvider.ApiKeys;
var channelIds = _settingsProvider.ChannelIds ?? new List <string>();
var listIds = _settingsProvider.ListIds ?? new List <string>();
var fromChannelsTask = GetVideoIdsFromChannels(channelIds, keys.Next());
var fromListTask = GetVideoIdsFromList(listIds, keys.Next());
var fromDbTask = _repository.GetAllVideoIdsAsync();
var videoIds = new HashSet <string>();
videoIds.AddRange(await fromChannelsTask);
videoIds.AddRange(await fromListTask);
videoIds.AddRange(await fromDbTask);
var videoIdsPartitions = videoIds.Partition(_settingsProvider.Parallelism);
var videoCountDown = new SyncCounter(videoIds.Count);
var videoTasks = videoIdsPartitions.Select(part => GetVideoFromId(part, keys.Next(), videoCountDown, _ct)).ToList();
while (!_ct.IsCancellationRequested)
{
var runningTasks = videoTasks.Count(t => !t.IsCompleted);
_logger.LogInformation($"videos left: {videoCountDown.Read()}, running tasks: {runningTasks}");
if (runningTasks <= 0)
{
break;
}
await videoTasks.WaitOneOrTimeout(4000, ct : _ct);
}
var videos = new List <Video>();
foreach (var videoTask in videoTasks)
{
videos.AddRange(await videoTask);
}
var updates = await _repository.SaveOrUpdate(videos);
_logger.LogInformation($"video db updates: {updates}");
}
private async Task GetAnswersFromComments(IEnumerable <CommentBase> parents, string apiKey, SyncCounter answersCounter, SyncCounter updatesCounter, SyncCounter parentsCountDown)
{
using (var api = new YoutubeApi(apiKey)) {
foreach (var parent in parents)
{
var ytComments = await api.GetAllAnswersFromComment(parent.Id, _ct).TryHarder(_logger, ct: _ct);
parentsCountDown.Decrement();
var dbAnswers = ytComments.Items.Select(i => i.MapToDbEntity(CommentType.Answer, parent.VideoId)).ToList();
answersCounter.Add(dbAnswers.Count);
var updates = await _repository.SaveOrUpdate(dbAnswers);
updatesCounter.Add(updates);
}
}
}
private async Task GetCommentsFromVideoIds(IEnumerable <string> videoIds, string apiKey, SyncCounter commentsCnt, SyncCounter updatesCnt, SyncCounter vidCntDown)
{
using (var api = new YoutubeApi(apiKey)) {
foreach (var videoId in videoIds)
{
CommentThreadListResponse ytComments = null;
try {
ytComments = await api.GetAllCommentsAndAnswersFromVideo(videoId, _ct);
}
catch (Google.GoogleApiException e) {
_logger.LogError($"error getting comments and answers for videoId={videoId} ({e.Message})");
}
if (ytComments == null)
{
try {
ytComments = await api.GetAllCommentsFromVideo(videoId, _ct);
}
catch (Google.GoogleApiException e) {
_logger.LogError($"error getting comments for videoId={videoId} ({e.Message})");
}
}
vidCntDown.Decrement();
if (ytComments == null)
{
continue;
}
var dbComments = new List <Comment>();
foreach (var thread in ytComments.Items)
{
var hasAnswers = (thread.Snippet.TotalReplyCount ?? 0) > 0;
dbComments.Add(thread.Snippet.TopLevelComment.MapToDbEntity(CommentType.Comment, videoId, hasAnswers));
if (hasAnswers && thread.Replies?.Comments != null)
{
dbComments.AddRange(thread.Replies.Comments.Select(a => a.MapToDbEntity(CommentType.Answer, videoId)));
}
}
commentsCnt.Add(dbComments.Count);
var updates = await _repository.SaveOrUpdate(dbComments);
updatesCnt.Add(updates);
}
}
}
private async Task <List <Video> > GetVideoFromId(IEnumerable <string> videoIds, string apiKey, SyncCounter videoCountDown, CancellationToken ct)
{
var videos = new List <Video>();
using (var api = new YoutubeApi(apiKey)) {
foreach (var videoId in videoIds)
{
var ytVid = await api.GetVideoDetails(videoId, ct);
var dbVid = ytVid.Items.Select(v => v.MapToDbEntity());
videoCountDown.Decrement();
videos.AddRange(dbVid);
}
}
return(videos);
}
