public async Task <YouTubeRssItem[]> GetUploadedMoviesFromRss(string[] youtubeChannelIds, DateTime fromUtc, DateTime toUtc)
{
log.Infomation($"GetUploadedMoviesFromRss(youtubeChannelIds, fromUtc:{fromUtc}, toUtc:{toUtc})");
var list = new List <YouTubeRssItem>();
var count = 0;
foreach (var youtubeChannelId in youtubeChannelIds.Take(300))
{
if (count++ % 10 == 0)
{
log.Infomation($"** count={count} GetRssItems");
}
var tmp = GetRssItems(youtubeChannelId);
var targetMovieIds = tmp
.Where(x => x.PublishDateUtc >= fromUtc)
.Where(x => x.PublishDateUtc <= toUtc);
list.AddRange(targetMovieIds);
await Task.Delay(TimeSpan.FromSeconds(1));
}
return(list.ToArray());
}
public async Task <YouTubeChannelRankingItem[]> Run()
{
var retryCount = 0;
var maxRetryCount = 3;
do
{
try
{
retryCount++;
var html = await GetPageSource();
log.Infomation("** Begin Parse");
var rankingItems = Parse(html).ToArray();
log.Infomation($"** rankingItems.Length={rankingItems.Length}");
if (rankingItems.Length > 0)
{
return(rankingItems);
}
}
catch (Exception e)
{
log.Error($"** e.Message={e.Message}");
log.Error($"** e.StackTrace={e.StackTrace}");
if (retryCount >= maxRetryCount)
{
throw;
}
}
log.Error($"** failed GetRankingItems. retryCount={retryCount}");
await Task.Delay(TimeSpan.FromSeconds(5));
}while (retryCount < maxRetryCount);
throw new Exception("not found element");
}
public async Task GetNewMovies()
{
log.Infomation("GetNewMovies");
log.Infomation("vtuberInsightCrawler.Run()");
var rankingItems = await vtuberInsightCrawler.Run();
log.Infomation($"rankingItems.Length={rankingItems.Length}");
//foreach (var rankingItem in rankingItems)
//{
// log.Infomation($"{rankingItem.Rank:D3} {rankingItem.ChannelName} {rankingItem.ChannelId}");
//}
await youtubeBlobService.UploadVTuberInsightCsvFile(rankingItems);
var fromUtc = DateTime.UtcNow.AddHours(9).Date.AddDays(-1).AddHours(-9);
var toUtc = fromUtc.AddDays(1);
var latestYouYubeRssItems = await GetMovieIdsFromRss(fromUtc, toUtc);
await youtubeBlobService.UploadLatestYouTubeMovies(fromUtc, toUtc, latestYouYubeRssItems);
}