async Task <ChannelUpdatePlan> UpdateChannelDetail(ChannelUpdatePlan plan, ILogger log)
{
var channel = plan.Channel;
var channelLog = log.ForContext("Channel", channel.ChannelId).ForContext("ChannelId", channel.ChannelId);
var full = plan.Update == Full;
var c = channel.JsonClone();
try {
c.Updated = DateTime.Now;
var d = await Api.ChannelData(c.ChannelId, full); // to save quota - full update only when missing features channels
if (d != null)
{
c.ChannelTitle = d.Title;
c.Description = d.Description;
c.LogoUrl = d.Thumbnails?.Default__?.Url;
c.Subs = d.Stats?.SubCount;
c.ChannelViews = d.Stats?.ViewCount;
c.Country = d.Country;
c.FeaturedChannelIds = d.FeaturedChannelIds ?? c.FeaturedChannelIds;
c.Keywords = d.Keywords ?? c.Keywords;
c.Subscriptions = d.Subscriptions ?? c.Subscriptions;
c.DefaultLanguage = d.DefaultLanguage ?? c.DefaultLanguage;
c.Status = ChannelStatus.Alive;
if (full)
{
c.LastFullUpdate = c.Updated;
}
}
else
{
c.Status = ChannelStatus.Dead;
}
channelLog.Information("Collect - {Channel} - channel details ({Update})", c.ChannelTitle, plan.Update.EnumString());
}
catch (Exception ex) {
channelLog.Error(ex, "Collect - {Channel} - Error when updating details for channel : {Error}", c.ChannelTitle, ex.Message);
}
return(new ChannelUpdatePlan(c, plan.Update, plan.VideosFrom));
}
async Task UpdateAllInChannel(ChannelUpdatePlan plan, CollectPart[] parts, IReadOnlyCollection <string> videosForChromeUpdate,
IReadOnlyCollection <string> deadVideosForExtraUpdate,
ILogger log)
{
var c = plan.Channel;
void NotUpdatingLog(string reason) =>
log.Information("Collect - {Channel} - Not updating videos/recs/captions because: {Reason} ",
c.ChannelTitle, reason);
if (c.Status == ChannelStatus.Dead)
{
NotUpdatingLog("it's dead");
return;
}
if (c.Subs < RCfg.MinChannelSubs && c.ChannelViews < RCfg.MinChannelViews)
{
NotUpdatingLog($"too small - subs ({c.Subs}), channel views ({c.ChannelViews})");
return;
}
if (c.StatusMessage.HasValue())
{
NotUpdatingLog($"status msg ({c.StatusMessage})");
return;
}
var discover = plan.Update == Discover; // no need to check this against parts, that is done when planning the update
var forChromeUpdate = new HashSet <string>();
var vids = new List <VideoItem>();
var discoverVids = new List <VideoItem>();
if (parts.ShouldRun(VidStats) || discover && parts.ShouldRun(DiscoverPart))
{
var md = await Store.Videos.LatestFile(c.ChannelId);
var lastUpload = md?.Ts?.ParseFileSafeTimestamp();
log.Information("Collect - {Channel} - Starting channel update of videos/recs/captions. Last video stage {LastUpload}",
c.ChannelTitle, lastUpload);
// get the oldest date for videos to store updated statistics for. This overlaps so that we have a history of video stats.
var uploadedFrom = plan.VideosFrom ?? DateTime.UtcNow - (lastUpload == null ? RCfg.RefreshVideosWithinNew : RCfg.RefreshVideosWithinDaily);
var vidsEnum = ChannelVidItems(c, uploadedFrom, log);
vids = discover ? await vidsEnum.Take(RCfg.DiscoverChannelVids).ToListAsync() : await vidsEnum.ToListAsync();
if (parts.ShouldRun(VidStats))
{
await SaveVids(c, vids, Store.Videos, lastUpload, log);
}
discoverVids = discover ? vids.OrderBy(v => v.Statistics.ViewCount).Take(RCfg.DiscoverChannelVids).ToList() : null;
forChromeUpdate = (discover
? discoverVids.Select(v => v.Id) // when discovering channels update all using chrome
: videosForChromeUpdate ?? new string[] { }
).ToHashSet();
}
if (parts.ShouldRunAny(VidRecs, VidExtra))
{
var forWebUpdate = new List <string>();
if (parts.ShouldRun(VidRecs) && !discover)
{
forWebUpdate.AddRange((await VideoToUpdateRecs(c, vids)).Where(v => !forChromeUpdate.Contains(v)));
}
if (parts.ShouldRun(VidExtra) && deadVideosForExtraUpdate?.Any() == true)
{
var ids = vids.Select(v => v.Id).ToHashSet();
deadVideosForExtraUpdate = deadVideosForExtraUpdate.Where(d => !ids.Contains(d)).ToList();
log.Debug("Collect - {Channel} - updating video extra for {Videos} suspected dead videos", c.ChannelTitle, deadVideosForExtraUpdate.Count);
forWebUpdate.AddRange(deadVideosForExtraUpdate);
}
await SaveRecsAndExtra(c, parts, forChromeUpdate, forWebUpdate.ToArray(), log);
}
if (parts.ShouldRun(Caption))
{
var forCaptionSave = discover ? discoverVids : vids;
await SaveNewCaptions(c, forCaptionSave, log);
}
}