async Task <ChannelUpdatePlan> UpdateChannelDetail(ChannelUpdatePlan plan, ILogger log) { var channel = plan.Channel; var channelLog = log.ForContext("Channel", channel.ChannelId).ForContext("ChannelId", channel.ChannelId); var full = plan.Update == Full; var c = channel.JsonClone(); try { c.Updated = DateTime.Now; var d = await Api.ChannelData(c.ChannelId, full); // to save quota - full update only when missing features channels if (d != null) { c.ChannelTitle = d.Title; c.Description = d.Description; c.LogoUrl = d.Thumbnails?.Default__?.Url; c.Subs = d.Stats?.SubCount; c.ChannelViews = d.Stats?.ViewCount; c.Country = d.Country; c.FeaturedChannelIds = d.FeaturedChannelIds ?? c.FeaturedChannelIds; c.Keywords = d.Keywords ?? c.Keywords; c.Subscriptions = d.Subscriptions ?? c.Subscriptions; c.DefaultLanguage = d.DefaultLanguage ?? c.DefaultLanguage; c.Status = ChannelStatus.Alive; if (full) { c.LastFullUpdate = c.Updated; } } else { c.Status = ChannelStatus.Dead; } channelLog.Information("Collect - {Channel} - channel details ({Update})", c.ChannelTitle, plan.Update.EnumString()); } catch (Exception ex) { channelLog.Error(ex, "Collect - {Channel} - Error when updating details for channel : {Error}", c.ChannelTitle, ex.Message); } return(new ChannelUpdatePlan(c, plan.Update, plan.VideosFrom)); }
async Task UpdateAllInChannel(ChannelUpdatePlan plan, CollectPart[] parts, IReadOnlyCollection <string> videosForChromeUpdate, IReadOnlyCollection <string> deadVideosForExtraUpdate, ILogger log) { var c = plan.Channel; void NotUpdatingLog(string reason) => log.Information("Collect - {Channel} - Not updating videos/recs/captions because: {Reason} ", c.ChannelTitle, reason); if (c.Status == ChannelStatus.Dead) { NotUpdatingLog("it's dead"); return; } if (c.Subs < RCfg.MinChannelSubs && c.ChannelViews < RCfg.MinChannelViews) { NotUpdatingLog($"too small - subs ({c.Subs}), channel views ({c.ChannelViews})"); return; } if (c.StatusMessage.HasValue()) { NotUpdatingLog($"status msg ({c.StatusMessage})"); return; } var discover = plan.Update == Discover; // no need to check this against parts, that is done when planning the update var forChromeUpdate = new HashSet <string>(); var vids = new List <VideoItem>(); var discoverVids = new List <VideoItem>(); if (parts.ShouldRun(VidStats) || discover && parts.ShouldRun(DiscoverPart)) { var md = await Store.Videos.LatestFile(c.ChannelId); var lastUpload = md?.Ts?.ParseFileSafeTimestamp(); log.Information("Collect - {Channel} - Starting channel update of videos/recs/captions. Last video stage {LastUpload}", c.ChannelTitle, lastUpload); // get the oldest date for videos to store updated statistics for. This overlaps so that we have a history of video stats. var uploadedFrom = plan.VideosFrom ?? DateTime.UtcNow - (lastUpload == null ? RCfg.RefreshVideosWithinNew : RCfg.RefreshVideosWithinDaily); var vidsEnum = ChannelVidItems(c, uploadedFrom, log); vids = discover ? await vidsEnum.Take(RCfg.DiscoverChannelVids).ToListAsync() : await vidsEnum.ToListAsync(); if (parts.ShouldRun(VidStats)) { await SaveVids(c, vids, Store.Videos, lastUpload, log); } discoverVids = discover ? vids.OrderBy(v => v.Statistics.ViewCount).Take(RCfg.DiscoverChannelVids).ToList() : null; forChromeUpdate = (discover ? discoverVids.Select(v => v.Id) // when discovering channels update all using chrome : videosForChromeUpdate ?? new string[] { } ).ToHashSet(); } if (parts.ShouldRunAny(VidRecs, VidExtra)) { var forWebUpdate = new List <string>(); if (parts.ShouldRun(VidRecs) && !discover) { forWebUpdate.AddRange((await VideoToUpdateRecs(c, vids)).Where(v => !forChromeUpdate.Contains(v))); } if (parts.ShouldRun(VidExtra) && deadVideosForExtraUpdate?.Any() == true) { var ids = vids.Select(v => v.Id).ToHashSet(); deadVideosForExtraUpdate = deadVideosForExtraUpdate.Where(d => !ids.Contains(d)).ToList(); log.Debug("Collect - {Channel} - updating video extra for {Videos} suspected dead videos", c.ChannelTitle, deadVideosForExtraUpdate.Count); forWebUpdate.AddRange(deadVideosForExtraUpdate); } await SaveRecsAndExtra(c, parts, forChromeUpdate, forWebUpdate.ToArray(), log); } if (parts.ShouldRun(Caption)) { var forCaptionSave = discover ? discoverVids : vids; await SaveNewCaptions(c, forCaptionSave, log); } }