Пример #1
0
        async Task <ChannelUpdatePlan> UpdateChannelDetail(ChannelUpdatePlan plan, ILogger log)
        {
            var channel    = plan.Channel;
            var channelLog = log.ForContext("Channel", channel.ChannelId).ForContext("ChannelId", channel.ChannelId);
            var full       = plan.Update == Full;
            var c          = channel.JsonClone();

            try {
                c.Updated = DateTime.Now;
                var d = await Api.ChannelData(c.ChannelId, full); // to save quota - full update only when missing features channels

                if (d != null)
                {
                    c.ChannelTitle       = d.Title;
                    c.Description        = d.Description;
                    c.LogoUrl            = d.Thumbnails?.Default__?.Url;
                    c.Subs               = d.Stats?.SubCount;
                    c.ChannelViews       = d.Stats?.ViewCount;
                    c.Country            = d.Country;
                    c.FeaturedChannelIds = d.FeaturedChannelIds ?? c.FeaturedChannelIds;
                    c.Keywords           = d.Keywords ?? c.Keywords;
                    c.Subscriptions      = d.Subscriptions ?? c.Subscriptions;
                    c.DefaultLanguage    = d.DefaultLanguage ?? c.DefaultLanguage;
                    c.Status             = ChannelStatus.Alive;
                    if (full)
                    {
                        c.LastFullUpdate = c.Updated;
                    }
                }
                else
                {
                    c.Status = ChannelStatus.Dead;
                }
                channelLog.Information("Collect - {Channel} - channel details ({Update})", c.ChannelTitle, plan.Update.EnumString());
            }
            catch (Exception ex) {
                channelLog.Error(ex, "Collect - {Channel} - Error when updating details for channel : {Error}", c.ChannelTitle, ex.Message);
            }
            return(new ChannelUpdatePlan(c, plan.Update, plan.VideosFrom));
        }
Пример #2
0
        async Task UpdateAllInChannel(ChannelUpdatePlan plan, CollectPart[] parts, IReadOnlyCollection <string> videosForChromeUpdate,
                                      IReadOnlyCollection <string> deadVideosForExtraUpdate,
                                      ILogger log)
        {
            var c = plan.Channel;

            void NotUpdatingLog(string reason) =>
            log.Information("Collect - {Channel} - Not updating videos/recs/captions because: {Reason} ",
                            c.ChannelTitle, reason);

            if (c.Status == ChannelStatus.Dead)
            {
                NotUpdatingLog("it's dead");
                return;
            }
            if (c.Subs < RCfg.MinChannelSubs && c.ChannelViews < RCfg.MinChannelViews)
            {
                NotUpdatingLog($"too small - subs ({c.Subs}), channel views ({c.ChannelViews})");
                return;
            }
            if (c.StatusMessage.HasValue())
            {
                NotUpdatingLog($"status msg ({c.StatusMessage})");
                return;
            }

            var discover        = plan.Update == Discover; // no need to check this against parts, that is done when planning the update
            var forChromeUpdate = new HashSet <string>();
            var vids            = new List <VideoItem>();
            var discoverVids    = new List <VideoItem>();

            if (parts.ShouldRun(VidStats) || discover && parts.ShouldRun(DiscoverPart))
            {
                var md = await Store.Videos.LatestFile(c.ChannelId);

                var lastUpload = md?.Ts?.ParseFileSafeTimestamp();
                log.Information("Collect - {Channel} - Starting channel update of videos/recs/captions. Last video stage {LastUpload}",
                                c.ChannelTitle, lastUpload);

                // get the oldest date for videos to store updated statistics for. This overlaps so that we have a history of video stats.
                var uploadedFrom = plan.VideosFrom ?? DateTime.UtcNow - (lastUpload == null ? RCfg.RefreshVideosWithinNew : RCfg.RefreshVideosWithinDaily);
                var vidsEnum     = ChannelVidItems(c, uploadedFrom, log);
                vids = discover ? await vidsEnum.Take(RCfg.DiscoverChannelVids).ToListAsync() : await vidsEnum.ToListAsync();

                if (parts.ShouldRun(VidStats))
                {
                    await SaveVids(c, vids, Store.Videos, lastUpload, log);
                }
                discoverVids    = discover ? vids.OrderBy(v => v.Statistics.ViewCount).Take(RCfg.DiscoverChannelVids).ToList() : null;
                forChromeUpdate = (discover
            ? discoverVids.Select(v => v.Id) // when discovering channels update all using chrome
            : videosForChromeUpdate ?? new string[] { }
                                   ).ToHashSet();
            }

            if (parts.ShouldRunAny(VidRecs, VidExtra))
            {
                var forWebUpdate = new List <string>();
                if (parts.ShouldRun(VidRecs) && !discover)
                {
                    forWebUpdate.AddRange((await VideoToUpdateRecs(c, vids)).Where(v => !forChromeUpdate.Contains(v)));
                }

                if (parts.ShouldRun(VidExtra) && deadVideosForExtraUpdate?.Any() == true)
                {
                    var ids = vids.Select(v => v.Id).ToHashSet();
                    deadVideosForExtraUpdate = deadVideosForExtraUpdate.Where(d => !ids.Contains(d)).ToList();
                    log.Debug("Collect -  {Channel} - updating video extra for {Videos} suspected dead videos", c.ChannelTitle, deadVideosForExtraUpdate.Count);
                    forWebUpdate.AddRange(deadVideosForExtraUpdate);
                }
                await SaveRecsAndExtra(c, parts, forChromeUpdate, forWebUpdate.ToArray(), log);
            }

            if (parts.ShouldRun(Caption))
            {
                var forCaptionSave = discover ? discoverVids : vids;
                await SaveNewCaptions(c, forCaptionSave, log);
            }
        }