async Task <(ChannelStored2 c, UpdateChannelType update)> UpdateChannelDetail(ChannelStored2 channel, UpdateChannelType update, ILogger log) { var channelLog = log.ForContext("Channel", channel.ChannelId).ForContext("ChannelId", channel.ChannelId); var full = update == Full; var c = channel.JsonClone(); try { c.Updated = DateTime.Now; var d = await Api.ChannelData(c.ChannelId, full); // to save quota - full update only when missing features channels if (d != null) { c.ChannelTitle = d.Title; c.Description = d.Description; c.LogoUrl = d.Thumbnails?.Default__?.Url; c.Subs = d.Stats?.SubCount; c.ChannelViews = d.Stats?.ViewCount; c.Country = d.Country; c.FeaturedChannelIds = d.FeaturedChannelIds ?? c.FeaturedChannelIds; c.Keywords = d.Keywords ?? c.Keywords; c.Subscriptions = d.Subscriptions ?? c.Subscriptions; c.DefaultLanguage = d.DefaultLanguage ?? c.DefaultLanguage; c.Status = ChannelStatus.Alive; if (full) { c.LastFullUpdate = c.Updated; } } else { c.Status = ChannelStatus.Dead; } channelLog.Information("Collect - {Channel} - channel details ({Update})", c.ChannelTitle, update.EnumString()); } catch (Exception ex) { channelLog.Error(ex, "Collect - {Channel} - Error when updating details for channel : {Error}", c.ChannelTitle, ex.Message); } return(c, update); }
async Task <IReadOnlyCollection <ChannelStored2> > UpdateChannels() { var store = Store.ChannelStore; Log.Information("Starting channels update. Limited to ({Included})", Cfg.LimitedToSeedChannels?.HasItems() == true ? Cfg.LimitedToSeedChannels.Join("|") : "All"); async Task <ChannelStored2> UpdateChannel(ChannelSheet channel) { var log = Log.ForContext("Channel", channel.Title).ForContext("ChannelId", channel.Id); var channelData = new ChannelData { Id = channel.Id, Title = channel.Title }; try { channelData = await Api.ChannelData(channel.Id) ?? // Use API to get channel instead of scraper. We get better info faster new ChannelData { Id = channel.Id, Title = channel.Title, Status = ChannelStatus.Dead }; log.Information("{Channel} - read channel details", channelData.Title); } catch (Exception ex) { channelData.Status = ChannelStatus.Dead; log.Error(ex, "{Channel} - Error when updating details for channel : {Error}", channel.Title, ex.Message); } var channelStored = new ChannelStored2 { ChannelId = channel.Id, ChannelTitle = channelData.Title ?? channel.Title, Status = channelData.Status, MainChannelId = channel.MainChannelId, Description = channelData.Description, LogoUrl = channelData.Thumbnails?.Default__?.Url, Subs = channelData.Stats?.SubCount, ChannelViews = channelData.Stats?.ViewCount, Country = channelData.Country, Updated = DateTime.UtcNow, Relevance = channel.Relevance, LR = channel.LR, HardTags = channel.HardTags, SoftTags = channel.SoftTags, UserChannels = channel.UserChannels }; return(channelStored); } var seeds = await ChannelSheets.Channels(Cfg.Sheets, Log); var channels = await seeds.Where(c => Cfg.LimitedToSeedChannels.IsEmpty() || Cfg.LimitedToSeedChannels.Contains(c.Id)) .BlockTransform(UpdateChannel, Cfg.DefaultParallel, progressUpdate: p => Log.Debug("Reading channels {ChannelCount}/{ChannelTotal}", p.CompletedTotal, seeds.Count)); if (channels.Any()) { await store.Append(channels); } return(channels); }