async Task <(ChannelStored2 c, UpdateChannelType update)> UpdateChannelDetail(ChannelStored2 channel, UpdateChannelType update, ILogger log)
{
var channelLog = log.ForContext("Channel", channel.ChannelId).ForContext("ChannelId", channel.ChannelId);
var full = update == Full;
var c = channel.JsonClone();
try {
c.Updated = DateTime.Now;
var d = await Api.ChannelData(c.ChannelId, full); // to save quota - full update only when missing features channels
if (d != null)
{
c.ChannelTitle = d.Title;
c.Description = d.Description;
c.LogoUrl = d.Thumbnails?.Default__?.Url;
c.Subs = d.Stats?.SubCount;
c.ChannelViews = d.Stats?.ViewCount;
c.Country = d.Country;
c.FeaturedChannelIds = d.FeaturedChannelIds ?? c.FeaturedChannelIds;
c.Keywords = d.Keywords ?? c.Keywords;
c.Subscriptions = d.Subscriptions ?? c.Subscriptions;
c.DefaultLanguage = d.DefaultLanguage ?? c.DefaultLanguage;
c.Status = ChannelStatus.Alive;
if (full)
{
c.LastFullUpdate = c.Updated;
}
}
else
{
c.Status = ChannelStatus.Dead;
}
channelLog.Information("Collect - {Channel} - channel details ({Update})", c.ChannelTitle, update.EnumString());
}
catch (Exception ex) {
channelLog.Error(ex, "Collect - {Channel} - Error when updating details for channel : {Error}", c.ChannelTitle, ex.Message);
}
return(c, update);
}
async Task <IReadOnlyCollection <ChannelStored2> > UpdateChannels()
{
var store = Store.ChannelStore;
Log.Information("Starting channels update. Limited to ({Included})",
Cfg.LimitedToSeedChannels?.HasItems() == true ? Cfg.LimitedToSeedChannels.Join("|") : "All");
async Task <ChannelStored2> UpdateChannel(ChannelSheet channel)
{
var log = Log.ForContext("Channel", channel.Title).ForContext("ChannelId", channel.Id);
var channelData = new ChannelData {
Id = channel.Id, Title = channel.Title
};
try {
channelData = await Api.ChannelData(channel.Id) ?? // Use API to get channel instead of scraper. We get better info faster
new ChannelData
{
Id = channel.Id, Title = channel.Title, Status = ChannelStatus.Dead
};
log.Information("{Channel} - read channel details", channelData.Title);
}
catch (Exception ex) {
channelData.Status = ChannelStatus.Dead;
log.Error(ex, "{Channel} - Error when updating details for channel : {Error}", channel.Title, ex.Message);
}
var channelStored = new ChannelStored2 {
ChannelId = channel.Id,
ChannelTitle = channelData.Title ?? channel.Title,
Status = channelData.Status,
MainChannelId = channel.MainChannelId,
Description = channelData.Description,
LogoUrl = channelData.Thumbnails?.Default__?.Url,
Subs = channelData.Stats?.SubCount,
ChannelViews = channelData.Stats?.ViewCount,
Country = channelData.Country,
Updated = DateTime.UtcNow,
Relevance = channel.Relevance,
LR = channel.LR,
HardTags = channel.HardTags,
SoftTags = channel.SoftTags,
UserChannels = channel.UserChannels
};
return(channelStored);
}
var seeds = await ChannelSheets.Channels(Cfg.Sheets, Log);
var channels = await seeds.Where(c => Cfg.LimitedToSeedChannels.IsEmpty() || Cfg.LimitedToSeedChannels.Contains(c.Id))
.BlockTransform(UpdateChannel, Cfg.DefaultParallel,
progressUpdate: p => Log.Debug("Reading channels {ChannelCount}/{ChannelTotal}", p.CompletedTotal, seeds.Count));
if (channels.Any())
{
await store.Append(channels);
}
return(channels);
}