static async Task SaveVids(ChannelStored2 c, IReadOnlyCollection <VideoItem> vids, AppendCollectionStore <VideoStored2> vidStore, DateTime?uploadedFrom, ILogger log) { var updated = DateTime.UtcNow; var vidsStored = vids.Select(v => new VideoStored2 { VideoId = v.Id, Title = v.Title, Description = v.Description, Duration = v.Duration, Keywords = v.Keywords.ToList(), Statistics = v.Statistics, Thumbnails = v.Thumbnails, ChannelId = c.ChannelId, ChannelTitle = c.ChannelTitle, UploadDate = v.UploadDate.UtcDateTime, Updated = updated }).ToList(); if (vidsStored.Count > 0) { await vidStore.Append(vidsStored); } var newVideos = vidsStored.Count(v => uploadedFrom == null || v.UploadDate > uploadedFrom); log.Information("{Channel} - Recorded {VideoCount} videos. {NewCount} new, {UpdatedCount} updated", c.ChannelTitle, vids.Count, newVideos, vids.Count - newVideos); }
async Task <IReadOnlyCollection <(string Id, string Title)> > VideoToUpdateRecs(IEnumerable <VideoItem> vids, AppendCollectionStore <RecStored2> recStore) { var prevUpdateMeta = await recStore.LatestFileMetadata(); var prevUpdate = prevUpdateMeta?.Ts.ParseFileSafeTimestamp(); var vidsDesc = vids.OrderByDescending(v => v.UploadDate).ToList(); var toUpdate = prevUpdate == null ? vidsDesc : // all videos if this is the first time for this channel // new vids since the last rec update, or the vid was created in the last RefreshRecsWithin (e.g. 30d) vidsDesc.Where(v => v.UploadDate > prevUpdate || v.UploadDate.UtcDateTime.IsYoungerThan(RCfg.RefreshRecsWithin)).ToList(); var deficit = RCfg.RefreshRecsMin - toUpdate.Count; if (deficit > 0) { toUpdate.AddRange(vidsDesc.Where(v => toUpdate.All(u => u.Id != v.Id)) .Take(deficit)); // if we don't have new videos, refresh the min amount by adding videos } return(toUpdate.Select(v => (v.Id, v.Title)).ToList()); }
public YtStore(ISimpleFileStore store, ILogger log) { Store = store; Log = log; ChannelStore = new AppendCollectionStore <ChannelStored2>(Store, "channels", c => c.Updated.FileSafeTimestamp(), StoreVersion.ToString(), Log); }