private static PropertyDict GetReportOnVideoTable() { using (var context = new DataLakeYouTubeDataContext()) { var(schema, table, keys) = ContextIntrospection.GetDatabaseInfo(context, typeof(Video)); return(ContinuityReportCommand.ReportOnTable(YearDatabase.DataLakeDatabase, schema, table, keys)); } }
public static List <MetricDelta <Video, long> > CompareVideoLifetimeDailyTotal() { var now = DateTime.UtcNow; using (var analytics = new DataLakeYouTubeAnalyticsContext()) using (var db = new DataLakeYouTubeDataContext()) { var dayTotal = analytics.VideoDailyMetrics.Where(x => x.ValidityStart <= now && now < x.ValidityEnd) .GroupBy(x => x.VideoId, y => y, (k, v) => new { VideoId = k, Views = v.Sum(y => y.Views), StartDate = v.Min(y => y.Date), EndDate = v.Max(y => y.Date) }).ToList(); var stats = from s in db.Statistics.Where(x => x.CaptureDate == now.Date && x.ValidityStart <= now && now < x.ValidityEnd) join dt in dayTotal on s.VideoId equals dt.VideoId join v in db.Videos.Where(x => x.ValidityStart <= now && now < x.ValidityEnd) on s.VideoId equals v.VideoId select new MetricDelta <Video, long>() { Id = v, Lifetime = s.ViewCount, Total = dt.Views, DailyStart = dt.StartDate, DailyEnd = dt.EndDate, LifetimeDate = s.CaptureDate }; return(stats.ToList()); } }
public static void Write(IEnumerable <Statistics> videos, Logger logger) { using (var dlContext = new DataLakeYouTubeDataContext()) { var now = DateTime.UtcNow; foreach (var newObj in videos) { var storedObj = dlContext.Statistics.SingleOrDefault(v => v.VideoId == newObj.VideoId && v.CaptureDate == newObj.CaptureDate && v.ValidityStart <= now && now < v.ValidityEnd); newObj.ValidityEnd = DateTime.MaxValue; newObj.ValidityStart = DateTime.UtcNow; var modified = compareOldAndNew(storedObj, newObj); switch (modified) { case Modified.New: logger.Debug("Found new statistics: {VideoId} {CaptureDate}", newObj.VideoId, newObj.CaptureDate); dlContext.Add(newObj); break; case Modified.Updated: logger.Debug("Found update to: {VideoId} {CaptureDate}", newObj.VideoId, newObj.CaptureDate); storedObj.ValidityEnd = newObj.ValidityStart; dlContext.Add(newObj); break; default: break; } } dlContext.SaveChanges(); } }
public static List <Video> GetVideos() { var now = DateTime.UtcNow; using (var db = new DataLakeYouTubeDataContext()) { return(db.Videos.Where(x => x.ValidityStart <= now && now < x.ValidityEnd).OrderByDescending(x => x.PublishedAt).ToList()); } }
public void TestDeleteFromTable() { var videos = new List <(string VideoTitle, List <(DateTime ValidityStart, DateTime ValidityEnd)>)>() { ("a-yt-video", new List <(DateTime ValidityStart, DateTime ValidityEnd)>() { (new DateTime(2018, 1, 1), new DateTime(2018, 1, 3)), (new DateTime(2018, 1, 2), DateTime.MaxValue) }) }; YDS.ThereAreVideosWithManyVersions(videos); using (var context = new DataLakeYouTubeDataContext()) { Assert.Equal(2, context.Videos.Count()); var(schema, table, keys) = ContextIntrospection.GetDatabaseInfo(context, typeof(Video)); TableOperations.DeleteFromTable(YearDatabase.DataLakeDatabase, schema, table); Assert.Empty(context.Videos); } }
public void YouTubeVideosInDataLakeAreTransformedIntoSourceVideos() { var channelId = "fee"; var titleAtypo = "The True Mening of Patriotism"; var titleA = "The True Meaning of Patriotism"; var titleB = "The Population Boom"; var someLakeVideos = new[] { new Video() { VideoId = "xyz", Title = titleAtypo, Duration = "PT5M11S" }, new Video() { VideoId = "zxy", Title = titleB, Duration = "PT2M6S" }, }; YDS.SomeVideosWereFound(someLakeVideos, channelId); APS.YouTubeVideoSyncJobHasRun(); using (var context = new DataLakeYouTubeDataContext()) { var videos = context.Videos.OrderBy(x => x.Title).ToList(); var titles = new[] { titleB, titleAtypo }; Assert.Equal(titles.Count(), videos.Count()); for (var i = 0; i < titles.Count(); i++) { Assert.Equal(titles[i], videos[i].Title); } } using (var context = new ApplicationDbContext()) { var videos = context.SourceVideos.OrderBy(x => x.Title).ToList(); var titles = new[] { titleB, titleAtypo }; Assert.Equal(titles.Count(), videos.Count()); for (var i = 0; i < titles.Count(); i++) { Assert.Equal(titles[i], videos[i].Title); } } someLakeVideos = new[] { new Video() { VideoId = "xyz", Title = titleA, Duration = "PT5M11S" }, new Video() { VideoId = "zxy", Title = titleB, Duration = "PT2M6S" }, }; YDS.SomeVideosWereFound(someLakeVideos, channelId); APS.YouTubeVideoSyncJobHasRun(); using (var context = new DataLakeYouTubeDataContext()) { var videos = context.Videos.OrderBy(x => x.Title).ToList(); var titles = new[] { titleB, titleA, titleAtypo }; Assert.Equal(titles.Count(), videos.Count()); for (var i = 0; i < titles.Count(); i++) { Assert.Equal(titles[i], videos[i].Title); } } using (var context = new ApplicationDbContext()) { var videos = context.SourceVideos.OrderBy(x => x.Title).ToList(); var titles = new[] { titleB, titleA }; Assert.Equal(titles.Count(), videos.Count()); for (var i = 0; i < titles.Count(); i++) { Assert.Equal(titles[i], videos[i].Title); } } }