private static PropertyDict GetReportOnVideoTable()
{
using (var context = new DataLakeYouTubeDataContext()) {
var(schema, table, keys) = ContextIntrospection.GetDatabaseInfo(context, typeof(Video));
return(ContinuityReportCommand.ReportOnTable(YearDatabase.DataLakeDatabase, schema, table, keys));
}
}
public static List <MetricDelta <Video, long> > CompareVideoLifetimeDailyTotal()
{
var now = DateTime.UtcNow;
using (var analytics = new DataLakeYouTubeAnalyticsContext())
using (var db = new DataLakeYouTubeDataContext()) {
var dayTotal =
analytics.VideoDailyMetrics.Where(x => x.ValidityStart <= now && now < x.ValidityEnd)
.GroupBy(x => x.VideoId, y => y, (k, v) => new { VideoId = k, Views = v.Sum(y => y.Views), StartDate = v.Min(y => y.Date), EndDate = v.Max(y => y.Date) }).ToList();
var stats =
from s in db.Statistics.Where(x => x.CaptureDate == now.Date && x.ValidityStart <= now && now < x.ValidityEnd)
join dt in dayTotal on s.VideoId equals dt.VideoId
join v in db.Videos.Where(x => x.ValidityStart <= now && now < x.ValidityEnd) on s.VideoId equals v.VideoId
select new MetricDelta <Video, long>()
{
Id = v,
Lifetime = s.ViewCount,
Total = dt.Views,
DailyStart = dt.StartDate,
DailyEnd = dt.EndDate,
LifetimeDate = s.CaptureDate
};
return(stats.ToList());
}
}
public static void Write(IEnumerable <Statistics> videos, Logger logger)
{
using (var dlContext = new DataLakeYouTubeDataContext()) {
var now = DateTime.UtcNow;
foreach (var newObj in videos)
{
var storedObj = dlContext.Statistics.SingleOrDefault(v => v.VideoId == newObj.VideoId && v.CaptureDate == newObj.CaptureDate && v.ValidityStart <= now && now < v.ValidityEnd);
newObj.ValidityEnd = DateTime.MaxValue;
newObj.ValidityStart = DateTime.UtcNow;
var modified = compareOldAndNew(storedObj, newObj);
switch (modified)
{
case Modified.New:
logger.Debug("Found new statistics: {VideoId} {CaptureDate}", newObj.VideoId, newObj.CaptureDate);
dlContext.Add(newObj);
break;
case Modified.Updated:
logger.Debug("Found update to: {VideoId} {CaptureDate}", newObj.VideoId, newObj.CaptureDate);
storedObj.ValidityEnd = newObj.ValidityStart;
dlContext.Add(newObj);
break;
default:
break;
}
}
dlContext.SaveChanges();
}
}
public static List <Video> GetVideos()
{
var now = DateTime.UtcNow;
using (var db = new DataLakeYouTubeDataContext()) {
return(db.Videos.Where(x => x.ValidityStart <= now && now < x.ValidityEnd).OrderByDescending(x => x.PublishedAt).ToList());
}
}
public void TestDeleteFromTable()
{
var videos = new List <(string VideoTitle, List <(DateTime ValidityStart, DateTime ValidityEnd)>)>()
{
("a-yt-video",
new List <(DateTime ValidityStart, DateTime ValidityEnd)>()
{
(new DateTime(2018, 1, 1), new DateTime(2018, 1, 3)),
(new DateTime(2018, 1, 2), DateTime.MaxValue)
})
};
YDS.ThereAreVideosWithManyVersions(videos);
using (var context = new DataLakeYouTubeDataContext()) {
Assert.Equal(2, context.Videos.Count());
var(schema, table, keys) = ContextIntrospection.GetDatabaseInfo(context, typeof(Video));
TableOperations.DeleteFromTable(YearDatabase.DataLakeDatabase, schema, table);
Assert.Empty(context.Videos);
}
}
public void YouTubeVideosInDataLakeAreTransformedIntoSourceVideos()
{
var channelId = "fee";
var titleAtypo = "The True Mening of Patriotism";
var titleA = "The True Meaning of Patriotism";
var titleB = "The Population Boom";
var someLakeVideos = new[] {
new Video()
{
VideoId = "xyz", Title = titleAtypo, Duration = "PT5M11S"
},
new Video()
{
VideoId = "zxy", Title = titleB, Duration = "PT2M6S"
},
};
YDS.SomeVideosWereFound(someLakeVideos, channelId);
APS.YouTubeVideoSyncJobHasRun();
using (var context = new DataLakeYouTubeDataContext()) {
var videos = context.Videos.OrderBy(x => x.Title).ToList();
var titles = new[] { titleB, titleAtypo };
Assert.Equal(titles.Count(), videos.Count());
for (var i = 0; i < titles.Count(); i++)
{
Assert.Equal(titles[i], videos[i].Title);
}
}
using (var context = new ApplicationDbContext()) {
var videos = context.SourceVideos.OrderBy(x => x.Title).ToList();
var titles = new[] { titleB, titleAtypo };
Assert.Equal(titles.Count(), videos.Count());
for (var i = 0; i < titles.Count(); i++)
{
Assert.Equal(titles[i], videos[i].Title);
}
}
someLakeVideos = new[] {
new Video()
{
VideoId = "xyz", Title = titleA, Duration = "PT5M11S"
},
new Video()
{
VideoId = "zxy", Title = titleB, Duration = "PT2M6S"
},
};
YDS.SomeVideosWereFound(someLakeVideos, channelId);
APS.YouTubeVideoSyncJobHasRun();
using (var context = new DataLakeYouTubeDataContext()) {
var videos = context.Videos.OrderBy(x => x.Title).ToList();
var titles = new[] { titleB, titleA, titleAtypo };
Assert.Equal(titles.Count(), videos.Count());
for (var i = 0; i < titles.Count(); i++)
{
Assert.Equal(titles[i], videos[i].Title);
}
}
using (var context = new ApplicationDbContext()) {
var videos = context.SourceVideos.OrderBy(x => x.Title).ToList();
var titles = new[] { titleB, titleA };
Assert.Equal(titles.Count(), videos.Count());
for (var i = 0; i < titles.Count(); i++)
{
Assert.Equal(titles[i], videos[i].Title);
}
}
}
