public static void FetchViewerPercentageMetrics(YouTubeAnalyticsService analyticsService, string channelId, YTD.Video video, Logger logger)
{
using (var dbContext = new DataLakeYouTubeAnalyticsContext()) {
var now = DateTime.UtcNow;
var lastFetchedDate = dbContext.ViewerPercentageLastDates.SingleOrDefault(x => x.VideoId == video.VideoId);
var initialDate = video.PublishedAt;
if (lastFetchedDate == null)
{
// maybe the video simply doesn't support this report, check for early termination
if (!RunViewerPercentageReport(analyticsService, channelId, video.PublishedAt, now, video).Any())
{
logger.Debug("Report not available for video {VideoId}", video.VideoId);
return;
}
lastFetchedDate = new YTA.ViewerPercentageLastDate()
{
VideoId = video.VideoId, Date = initialDate
};
dbContext.Add(lastFetchedDate);
}
else
{
initialDate = DateHelper.Max(video.PublishedAt, DateHelper.Min(lastFetchedDate.Date, now - TimeMargin));
}
int replicatedDays = 0;
foreach (var date in DateHelper.DaysInRange(initialDate.Date, now.Date))
{
var viewerPercentages = RunViewerPercentageReport(analyticsService, channelId, video.PublishedAt.Date, date, video);
lastFetchedDate.Date = date;
if (!viewerPercentages.Any())
{
continue;
}
DbWriter.Write(
video.VideoId,
date,
viewerPercentages.Select(x => Api2DbObjectConverter.ConvertViewerPercentageRow(x)),
now
);
dbContext.SaveChanges();
replicatedDays++;
if (replicatedDays % MaxDaysToReplicateInIteration == 0)
{
break;
}
}
logger.Debug("Replicated {Days} days for video {VideoId}", replicatedDays, video.VideoId);
}
}
public static IEnumerable <YTA.VideoDailyMetric> FetchDailyMetrics(YouTubeAnalyticsService analyticsService, string channelId, YTD.Video video, Logger logger, bool reprocess = false)
{
using (var dbContext = new DataLakeYouTubeAnalyticsContext()) {
var now = DateTime.UtcNow;
var mostRecentRecord = dbContext.VideoDailyMetrics
.Where(x => x.VideoId == video.VideoId && x.ValidityStart <= now && now < x.ValidityEnd)
.OrderByDescending(x => x.Date)
.FirstOrDefault();
return(FetchVideoDailyMetrics(mostRecentRecord, channelId, video, now, analyticsService, logger, reprocess)
.Select(x => Api2DbObjectConverter.ConvertDailyMetricRow(video.VideoId, x)));
}
}
private static IList <IList <object> > RunViewerPercentageReport(YouTubeAnalyticsService analyticsService, string channelId, DateTime fromDate, DateTime toDate, YTD.Video video)
{
var reportRequest = analyticsService.Reports.Query();
reportRequest.Ids = $"channel=={channelId}";
reportRequest.StartDate = fromDate.ToString("yyyy-MM-dd");
reportRequest.EndDate = toDate.ToString("yyyy-MM-dd");
reportRequest.Metrics = "viewerPercentage";
reportRequest.Filters = String.Format("video=={0}", video.VideoId);
reportRequest.Dimensions = "gender,ageGroup";
reportRequest.Sort = "gender,ageGroup";
return(reportRequest.ExecuteAsync().Result.Rows);
}
private static IEnumerable <IList <object> > FetchVideoDailyMetrics(YTA.VideoDailyMetric mostRecentRecord, string channelId, YTD.Video video, DateTime now, YouTubeAnalyticsService analyticsService, Logger logger, bool reprocessMetrics = false)
{
DateTime mostRecentMetricDate;
if (mostRecentRecord == null || reprocessMetrics)
{
TimeSpan PublishedAtOffset;
if (reprocessMetrics)
{
PublishedAtOffset = new TimeSpan(30, 0, 0, 0);
}
else
{
PublishedAtOffset = TimeMargin;
}
mostRecentMetricDate = (video.PublishedAt != null) ? video.PublishedAt - PublishedAtOffset : now;
}
else
{
mostRecentMetricDate = mostRecentRecord.Date;
}
var fromDate = DateHelper.Min(now - TimeMargin, mostRecentMetricDate);
var toDate = now;
var reportRequest = analyticsService.Reports.Query();
reportRequest.Ids = $"channel=={channelId}";
reportRequest.StartDate = fromDate.ToString("yyyy-MM-dd");
reportRequest.EndDate = toDate.ToString("yyyy-MM-dd");
reportRequest.Metrics = "views,likes,shares,comments,averageViewDuration,dislikes,subscribersGained,subscribersLost,videosAddedToPlaylists,videosRemovedFromPlaylists";
reportRequest.Filters = $"video=={video.VideoId}";
reportRequest.Dimensions = "day";
reportRequest.Sort = "day";
var report = reportRequest.ExecuteAsync().Result;
if (report.Rows != null)
{
logger.Debug("Found {Rows} rows", report.Rows.Count);
foreach (var row in report.Rows)
{
yield return(row);
}
}
}
