public override Dictionary <MetricType, TimeSeries> ComputePrimitiveTimeSeries(MetricType[] metrics, string type, DateTime start, DateTime end, Tag[] filters, ArchiveMode archive)
{
var store = new Dictionary <MetricType, TimeSeries>();
var dates = DateUtilities.GetDatesBetween(start, end);
foreach (var metric in metrics)
{
var Values = new List <TimeSeriesDataGroup>();
IEnumerable <string> groups;
// Marketing ignores type
if (type == null)
{
groups = PersonaList();
}
else
{
groups = ApplyFilters(Videos, filters, archive)
.Select(x => x.Tags.Where(y => y.Type.ToLower() == type.ToLower()).Select(y => y.Value))
.Where(v => v.Any())
.Aggregate(new List <string>().AsEnumerable(), (x, y) => x.Concat(y));
}
foreach (var group in groups.Distinct())
{
var values = dates.Select(x => (rnd.Next(1, 300)) * 1.0).ToArray();
Values.Add(new TimeSeriesDataGroup {
GroupName = group, Values = values
});
}
store.Add(metric, ReduceTimeSeries(dates, Values));
}
return(store);
}
public override Dictionary <MetricType, TimeSeries> ComputePrimitiveTimeSeries(MetricType[] metricNames, string type, DateTime start, DateTime end, Tag[] filters, ArchiveMode archive)
{
using (var context = new ApplicationDbContext()) {
var videoMetricPersonas = MetricsByPersona(start, end, context, filters, archive);
var allSamples = (from vmp in videoMetricPersonas
select vmp).ToList();
// metricNames.Select(metricName => )).ToList();
var dateSet = DateUtilities.GetDatesBetween(start, end);
return(metricNames.Select((v, ix) => {
var timeSeriesGroup = from personaMetricRow in allSamples.Select(vmp => new Sample <string>()
{
Date = vmp.Metric.EventDate, Value = ProjectMetric(vmp.Metric, v), Group = vmp.Persona
})
group personaMetricRow by personaMetricRow.Group into personaMetric
select new TimeSeriesDataGroup()
{
GroupName = personaMetric.Key,
Values = (from date in dateSet
join m in personaMetric on date equals m.Date into dateMetrics
select dateMetrics.Select(x => x.Value).DefaultIfEmpty(0).Sum()).ToArray()
};
return (v, ReduceTimeSeries(dateSet, timeSeriesGroup.AsEnumerable()));
}).ToDictionary(x => x.Item1, x => x.Item2));
}
}
public List <VideoMetric> GetMetricList(string startDate, string endDate, string filters = "[]", ArchiveMode archive = ArchiveMode.UnArchived)
{
var filterList = JsonConvert.DeserializeObject <Tag[]>(filters);
var start = DateUtilities.ReadDate(startDate);
var end = DateUtilities.ReadDate(endDate);
return(Backend.MetricList(start, end, filterList, GetMetricInfoList(), archive));
}
public IEnumerable <TimeSeriesChartData> GetChartData(string metrics, string type, string startDate, string endDate, string filters = "[]", ArchiveMode archive = ArchiveMode.UnArchived)
{
var filterList = JsonConvert.DeserializeObject <Tag[]>(filters);
var metricList = JsonConvert.DeserializeObject <string[]>(metrics);
var start = DateUtilities.ReadDate(startDate);
var end = DateUtilities.ReadDate(endDate);
var timeSeries = ChartData(metricList, type, start, end, filterList, archive);
return(timeSeries.Select(x => CreateTimeSeriesChartData(x.Value, start, end, x.Key)));
}
public JsonResult GetDemographicsChartData(string metric, string type, string startDate, string endDate, string filters, ArchiveMode archive = ArchiveMode.UnArchived)
{
var filterList = JsonConvert.DeserializeObject <Tag[]>(filters);
var metricList = JsonConvert.DeserializeObject <string[]>(metric);
var start = DateUtilities.ReadDate(startDate);
var end = DateUtilities.ReadDate(endDate);
return(Json(metricList.Select(m => new {
StartDate = startDate,
EndDate = endDate,
Metrics = m,
Data = GetDemographics(m, type, start, end, filterList, archive)
})
));
}
// returns map of video id -> date -> metric name -> double
public Dictionary <int, Dictionary <string, Dictionary <string, double> > > VideoMetricByDay(IEnumerable <int> apVideoIds, DateTime start, DateTime end)
{
using (var context = new ApplicationDbContext())
{
var datesInRange = DateUtilities.GetDatesBetween(start, end);
var filteredIds = apVideoIds.ToList();
var sourceDaily = from v in context.ApplicationVideos.Where(video => filteredIds.Contains(video.Id))
join avsv in context.ApplicationVideoSourceVideos on v.Id equals avsv.ApplicationVideoId
join sv in context.SourceVideos on avsv.SourceVideoId equals sv.Id
join daily in context.SourceVideoMetrics.AsNoTracking().Where(m => m.EventDate >= start && m.EventDate <= end) on sv.Id equals daily.VideoId
group daily by new { v.Id, daily.EventDate } into x
select new
{
Id = x.Key.Id,
EventDate = x.Key.EventDate.Date,
Views = x.Sum(m => m.ViewCount ?? 0),
Reactions = x.Sum(m => m.ReactionCount ?? 0),
};
return(sourceDaily
.ToList()
.GroupBy(x => x.Id)
.ToDictionary(
x => x.Key,
x => datesInRange.Select(
d => new {
Date = DateUtilities.ToRestApiDateFormat(d),
Metric =
new List <Metric>()
{
new Metric()
{
Type = "Views",
Value = x.SingleOrDefault(dm => d == dm.EventDate)?.Views ?? 0
}, new Metric()
{
Type = "Reactions",
Value = x.SingleOrDefault(dm => d == dm.EventDate)?.Reactions ?? 0
}
}
}).ToDictionary(
z => z.Date,
z => z.Metric.ToDictionary(j => j.Type.ToLower(), j => j.Value)
)
));
}
}
private TimeSeries GenerateMockedChartData(DateTime startDate, DateTime endDate, IEnumerable <string> groups)
{
IEnumerable <DateTime> dates = DateUtilities.GetDatesBetween(startDate, endDate);
var Values = new List <TimeSeriesDataGroup>();
Random rnd = new Random();
foreach (var groupName in groups)
{
var values = dates.Select(x => (rnd.Next(1, 300)) * 1.0).ToArray();
Values.Add(new TimeSeriesDataGroup {
GroupName = groupName, Values = values
});
}
return(AbstractDataBackend.ReduceTimeSeries(dates, Values));
}
public override Dictionary <MetricType, TimeSeries> ComputePrimitiveTimeSeries(MetricType[] metrics, string type, DateTime start, DateTime end, Tag[] filters, ArchiveMode archive)
{
using (var context = new ApplicationDbContext()) {
var dateInRange = DateUtilities.GetDatesBetween(start, end);
//To prevent null pointer exception when selected metaTagType is "generic", set metaTagTypeId 0
var filteredApplicationVideos = ApplyFilters(context, context.ApplicationVideos, filters, archive).Select(x => x.Id).ToList();
var filteredSourceVideos = ApplySourceFilters(context, filters).Select(x => x.Id).ToList();
var store = new ConcurrentDictionary <MetricType, TimeSeries>();
var sourceVideoMetrics = metrics.Select(metric => Array.Find(SourceVideoMetrics, x => x == metric)).Distinct();
switch (type.ToLower())
{
case "generic":
TimeSeriesGroupedByGenericTag(metrics, start, end, dateInRange, filteredApplicationVideos, filteredSourceVideos, sourceVideoMetrics, context, store);
break;
default:
TimeSeriesGroupedByMetaTag(metrics, start, end, type, dateInRange, filteredApplicationVideos, filteredSourceVideos, sourceVideoMetrics, context, store);
break;
}
return(store.ToDictionary(x => x.Key, x => x.Value));
}
}
public ActionResult <TrendingResult> TopK(
[FromQuery(Name = "k")] int size = 10,
[FromQuery(Name = "from")] DateTime?dateStart = null,
[FromQuery(Name = "until")] DateTime?dateStop = null,
[FromQuery(Name = "sort")] string sortMetric = "Views",
[FromQuery(Name = "when")] string when = "custom"
)
{
sortMetric = sortMetric.ToLower();
DateTime queryDateStart;
DateTime queryDateStop;
switch (when)
{
case "custom":
queryDateStart = dateStart ?? DateTime.UtcNow.Date.Subtract(new TimeSpan(15, 0, 0, 0));
queryDateStop = dateStop ?? queryDateStart.Subtract(new TimeSpan(-15, 0, 0, 0));
break;
case "last_week":
queryDateStart = DateTime.UtcNow.Date.Subtract(new TimeSpan(7, 0, 0, 0));
queryDateStop = DateTime.UtcNow.Date;
break;
case "yesterday":
queryDateStart = DateTime.UtcNow.Date.Subtract(new TimeSpan(1, 0, 0, 0));
queryDateStop = queryDateStart;
break;
case "today":
queryDateStart = DateTime.UtcNow.Date;
queryDateStop = queryDateStart;
break;
default:
return(BadRequest($"Parameter 'when' must be one of: 'custom', 'last_week', 'today', 'yesterday'"));
}
var queryRangeLength = (queryDateStop - queryDateStart).TotalDays;
// the set of valid parameters is restricted in order to bound impact of query on the system
if (size > 20 | size < 1)
{
return(BadRequest("Parameter 'k' must be one of 1,2,...,20"));
}
if (!AcceptedSortingColumns.Exists(x => x == sortMetric))
{
return(BadRequest($"Cannot sort on '{sortMetric}'. Accepted values: {string.Join(",", AcceptedSortingColumns)}"));
}
if (queryRangeLength > 90)
{
return(BadRequest("Selected period cannot be greater than 90 days"));
}
if (queryRangeLength < 0)
{
return(BadRequest("Invalid date range"));
}
var metricList = _dataController.GetMetricList(
DateUtilities.ToControllersInputFormat(queryDateStart),
DateUtilities.ToControllersInputFormat(queryDateStop)
);
var scores = metricList
.Select(x => new { id = x.Id, score = x.TotalMetrics.SingleOrDefault(m => m.Type.ToLower() == sortMetric)?.Value ?? 0, m = x })
.OrderByDescending(x => x.score)
.Take(size);
var dailyBreakDown = _dataBackend.VideoMetricByDay(
scores.Select(x => Convert.ToInt32(x.id)),
queryDateStart,
queryDateStop
);
var videoInfo = _dataController.GetVideoList();
var queryResult = (
from s in scores
join v in videoInfo on s.id equals v.Id
select new VideoTrendingInfo {
Title = v.Title,
DatePublished = DateUtilities.ToRestApiDateFormat(v.PublishedAt.Date),
Tags = Clean(v.Tags),
Urls = Clean(v.Sources),
Total = AcceptedSortingColumns.
ToDictionary(
x => x,
x => s.m.TotalMetrics
.SingleOrDefault(m => m.Type.ToLower() == x)
?.Value
),
DailyBreakDown = dailyBreakDown[Convert.ToInt32(s.id)]
}).ToList();
if (!queryResult.Any())
{
return(BadRequest($"No '{sortMetric}' on the selected period"));
}
return(Ok(new TrendingResult {
From = DateUtilities.ToRestApiDateFormat(queryDateStart),
Until = DateUtilities.ToRestApiDateFormat(queryDateStop),
K = size,
Sort = sortMetric,
TopK = queryResult,
}));
}