public PlanPagedResult <ErrorStackResult> Frequent(string projectId, int page = 1, int pageSize = 10, DateTime?start = null, DateTime?end = null, bool hidden = false, bool @fixed = false, bool notfound = true)
{
if (String.IsNullOrEmpty(projectId))
{
throw new ArgumentNullException("projectId"); // TODO: These should probably throw http Response exceptions.
}
Project project = _projectRepository.GetByIdCached(projectId);
if (project == null || !User.CanAccessOrganization(project.OrganizationId))
{
throw new ArgumentException("Invalid project id.", "projectId"); // TODO: These should probably throw http Response exceptions.
}
DateTime retentionUtcCutoff = _organizationRepository.GetByIdCached(project.OrganizationId).GetRetentionUtcCutoff();
ProjectErrorStatsResult result = _statsHelper.GetProjectErrorStats(projectId, _projectRepository.GetDefaultTimeOffset(projectId), start, end, retentionUtcCutoff, hidden, @fixed, notfound);
return(Frequent(result.MostFrequent.Results, result.TotalLimitedByPlan, page, pageSize));
}
public ProjectErrorStatsResult GetProjectErrorStatsByDay(string projectId, TimeSpan utcOffset, DateTime? localStartDate = null, DateTime? localEndDate = null, DateTime? retentionStartDate = null, bool includeHidden = false, bool includeFixed = false, bool includeNotFound = true) {
if (localStartDate == null)
localStartDate = new DateTime(2012, 1, 1);
if (localEndDate == null)
localEndDate = DateTime.UtcNow.Add(utcOffset);
// TODO: Check to see if we have day stats available for the project, if not, then use minute stats and queue new project offset to be created in a job
// round date range to be full days since stats are per day
localStartDate = localStartDate.Value.Floor(TimeSpan.FromDays(1));
localEndDate = localEndDate.Value.Ceiling(TimeSpan.FromDays(1));
if (retentionStartDate.HasValue)
retentionStartDate = retentionStartDate.Value.Floor(TimeSpan.FromDays(1));
if (localEndDate.Value <= localStartDate.Value)
throw new ArgumentException("End date must be greater than start date.", "localEndDate");
var results = _monthProjectStats.Collection.Find(Query.And(
Query.GTE(MonthProjectStatsRepository.FieldNames.Id, GetMonthProjectStatsId(localStartDate.Value, utcOffset, projectId)),
Query.LTE(MonthProjectStatsRepository.FieldNames.Id, GetMonthProjectStatsId(localEndDate.Value, utcOffset, projectId))
)).ToList();
if (results.Count > 0) {
var firstWithOccurrence = results.OrderBy(r => r.Id).FirstOrDefault(r => r.DayStats.Any(ds => ds.Value.Total > 0));
if (firstWithOccurrence != null) {
var firstErrorDate = firstWithOccurrence.GetDateFromDayStatKey(firstWithOccurrence.DayStats.OrderBy(ds => Int32.Parse(ds.Key)).First(ds => ds.Value.Total > 0).Key);
if (localStartDate < firstErrorDate)
localStartDate = firstErrorDate;
}
if (!includeHidden) {
// remove stats from hidden doc ids.
var hiddenIds = _errorStackRepository.GetHiddenIds(projectId);
if (hiddenIds.Length > 0)
DecrementMonthProjectStatsByStackIds(results, hiddenIds);
}
if (!includeNotFound) {
// remove stats from not found doc ids.
var notFoundIds = _errorStackRepository.GetNotFoundIds(projectId);
if (notFoundIds.Length > 0)
DecrementMonthProjectStatsByStackIds(results, notFoundIds);
}
if (!includeFixed) {
// remove stats from not found doc ids.
var fixedIds = _errorStackRepository.GetFixedIds(projectId);
if (fixedIds.Length > 0)
DecrementMonthProjectStatsByStackIds(results, fixedIds);
}
}
// use finer grained minute blocks if the range is less than 5 days
if (localEndDate.Value.Subtract(localStartDate.Value).TotalDays < 5)
return GetProjectErrorStatsByMinuteBlock(projectId, utcOffset, localStartDate, localEndDate, retentionStartDate, includeHidden, includeFixed, includeNotFound);
var monthDocIds = new List<Tuple<int, int>>();
DateTime currentDay = localStartDate.Value;
var endOfMonthEndDate = new DateTime(localEndDate.Value.Year, localEndDate.Value.Month, DateTime.DaysInMonth(localEndDate.Value.Year, localEndDate.Value.Month));
while (currentDay <= endOfMonthEndDate) {
monthDocIds.Add(Tuple.Create(currentDay.Year, currentDay.Month));
currentDay = currentDay.AddMonths(1);
}
// add missing month documents
foreach (var monthDocId in monthDocIds) {
if (!results.Exists(d => d.Id == GetMonthProjectStatsId(monthDocId.Item1, monthDocId.Item2, utcOffset, projectId)))
results.Add(CreateBlankMonthProjectStats(utcOffset, new DateTime(monthDocId.Item1, monthDocId.Item2, 1), projectId));
}
// fill out missing days with blank stats
foreach (MonthProjectStats r in results) {
DateTime date = r.GetDateFromDayStatKey("1");
int daysInMonth = DateTime.DaysInMonth(date.Year, date.Month);
for (int i = 1; i <= daysInMonth; i++) {
if (!r.DayStats.ContainsKey(i.ToString()))
r.DayStats.Add(i.ToString(), new ErrorStatsWithStackIds());
}
}
var days = new List<KeyValuePair<DateTime, ErrorStatsWithStackIds>>();
days = results.Aggregate(days, (current, result) => current.Concat(result.DayStats.ToDictionary(kvp => result.GetDateFromDayStatKey(kvp.Key), kvp => kvp.Value)).ToList())
.Where(kvp => kvp.Key >= localStartDate.Value && kvp.Key <= localEndDate.Value).OrderBy(kvp => kvp.Key).ToList();
int totalLimitedByPlan = retentionStartDate != null && localStartDate < retentionStartDate
? days.Where(kvp => kvp.Key < retentionStartDate).SelectMany(kvp => kvp.Value.ErrorStackIds.Select(s => s.Key)).Distinct()
.Except(days.Where(kvp => kvp.Key >= retentionStartDate).SelectMany(kvp => kvp.Value.ErrorStackIds.Select(s => s.Key)).Distinct())
.Count()
: 0;
if (totalLimitedByPlan > 0)
days = days.Where(kvp => kvp.Key >= retentionStartDate).ToList();
// group data points by a time span to limit the number of returned data points
TimeSpan groupTimeSpan = TimeSpan.FromDays(1);
if (days.Count > 50) {
DateTime first = days.Min(m => m.Key);
DateTime last = days.Max(m => m.Key);
TimeSpan span = last - first;
groupTimeSpan = TimeSpan.FromDays(((int)Math.Round((span.TotalDays / 50) / 1.0)) * 1);
}
var stats = new ProjectErrorStatsResult {
TotalLimitedByPlan = totalLimitedByPlan,
StartDate = localStartDate.Value,
EndDate = localEndDate.Value,
Total = days.Sum(kvp => kvp.Value.Total),
UniqueTotal = days.SelectMany(kvp => kvp.Value.ErrorStackIds.Select(s => s.Key)).Distinct().Count(),
NewTotal = days.Sum(kvp => kvp.Value.NewTotal),
MostFrequent = new PlanPagedResult<ErrorStackResult>(days.SelectMany(kvp => kvp.Value.ErrorStackIds).GroupBy(kvp => kvp.Key)
.Select(v => new ErrorStackResult {
Id = v.Key,
Total = v.Sum(kvp => kvp.Value)
}).OrderByDescending(s => s.Total).ToList(), totalLimitedByPlan),
Stats = days.GroupBy(s => s.Key.Floor(groupTimeSpan)).Select(kvp => new DateProjectStatsResult {
Date = kvp.Key,
Total = kvp.Sum(b => b.Value.Total),
NewTotal = kvp.Sum(b => b.Value.NewTotal),
UniqueTotal = kvp.Select(b => b.Value.ErrorStackIds).Aggregate((current, result) => Merge(result, current)).Count()
}).ToList()
};
return stats;
}
public ProjectErrorStatsResult GetProjectErrorStatsByMinuteBlock(string projectId, TimeSpan utcOffset, DateTime? localStartDate = null, DateTime? localEndDate = null, DateTime? retentionStartDate = null, bool includeHidden = false, bool includeFixed = false, bool includeNotFound = true) {
if (localStartDate == null)
localStartDate = new DateTime(2012, 1, 1);
if (localEndDate == null)
localEndDate = DateTime.UtcNow.Add(utcOffset);
// round date range to blocks of 15 minutes since stats are per 15 minute block
localStartDate = localStartDate.Value.Floor(TimeSpan.FromMinutes(15));
localEndDate = localEndDate.Value.Ceiling(TimeSpan.FromMinutes(15));
if (retentionStartDate.HasValue)
retentionStartDate = retentionStartDate.Value.Floor(TimeSpan.FromMinutes(15));
if (localEndDate.Value <= localStartDate.Value)
throw new ArgumentException("End date must be greater than start date.", "localEndDate");
DateTime utcStartDate = new DateTimeOffset(localStartDate.Value.Ticks, utcOffset).UtcDateTime;
DateTime utcEndDate = new DateTimeOffset(localEndDate.Value.Ticks, utcOffset).UtcDateTime;
List<DayProjectStats> results = _dayProjectStats.Collection.Find(Query.And(
Query.GTE(MonthProjectStatsRepository.FieldNames.Id, GetDayProjectStatsId(projectId, utcStartDate)),
Query.LTE(MonthProjectStatsRepository.FieldNames.Id, GetDayProjectStatsId(projectId, utcEndDate))
)).ToList();
if (results.Count > 0) {
DayProjectStats firstWithOccurrence = results.OrderBy(r => r.Id).FirstOrDefault(r => r.MinuteStats.Any(ds => ds.Value.Total > 0));
if (firstWithOccurrence != null) {
DateTime firstErrorDate = firstWithOccurrence.GetDateFromMinuteStatKey(firstWithOccurrence.MinuteStats.OrderBy(ds => Int32.Parse(ds.Key)).First(ds => ds.Value.Total > 0).Key);
if (utcStartDate < firstErrorDate)
utcStartDate = firstErrorDate;
}
if (!includeHidden) {
// remove stats from hidden doc ids.
string[] hiddenIds = _errorStackRepository.GetHiddenIds(projectId);
if (hiddenIds.Length > 0)
DecrementDayProjectStatsByStackIds(results, hiddenIds);
}
if (!includeNotFound) {
// remove stats from not found doc ids.
string[] notFoundIds = _errorStackRepository.GetNotFoundIds(projectId);
if (notFoundIds.Length > 0)
DecrementDayProjectStatsByStackIds(results, notFoundIds);
}
if (!includeFixed) {
// remove stats from not found doc ids.
string[] fixedIds = _errorStackRepository.GetFixedIds(projectId);
if (fixedIds.Length > 0)
DecrementDayProjectStatsByStackIds(results, fixedIds);
}
}
var dayDocDates = new List<DateTime>();
DateTime currentDay = utcStartDate;
DateTime endOfDayEndDate = utcEndDate.ToEndOfDay();
while (currentDay <= endOfDayEndDate) {
dayDocDates.Add(currentDay);
currentDay = currentDay.AddDays(1);
}
// add missing day documents
foreach (DateTime dayDocDate in dayDocDates) {
if (!results.Exists(d => d.Id == GetDayProjectStatsId(projectId, dayDocDate)))
results.Add(CreateBlankDayProjectStats(dayDocDate, projectId));
}
// fill out missing minute blocks with blank stats
foreach (DayProjectStats r in results) {
const int minuteBlocksInDay = 96;
for (int i = 0; i <= minuteBlocksInDay - 1; i++) {
int minuteBlock = i * 15;
if (!r.MinuteStats.ContainsKey(minuteBlock.ToString("0000")))
r.MinuteStats.Add(minuteBlock.ToString("0000"), new ErrorStatsWithStackIds());
}
}
var minuteBlocks = new List<KeyValuePair<DateTime, ErrorStatsWithStackIds>>();
minuteBlocks = results.Aggregate(minuteBlocks, (current, result) => current.Concat(result.MinuteStats.ToDictionary(kvp => result.GetDateFromMinuteStatKey(kvp.Key), kvp => kvp.Value)).ToList())
.Where(kvp => kvp.Key >= utcStartDate && kvp.Key <= utcEndDate).OrderBy(kvp => kvp.Key).ToList();
int totalLimitedByPlan = retentionStartDate != null && utcStartDate < retentionStartDate
? minuteBlocks.Where(kvp => kvp.Key < retentionStartDate).SelectMany(kvp => kvp.Value.ErrorStackIds.Select(s => s.Key)).Distinct()
.Except(minuteBlocks.Where(kvp => kvp.Key >= retentionStartDate).SelectMany(kvp => kvp.Value.ErrorStackIds.Select(s => s.Key)).Distinct())
.Count()
: 0;
if (totalLimitedByPlan > 0)
minuteBlocks = minuteBlocks.Where(kvp => kvp.Key >= retentionStartDate).ToList();
// group data points by a timespan to limit the number of returned data points
TimeSpan groupTimeSpan = TimeSpan.FromMinutes(15);
if (minuteBlocks.Count > 50) {
DateTime first = minuteBlocks.Min(m => m.Key);
DateTime last = minuteBlocks.Max(m => m.Key);
TimeSpan span = last - first;
groupTimeSpan = TimeSpan.FromMinutes(((int)Math.Round(span.TotalMinutes / 50 / 15.0)) * 15);
}
var stats = new ProjectErrorStatsResult {
TotalLimitedByPlan = totalLimitedByPlan,
StartDate = utcStartDate,
EndDate = utcEndDate,
Total = minuteBlocks.Sum(kvp => kvp.Value.Total),
UniqueTotal = minuteBlocks.SelectMany(kvp => kvp.Value.ErrorStackIds.Select(s => s.Key)).Distinct().Count(),
NewTotal = minuteBlocks.Sum(kvp => kvp.Value.NewTotal),
MostFrequent = new PlanPagedResult<ErrorStackResult>(minuteBlocks.SelectMany(kvp => kvp.Value.ErrorStackIds).GroupBy(kvp => kvp.Key)
.Select(v => new ErrorStackResult {
Id = v.Key,
Total = v.Sum(kvp => kvp.Value),
}).OrderByDescending(s => s.Total).ToList(), totalLimitedByPlan),
Stats = minuteBlocks.GroupBy(s => s.Key.Floor(groupTimeSpan)).Select(kvp => new DateProjectStatsResult {
Date = kvp.Key,
Total = kvp.Sum(b => b.Value.Total),
NewTotal = kvp.Sum(b => b.Value.NewTotal),
UniqueTotal = kvp.Select(b => b.Value.ErrorStackIds).Aggregate((current, result) => Merge(result, current)).Count()
}).ToList()
};
stats.ApplyTimeOffset(utcOffset);
return stats;
}
