public async Task CanGetEventTermStatsByTagAsync()
{
// capture start date before generating data to make sure that our time range for stats includes all items
var startDate = DateTime.UtcNow.SubtractDays(60);
const int eventCount = 100;
await RemoveDataAsync();
await CreateDataAsync(eventCount, false);
await _client.RefreshAsync(d => d.Force());
_metricsClient.DisplayStats();
var result = await _stats.GetTermsStatsAsync(startDate, DateTime.UtcNow, "tags", null, userFilter : "project:" + TestConstants.ProjectId);
Assert.Equal(eventCount, result.Total);
// each event can be in multiple tag buckets since an event can have up to 3 sample tags
Assert.InRange(result.Terms.Sum(t => t.Total), eventCount, eventCount * 3);
Assert.InRange(result.Terms.Sum(t => t.New), 1, 25 * TestConstants.EventTags.Count);
Assert.InRange(result.Terms.Count, 1, TestConstants.EventTags.Count);
foreach (var term in result.Terms)
{
Assert.InRange(term.New, 1, 25);
//Assert.InRange(term.Unique, 1, 25);
Assert.Equal(term.Total, term.Timeline.Sum(t => t.Total));
}
}
private async Task <IHttpActionResult> FrequentInternalAsync(string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10)
{
page = GetPage(page);
limit = GetLimit(limit);
var skip = GetSkip(page, limit);
if (skip > MAXIMUM_SKIP)
{
return(Ok(new object[0]));
}
var validationResult = QueryProcessor.Process(userFilter);
if (!validationResult.IsValid)
{
return(BadRequest(validationResult.Message));
}
if (String.IsNullOrEmpty(systemFilter))
{
systemFilter = await GetAssociatedOrganizationsFilterAsync(_organizationRepository, validationResult.UsesPremiumFeatures, HasOrganizationOrProjectFilter(userFilter));
}
var timeInfo = GetTimeInfo(time, offset);
ICollection <EventTermStatsItem> terms;
try {
terms = (await _eventStats.GetTermsStatsAsync(timeInfo.UtcRange.Start, timeInfo.UtcRange.End, "stack_id", systemFilter, userFilter, timeInfo.Offset, GetSkip(page + 1, limit) + 1)).Terms;
} catch (ApplicationException ex) {
Logger.Error().Exception(ex)
.Property("Search Filter", new { SystemFilter = systemFilter, UserFilter = userFilter, Time = time, Offset = offset, Page = page, Limit = limit })
.Tag("Search")
.Identity(ExceptionlessUser.EmailAddress)
.Property("User", ExceptionlessUser)
.SetActionContext(ActionContext)
.Write();
return(BadRequest("An error has occurred. Please check your search filter."));
}
if (terms.Count == 0)
{
return(Ok(new object[0]));
}
var stackIds = terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray();
var stacks = (await _stackRepository.GetByIdsAsync(stackIds)).Documents.Select(s => s.ApplyOffset(timeInfo.Offset)).ToList();
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.InvariantCultureIgnoreCase))
{
var summaries = GetStackSummaries(stacks, terms);
return(OkWithResourceLinks(GetStackSummaries(stacks, terms).Take(limit).ToList(), summaries.Count > limit, page));
}
return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page));
}
private async Task ProcessSummaryNotificationAsync(SummaryNotification data)
{
var project = await _projectRepository.GetByIdAsync(data.Id, true).AnyContext();
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
var users = (await _userRepository.GetByIdsAsync(userIds).AnyContext()).Documents.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList();
if (users.Count == 0)
{
_logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
_logger.Info().Message("Sending daily summary: users={0} project={1}", users.Count, project.Id).Write();
var result = await _stats.GetTermsStatsAsync(data.UtcStartTime, data.UtcEndTime, "stack_id", "type:error project:" + data.Id, max : 5).AnyContext();
bool hasSubmittedErrors = result.Total > 0;
if (!hasSubmittedErrors)
{
hasSubmittedErrors = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;
}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.New,
UniqueTotal = result.Unique,
HasSubmittedEvents = hasSubmittedErrors,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext();
}
_logger.Info().Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total).Write();
}
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> projects)
{
if (projects.Count <= 0)
{
return(projects);
}
var organizations = await _organizationRepository.GetByIdsAsync(projects.Select(p => p.Id).ToArray(), true);
StringBuilder builder = new StringBuilder();
for (int index = 0; index < projects.Count; index++)
{
if (index > 0)
{
builder.Append(" OR ");
}
var project = projects[index];
var organization = organizations.Documents.FirstOrDefault(o => o.Id == project.Id);
if (organization != null && organization.RetentionDays > 0)
{
builder.AppendFormat("(project:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", project.Id, organization.RetentionDays);
}
else
{
builder.AppendFormat("project:{0}", project.Id);
}
}
var result = await _stats.GetTermsStatsAsync(DateTime.MinValue, DateTime.MaxValue, "project_id", builder.ToString());
foreach (var project in projects)
{
var projectStats = result.Terms.FirstOrDefault(t => t.Term == project.Id);
project.EventCount = projectStats?.Total ?? 0;
project.StackCount = projectStats?.Unique ?? 0;
}
return(projects);
}
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> organizations)
{
if (organizations.Count <= 0)
{
return(organizations);
}
StringBuilder builder = new StringBuilder();
for (int index = 0; index < organizations.Count; index++)
{
if (index > 0)
{
builder.Append(" OR ");
}
var organization = organizations[index];
if (organization.RetentionDays > 0)
{
builder.AppendFormat("(organization:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", organization.Id, organization.RetentionDays);
}
else
{
builder.AppendFormat("organization:{0}", organization.Id);
}
}
var result = await _stats.GetTermsStatsAsync(DateTime.MinValue, DateTime.MaxValue, "organization_id", builder.ToString());
foreach (var organization in organizations)
{
var organizationStats = result.Terms.FirstOrDefault(t => t.Term == organization.Id);
organization.EventCount = organizationStats?.Total ?? 0;
organization.StackCount = organizationStats?.Unique ?? 0;
organization.ProjectCount = (await _projectRepository.GetByOrganizationIdAsync(organization.Id, useCache: true)).Documents.Count;
}
return(organizations);
}
private async Task ProcessSummaryNotificationAsync(SummaryNotification data)
{
var project = await _projectRepository.GetByIdAsync(data.Id, true).AnyContext();
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
Logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
var users = (await _userRepository.GetByIdsAsync(userIds).AnyContext()).Documents.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList();
if (users.Count == 0)
{
Logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
Logger.Info().Message("Sending daily summary: users={0} project={1}", users.Count, project.Id).Write();
//var paging = new PagingOptions { Limit = 5 };
//List<Stack> newest = (await _stackRepository.GetNewAsync(project.Id, data.UtcStartTime, data.UtcEndTime, paging).AnyContext()).Documents.ToList();
var newest = new List <Stack>();
var result = await _stats.GetTermsStatsAsync(data.UtcStartTime, data.UtcEndTime, "stack_id", "type:error project:" + data.Id, max : 5).AnyContext();
//var termStatsList = result.Terms.Take(5).ToList();
//var stacks = _stackRepository.GetByIds(termStatsList.Select(s => s.Term).ToList());
bool hasSubmittedErrors = result.Total > 0;
if (!hasSubmittedErrors)
{
hasSubmittedErrors = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;
}
var mostFrequent = new List <EventStackResult>();
//foreach (var termStats in termStatsList) {
// var stack = stacks.SingleOrDefault(s => s.Id == termStats.Term);
// if (stack == null)
// continue;
// mostFrequent.Add(new EventStackResult {
// First = termStats.FirstOccurrence,
// Last = termStats.LastOccurrence,
// Id = stack.Id,
// Title = stack.Title,
// Total = termStats.Total,
// Type = stack.SignatureInfo.ContainsKey("ExceptionType") ? stack.SignatureInfo["ExceptionType"] : null,
// Method = stack.SignatureInfo.ContainsKey("Method") ? stack.SignatureInfo["Method"] : null,
// Path = stack.SignatureInfo.ContainsKey("Source") ? stack.SignatureInfo["Source"] : null,
// Is404 = stack.SignatureInfo.ContainsKey("Type") && stack.SignatureInfo["Type"] == "404"
// });
//}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.New,
New = newest,
UniqueTotal = result.Unique,
MostFrequent = mostFrequent,
HasSubmittedEvents = hasSubmittedErrors,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext();
}
Logger.Info().Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total).Write();
}