public async Task CanGetEventTermStatsByStackAsync()
{
// capture start date before generating data to make sure that our time range for stats includes all items
var startDate = SystemClock.UtcNow.SubtractDays(60);
const int eventCount = 100;
await CreateDataAsync(eventCount, false);
var fields = FieldAggregationProcessor.Process("distinct:stack_id,term:is_first_occurrence:-F", false);
Assert.True(fields.IsValid);
var sf = new ExceptionlessSystemFilterQuery(ProjectData.GenerateSampleProject(), OrganizationData.GenerateSampleOrganization());
var result = await _stats.GetNumbersTermsStatsAsync("stack_id", fields.Aggregations, startDate, SystemClock.UtcNow, sf);
Assert.Equal(eventCount, result.Total);
Assert.InRange(result.Terms.Count, 1, 25);
// TODO: Figure out why this is less than eventCount
Assert.Equal(eventCount, result.Terms.Sum(t => t.Total));
Assert.InRange(result.Terms.Sum(t => t.Numbers[1]), 1, 25); // new
foreach (var term in result.Terms)
{
Assert.Equal(1, term.Numbers[0]); //unique
Assert.Equal(1, term.Numbers[1]); // new
}
}
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> viewOrganizations)
{
if (viewOrganizations.Count <= 0)
{
return(viewOrganizations);
}
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
}
};
var organizations = viewOrganizations.Select(o => new Organization {
Id = o.Id, RetentionDays = o.RetentionDays
}).ToList();
var sf = new ExceptionlessSystemFilterQuery(organizations);
var result = await _stats.GetNumbersTermsStatsAsync("organization_id", fields, organizations.GetRetentionUtcCutoff(), DateTime.MaxValue, sf, max : viewOrganizations.Count);
foreach (var organization in viewOrganizations)
{
var organizationStats = result.Terms.FirstOrDefault(t => t.Term == organization.Id);
organization.EventCount = organizationStats?.Total ?? 0;
organization.StackCount = (long)(organizationStats?.Numbers[0] ?? 0);
organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id);
}
return(viewOrganizations);
}
public async Task CanGetEventTermStatsByTagAsync()
{
// capture start date before generating data to make sure that our time range for stats includes all items
var startDate = SystemClock.UtcNow.SubtractDays(60);
const int eventCount = 100;
await CreateDataAsync(eventCount, false);
Log.MinimumLevel = LogLevel.Trace;
var fields = FieldAggregationProcessor.Process("term:is_first_occurrence:-F", false);
Assert.True(fields.IsValid);
var sf = new ExceptionlessSystemFilterQuery(ProjectData.GenerateSampleProject(), OrganizationData.GenerateSampleOrganization());
var result = await _stats.GetNumbersTermsStatsAsync("tags", fields.Aggregations, startDate, SystemClock.UtcNow, sf, "fixed:false");
Assert.Equal(eventCount, result.Total);
// each event can be in multiple tag buckets since an event can have up to 3 sample tags
Assert.InRange(result.Terms.Sum(t => t.Total), eventCount, eventCount * 3);
Assert.InRange(result.Terms.Sum(t => t.Numbers[0]), 1, 25 * TestConstants.EventTags.Count); // new
Assert.InRange(result.Terms.Count, 1, TestConstants.EventTags.Count);
foreach (var term in result.Terms)
{
Assert.InRange(term.Numbers[0], 1, 25); // new
}
}
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> viewProjects)
{
if (viewProjects.Count <= 0)
{
return(viewProjects);
}
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
}
};
var organizations = await _organizationRepository.GetByIdsAsync(viewProjects.Select(p => p.OrganizationId).ToArray(), true);
var projects = viewProjects.Select(p => new Project {
Id = p.Id, OrganizationId = p.OrganizationId
}).ToList();
var sf = new ExceptionlessSystemFilterQuery(projects, organizations);
var ntsr = await _stats.GetNumbersTermsStatsAsync("project_id", fields, organizations.GetRetentionUtcCutoff(), DateTime.MaxValue, sf, max : viewProjects.Count);
foreach (var project in viewProjects)
{
var term = ntsr.Terms.FirstOrDefault(t => t.Term == project.Id);
project.EventCount = term?.Total ?? 0;
project.StackCount = (long)(term?.Numbers[0] ?? 0);
}
return(viewProjects);
}
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> viewProjects)
{
if (viewProjects.Count <= 0)
{
return(viewProjects);
}
var organizations = await _organizationRepository.GetByIdsAsync(viewProjects.Select(p => p.OrganizationId).ToArray(), true);
var projects = viewProjects.Select(p => new Project {
Id = p.Id, CreatedUtc = p.CreatedUtc, OrganizationId = p.OrganizationId
}).ToList();
var sf = new ExceptionlessSystemFilterQuery(projects, organizations);
var systemFilter = new ElasticQuery().WithSystemFilter(sf).WithDateRange(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow, (PersistentEvent e) => e.Date).WithIndexes(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow);
var result = await _eventRepository.CountBySearchAsync(systemFilter, null, $"terms:(project_id~{viewProjects.Count} cardinality:stack_id)");
foreach (var project in viewProjects)
{
var term = result.Aggregations.Terms <string>("terms_project_id")?.Buckets.FirstOrDefault(t => t.Key == project.Id);
project.EventCount = term?.Total ?? 0;
project.StackCount = (long)(term?.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0);
}
return(viewProjects);
}
public async Task <IHttpActionResult> GetTimelineAsync(string fields = null, string filter = null, string time = null, string offset = null)
{
var far = FieldAggregationProcessor.Process(fields);
if (!far.IsValid)
{
return(BadRequest(far.Message));
}
var pr = QueryProcessor.Process(filter);
if (!pr.IsValid)
{
return(BadRequest(pr.Message));
}
var organizations = await GetAssociatedActiveOrganizationsAsync(_organizationRepository);
if (organizations.Count == 0)
{
return(Ok(NumbersTimelineStatsResult.Empty));
}
var ti = GetTimeInfo(time, offset, organizations.GetRetentionUtcCutoff());
var sf = new ExceptionlessSystemFilterQuery(organizations)
{
UsesPremiumFeatures = far.UsesPremiumFeatures || pr.UsesPremiumFeatures,
IsUserOrganizationsFilter = true
};
NumbersTimelineStatsResult result;
try {
result = await _stats.GetNumbersTimelineStatsAsync(far.Aggregations, ti.UtcRange.Start, ti.UtcRange.End, ShouldApplySystemFilter(sf, filter)?sf : null, pr.ExpandedQuery, ti.Offset);
} catch (ApplicationException ex) {
_logger.Error().Exception(ex)
.Message("An error has occurred. Please check your search filter.")
.Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = time, Offset = offset })
.Tag("Search")
.Identity(ExceptionlessUser.EmailAddress)
.Property("User", ExceptionlessUser)
.SetActionContext(ActionContext).Write();
return(BadRequest("An error has occurred. Please check your search filter."));
}
return(Ok(result));
}
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> viewOrganizations)
{
if (viewOrganizations.Count <= 0)
{
return(viewOrganizations);
}
var organizations = viewOrganizations.Select(o => new Organization {
Id = o.Id, CreatedUtc = o.CreatedUtc, RetentionDays = o.RetentionDays
}).ToList();
var sf = new ExceptionlessSystemFilterQuery(organizations);
var systemFilter = new ElasticQuery().WithSystemFilter(sf).WithDateRange(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow, (PersistentEvent e) => e.Date).WithIndexes(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow);
var result = await _eventRepository.CountBySearchAsync(systemFilter, null, $"terms:(organization_id~{viewOrganizations.Count} cardinality:stack_id)");
foreach (var organization in viewOrganizations)
{
var organizationStats = result.Aggregations.Terms <string>("terms_organization_id")?.Buckets.FirstOrDefault(t => t.Key == organization.Id);
organization.EventCount = organizationStats?.Total ?? 0;
organization.StackCount = (long?)organizationStats?.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id);
}
return(viewOrganizations);
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write();
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, true).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
if (organization == null)
{
_logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id);
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
},
new TermFieldAggregation {
Field = "is_first_occurrence", ExcludePattern = "F"
}
};
var sf = new ExceptionlessSystemFilterQuery(project, organization);
var result = await _stats.GetNumbersStatsAsync(fields, data.UtcStartTime, data.UtcEndTime, sf, $"{EventIndexType.Fields.Type}:{Event.KnownTypes.Error}").AnyContext();
bool hasSubmittedEvents = result.Total > 0;
if (!hasSubmittedEvents)
{
hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;
}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.Numbers[1],
UniqueTotal = result.Numbers[0],
HasSubmittedEvents = hasSubmittedEvents,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext();
}
_logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, notification.Total);
return(true);
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write();
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, true).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
if (organization == null)
{
_logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id);
var sf = new ExceptionlessSystemFilterQuery(project, organization);
var systemFilter = new ElasticQuery().WithSystemFilter(sf).WithDateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).WithIndexes(data.UtcStartTime, data.UtcEndTime);
var result = await _eventRepository.CountBySearchAsync(systemFilter, $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error}", "terms:(is_first_occurrence @include:true) cardinality:stack_id").AnyContext();
bool hasSubmittedEvents = result.Total > 0;
if (!hasSubmittedEvents)
{
hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id, true).AnyContext() > 0;
}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.Aggregations.Terms <double>("terms_is_first_occurrence")?.Buckets.FirstOrDefault()?.Total ?? 0,
UniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0,
HasSubmittedEvents = hasSubmittedEvents,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext();
}
_logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, notification.Total);
return(true);
}
