private async Task <string> GetNextEventIdAsync(PersistentEvent ev, ExceptionlessSystemFilter systemFilter = null, string userFilter = null, DateTime?utcStart = null, DateTime?utcEnd = null)
{
if (ev == null)
{
return(null);
}
if (!utcStart.HasValue || utcStart.Value.IsBefore(ev.Date.UtcDateTime))
{
utcStart = ev.Date.UtcDateTime;
}
if (!utcEnd.HasValue || utcEnd.Value.IsAfter(SystemClock.UtcNow))
{
utcEnd = SystemClock.UtcNow;
}
var utcEventDate = ev.Date.UtcDateTime;
// utcEnd is before the current event date.
if (utcStart > utcEventDate || utcEnd < utcEventDate)
{
return(null);
}
if (String.IsNullOrEmpty(userFilter))
{
userFilter = String.Concat(EventIndexType.Alias.StackId, ":", ev.StackId);
}
var results = await FindAsync(q => q
.DateRange(utcEventDate, utcEnd, (PersistentEvent e) => e.Date)
.Index(utcEventDate, utcEnd)
.SortAscending(e => e.Date)
.Include(e => e.Id, e => e.Date)
.SystemFilter(systemFilter)
.ElasticFilter(!Query <PersistentEvent> .Ids(ids => ids.Values(ev.Id)))
.FilterExpression(userFilter), o => o.PageLimit(10)).AnyContext();
if (results.Total == 0)
{
return(null);
}
// make sure we don't have records with the exact same occurrence date
if (results.Documents.All(t => t.Date != ev.Date))
{
return(results.Documents.OrderBy(t => t.Date).ThenBy(t => t.Id).First().Id);
}
// we have records with the exact same occurrence date, we need to figure out the order of those
// put our target error into the mix, sort it and return the result after the target
var unionResults = results.Documents.Union(new[] { ev })
.OrderBy(t => t.Date.Ticks).ThenBy(t => t.Id)
.ToList();
int index = unionResults.FindIndex(t => t.Id == ev.Id);
return(index == unionResults.Count - 1 ? null : unionResults[index + 1].Id);
}
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> viewProjects)
{
if (viewProjects.Count <= 0)
{
return(viewProjects);
}
var organizations = await _organizationRepository.GetByIdsAsync(viewProjects.Select(p => p.OrganizationId).ToArray(), o => o.Cache());
var projects = viewProjects.Select(p => new Project {
Id = p.Id, CreatedUtc = p.CreatedUtc, OrganizationId = p.OrganizationId
}).ToList();
var sf = new ExceptionlessSystemFilter(projects, organizations);
var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow, (PersistentEvent e) => e.Date).Index(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow);
var result = await _eventRepository.CountBySearchAsync(systemFilter, null, $"terms:(project_id~{viewProjects.Count} cardinality:stack_id)");
foreach (var project in viewProjects)
{
var term = result.Aggregations.Terms <string>("terms_project_id")?.Buckets.FirstOrDefault(t => t.Key == project.Id);
project.EventCount = term?.Total ?? 0;
project.StackCount = (long)(term?.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0);
}
return(viewProjects);
}
protected async Task <ActionResult <CountResult> > GetCountImplAsync(ExceptionlessSystemFilter sf, TimeInfo ti, string filter = null, string aggregations = null)
{
var pr = await _validator.ValidateQueryAsync(filter);
if (!pr.IsValid)
{
return(BadRequest(pr.Message));
}
var far = await _validator.ValidateAggregationsAsync(aggregations);
if (!far.IsValid)
{
return(BadRequest(far.Message));
}
sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || far.UsesPremiumFeatures;
var query = new RepositoryQuery <TModel>()
.SystemFilter(ShouldApplySystemFilter(sf, filter) ? sf : null)
.DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field)
.Index(ti.Range.UtcStart, ti.Range.UtcEnd);
CountResult result;
try {
result = await _repository.CountBySearchAsync(query, filter, aggregations);
} catch (Exception ex) {
using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Aggregations = aggregations }).Tag("Search").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext)))
_logger.LogError(ex, "An error has occurred. Please check your filter or aggregations.");
return(BadRequest("An error has occurred. Please check your search filter."));
}
return(Ok(result));
}
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> viewOrganizations)
{
if (viewOrganizations.Count <= 0)
{
return(viewOrganizations);
}
int maximumRetentionDays = _options.Value.MaximumRetentionDays;
var organizations = viewOrganizations.Select(o => new Organization {
Id = o.Id, CreatedUtc = o.CreatedUtc, RetentionDays = o.RetentionDays
}).ToList();
var sf = new ExceptionlessSystemFilter(organizations);
var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(organizations.GetRetentionUtcCutoff(maximumRetentionDays), SystemClock.UtcNow, (PersistentEvent e) => e.Date).Index(organizations.GetRetentionUtcCutoff(maximumRetentionDays), SystemClock.UtcNow);
var result = await _eventRepository.CountBySearchAsync(systemFilter, null, $"terms:(organization_id~{viewOrganizations.Count} cardinality:stack_id)");
foreach (var organization in viewOrganizations)
{
var organizationStats = result.Aggregations.Terms <string>("terms_organization_id")?.Buckets.FirstOrDefault(t => t.Key == organization.Id);
organization.EventCount = organizationStats?.Total ?? 0;
organization.StackCount = (long?)organizationStats?.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id);
}
return(viewOrganizations);
}
public async Task <ActionResult <ViewProject> > GetAsync(string filter = null, string sort = null, int page = 1, int limit = 10, string mode = null)
{
var organizations = await GetSelectedOrganizationsAsync(_organizationRepository, _projectRepository, _stackRepository, filter);
if (organizations.Count == 0)
{
return(Ok(EmptyModels));
}
page = GetPage(page);
limit = GetLimit(limit, 1000);
var sf = new ExceptionlessSystemFilter(organizations)
{
IsUserOrganizationsFilter = true
};
var projects = await _repository.GetByFilterAsync(sf, filter, sort, o => o.PageNumber(page).PageLimit(limit));
var viewProjects = await MapCollectionAsync <ViewProject>(projects.Documents, true);
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "stats", StringComparison.OrdinalIgnoreCase))
{
return(OkWithResourceLinks(await PopulateProjectStatsAsync(viewProjects.ToList()), projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total));
}
return(OkWithResourceLinks(viewProjects, projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total));
}
public static T SystemFilter <T>(this T query, ExceptionlessSystemFilter filter) where T : IRepositoryQuery
{
if (filter != null)
{
return(query.BuildOption(SystemFilterKey, filter));
}
return(query);
}
public Task <FindResults <Project> > GetByFilterAsync(ExceptionlessSystemFilter systemFilter, string userFilter, string sort, CommandOptionsDescriptor <Project> options = null)
{
IRepositoryQuery <Project> query = new RepositoryQuery <Project>()
.SystemFilter(systemFilter)
.FilterExpression(userFilter);
query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending(p => p.Name.Suffix("keyword"));
return(FindAsync(q => query, options));
}
public Task <FindResults <Stack> > GetByFilterAsync(ExceptionlessSystemFilter systemFilter, string userFilter, string sort, string field, DateTime utcStart, DateTime utcEnd, CommandOptionsDescriptor <Stack> options = null)
{
IRepositoryQuery <Stack> query = new RepositoryQuery <Stack>()
.DateRange(utcStart, utcEnd, field ?? ElasticType.GetFieldName(s => s.LastOccurrence))
.SystemFilter(systemFilter)
.FilterExpression(userFilter);
query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortDescending(s => s.LastOccurrence);
return(FindAsync(q => query, options));
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write();
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext();
if (organization == null)
{
_logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id);
var sf = new ExceptionlessSystemFilter(project, organization);
var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime);
var result = await _eventRepository.CountBySearchAsync(systemFilter, $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error}", "terms:(is_first_occurrence @include:true) cardinality:stack_id").AnyContext();
bool hasSubmittedEvents = result.Total > 0;
if (!hasSubmittedEvents)
{
hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id, true).AnyContext() > 0;
}
double newTotal = result.Aggregations.Terms <double>("terms_is_first_occurrence")?.Buckets.FirstOrDefault()?.Total ?? 0;
double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
bool isFreePlan = organization.PlanId == BillingManager.FreePlan.Id;
foreach (var user in users)
{
_logger.Info().Project(project.Id).Message("Queueing \"{0}\" daily summary email ({1}-{2}) for user {3}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress);
await _mailer.SendProjectDailySummaryAsync(user, project, data.UtcStartTime, hasSubmittedEvents, result.Total, uniqueTotal, newTotal, isFreePlan).AnyContext();
}
_logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, result.Total);
return(true);
}
public Task <FindResults <PersistentEvent> > GetByFilterAsync(ExceptionlessSystemFilter systemFilter, string userFilter, string sort, string field, DateTime utcStart, DateTime utcEnd, CommandOptionsDescriptor <PersistentEvent> options = null)
{
IRepositoryQuery <PersistentEvent> query = new RepositoryQuery <PersistentEvent>()
.DateRange(utcStart, utcEnd, field ?? InferField(e => e.Date))
.Index(utcStart, utcEnd)
.SystemFilter(systemFilter)
.FilterExpression(userFilter);
query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortDescending(e => e.Date);
return(FindAsync(q => query, options));
}
public async Task <ActionResult <IReadOnlyCollection <ViewProject> > > GetByOrganizationAsync(string organizationId, string filter = null, string sort = null, int page = 1, int limit = 10, string mode = null)
{
var organization = await GetOrganizationAsync(organizationId);
if (organization == null)
{
return(NotFound());
}
page = GetPage(page);
limit = GetLimit(limit, 1000);
var sf = new ExceptionlessSystemFilter(organization);
var projects = await _repository.GetByFilterAsync(sf, filter, sort, o => o.PageNumber(page).PageLimit(limit));
var viewProjects = (await MapCollectionAsync <ViewProject>(projects.Documents, true)).ToList();
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "stats", StringComparison.OrdinalIgnoreCase))
{
return(OkWithResourceLinks(await PopulateProjectStatsAsync(viewProjects), projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total));
}
return(OkWithResourceLinks(viewProjects, projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total));
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
// TODO: Add slack daily summaries
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name);
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext();
if (organization == null)
{
_logger.LogInformation("The organization {organization} for project {ProjectName} may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.LogInformation("Sending daily summary: users={UserCount} project={project}", users.Count, project.Id);
var sf = new ExceptionlessSystemFilter(project, organization);
var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime);
string filter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:false";
var result = await _eventRepository.CountBySearchAsync(systemFilter, filter, "terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1").AnyContext();
double total = result.Aggregations.Sum("sum_count")?.Value ?? result.Total;
double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0;
double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault();
bool isFreePlan = organization.PlanId == _plans.FreePlan.Id;
string fixedFilter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:true";
var fixedResult = await _eventRepository.CountBySearchAsync(systemFilter, fixedFilter, "sum:count~1").AnyContext();
double fixedTotal = fixedResult.Aggregations.Sum("sum_count")?.Value ?? fixedResult.Total;
var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime);
var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList();
int blockedTotal = usages.Sum(u => u.Blocked);
int tooBigTotal = usages.Sum(u => u.TooBig);
IReadOnlyCollection <Stack> mostFrequent = null;
var stackTerms = result.Aggregations.Terms <string>("terms_stack_id");
if (stackTerms?.Buckets.Count > 0)
{
mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext();
}
IReadOnlyCollection <Stack> newest = null;
if (newTotal > 0)
{
newest = (await _stackRepository.GetByFilterAsync(sf, filter, "-first", "first", data.UtcStartTime, data.UtcEndTime, o => o.PageLimit(3)).AnyContext()).Documents;
}
foreach (var user in users)
{
_logger.LogInformation("Queuing {ProjectName} daily summary email ({UtcStartTime}-{UtcEndTime}) for user {EmailAddress}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress);
await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext();
}
_logger.LogInformation("Done sending daily summary: users={UserCount} project={ProjectName} events={EventCount}", users.Count, project.Name, total);
return(true);
}
public async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(PersistentEvent ev, ExceptionlessSystemFilter systemFilter, string userFilter, DateTime?utcStart, DateTime?utcEnd)
{
var previous = GetPreviousEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd);
var next = GetNextEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd);
await Task.WhenAll(previous, next).AnyContext();
return(new PreviousAndNextEventIdResult {
Previous = previous.Result,
Next = next.Result
});
}
public async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(PersistentEvent ev, ExceptionlessSystemFilter systemFilter, string userFilter, DateTime?utcStart, DateTime?utcEnd)
{
string previous = await GetPreviousEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext();
string next = await GetNextEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext();
return(new PreviousAndNextEventIdResult {
Previous = previous,
Next = next
});
}
public static async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(this IEventRepository repository, string id, ExceptionlessSystemFilter systemFilter = null, string userFilter = null, DateTime?utcStart = null, DateTime?utcEnd = null)
{
var ev = await repository.GetByIdAsync(id, o => o.Cache()).AnyContext();
return(await repository.GetPreviousAndNextEventIdsAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext());
}
