private async Task <string> GetNextEventIdAsync(PersistentEvent ev, ExceptionlessSystemFilter systemFilter = null, string userFilter = null, DateTime?utcStart = null, DateTime?utcEnd = null) { if (ev == null) { return(null); } if (!utcStart.HasValue || utcStart.Value.IsBefore(ev.Date.UtcDateTime)) { utcStart = ev.Date.UtcDateTime; } if (!utcEnd.HasValue || utcEnd.Value.IsAfter(SystemClock.UtcNow)) { utcEnd = SystemClock.UtcNow; } var utcEventDate = ev.Date.UtcDateTime; // utcEnd is before the current event date. if (utcStart > utcEventDate || utcEnd < utcEventDate) { return(null); } if (String.IsNullOrEmpty(userFilter)) { userFilter = String.Concat(EventIndexType.Alias.StackId, ":", ev.StackId); } var results = await FindAsync(q => q .DateRange(utcEventDate, utcEnd, (PersistentEvent e) => e.Date) .Index(utcEventDate, utcEnd) .SortAscending(e => e.Date) .Include(e => e.Id, e => e.Date) .SystemFilter(systemFilter) .ElasticFilter(!Query <PersistentEvent> .Ids(ids => ids.Values(ev.Id))) .FilterExpression(userFilter), o => o.PageLimit(10)).AnyContext(); if (results.Total == 0) { return(null); } // make sure we don't have records with the exact same occurrence date if (results.Documents.All(t => t.Date != ev.Date)) { return(results.Documents.OrderBy(t => t.Date).ThenBy(t => t.Id).First().Id); } // we have records with the exact same occurrence date, we need to figure out the order of those // put our target error into the mix, sort it and return the result after the target var unionResults = results.Documents.Union(new[] { ev }) .OrderBy(t => t.Date.Ticks).ThenBy(t => t.Id) .ToList(); int index = unionResults.FindIndex(t => t.Id == ev.Id); return(index == unionResults.Count - 1 ? null : unionResults[index + 1].Id); }
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> viewProjects) { if (viewProjects.Count <= 0) { return(viewProjects); } var organizations = await _organizationRepository.GetByIdsAsync(viewProjects.Select(p => p.OrganizationId).ToArray(), o => o.Cache()); var projects = viewProjects.Select(p => new Project { Id = p.Id, CreatedUtc = p.CreatedUtc, OrganizationId = p.OrganizationId }).ToList(); var sf = new ExceptionlessSystemFilter(projects, organizations); var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow, (PersistentEvent e) => e.Date).Index(organizations.GetRetentionUtcCutoff(), SystemClock.UtcNow); var result = await _eventRepository.CountBySearchAsync(systemFilter, null, $"terms:(project_id~{viewProjects.Count} cardinality:stack_id)"); foreach (var project in viewProjects) { var term = result.Aggregations.Terms <string>("terms_project_id")?.Buckets.FirstOrDefault(t => t.Key == project.Id); project.EventCount = term?.Total ?? 0; project.StackCount = (long)(term?.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0); } return(viewProjects); }
protected async Task <ActionResult <CountResult> > GetCountImplAsync(ExceptionlessSystemFilter sf, TimeInfo ti, string filter = null, string aggregations = null) { var pr = await _validator.ValidateQueryAsync(filter); if (!pr.IsValid) { return(BadRequest(pr.Message)); } var far = await _validator.ValidateAggregationsAsync(aggregations); if (!far.IsValid) { return(BadRequest(far.Message)); } sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || far.UsesPremiumFeatures; var query = new RepositoryQuery <TModel>() .SystemFilter(ShouldApplySystemFilter(sf, filter) ? sf : null) .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field) .Index(ti.Range.UtcStart, ti.Range.UtcEnd); CountResult result; try { result = await _repository.CountBySearchAsync(query, filter, aggregations); } catch (Exception ex) { using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Aggregations = aggregations }).Tag("Search").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext))) _logger.LogError(ex, "An error has occurred. Please check your filter or aggregations."); return(BadRequest("An error has occurred. Please check your search filter.")); } return(Ok(result)); }
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> viewOrganizations) { if (viewOrganizations.Count <= 0) { return(viewOrganizations); } int maximumRetentionDays = _options.Value.MaximumRetentionDays; var organizations = viewOrganizations.Select(o => new Organization { Id = o.Id, CreatedUtc = o.CreatedUtc, RetentionDays = o.RetentionDays }).ToList(); var sf = new ExceptionlessSystemFilter(organizations); var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(organizations.GetRetentionUtcCutoff(maximumRetentionDays), SystemClock.UtcNow, (PersistentEvent e) => e.Date).Index(organizations.GetRetentionUtcCutoff(maximumRetentionDays), SystemClock.UtcNow); var result = await _eventRepository.CountBySearchAsync(systemFilter, null, $"terms:(organization_id~{viewOrganizations.Count} cardinality:stack_id)"); foreach (var organization in viewOrganizations) { var organizationStats = result.Aggregations.Terms <string>("terms_organization_id")?.Buckets.FirstOrDefault(t => t.Key == organization.Id); organization.EventCount = organizationStats?.Total ?? 0; organization.StackCount = (long?)organizationStats?.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0; organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id); } return(viewOrganizations); }
public async Task <ActionResult <ViewProject> > GetAsync(string filter = null, string sort = null, int page = 1, int limit = 10, string mode = null) { var organizations = await GetSelectedOrganizationsAsync(_organizationRepository, _projectRepository, _stackRepository, filter); if (organizations.Count == 0) { return(Ok(EmptyModels)); } page = GetPage(page); limit = GetLimit(limit, 1000); var sf = new ExceptionlessSystemFilter(organizations) { IsUserOrganizationsFilter = true }; var projects = await _repository.GetByFilterAsync(sf, filter, sort, o => o.PageNumber(page).PageLimit(limit)); var viewProjects = await MapCollectionAsync <ViewProject>(projects.Documents, true); if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "stats", StringComparison.OrdinalIgnoreCase)) { return(OkWithResourceLinks(await PopulateProjectStatsAsync(viewProjects.ToList()), projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total)); } return(OkWithResourceLinks(viewProjects, projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total)); }
public static T SystemFilter <T>(this T query, ExceptionlessSystemFilter filter) where T : IRepositoryQuery { if (filter != null) { return(query.BuildOption(SystemFilterKey, filter)); } return(query); }
public Task <FindResults <Project> > GetByFilterAsync(ExceptionlessSystemFilter systemFilter, string userFilter, string sort, CommandOptionsDescriptor <Project> options = null) { IRepositoryQuery <Project> query = new RepositoryQuery <Project>() .SystemFilter(systemFilter) .FilterExpression(userFilter); query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending(p => p.Name.Suffix("keyword")); return(FindAsync(q => query, options)); }
public Task <FindResults <Stack> > GetByFilterAsync(ExceptionlessSystemFilter systemFilter, string userFilter, string sort, string field, DateTime utcStart, DateTime utcEnd, CommandOptionsDescriptor <Stack> options = null) { IRepositoryQuery <Stack> query = new RepositoryQuery <Stack>() .DateRange(utcStart, utcEnd, field ?? ElasticType.GetFieldName(s => s.LastOccurrence)) .SystemFilter(systemFilter) .FilterExpression(userFilter); query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortDescending(s => s.LastOccurrence); return(FindAsync(q => query, options)); }
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data) { var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write(); return(false); } var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext(); var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList(); if (users.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name); return(false); } // TODO: What should we do about suspended organizations. var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { _logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name); return(false); } _logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id); var sf = new ExceptionlessSystemFilter(project, organization); var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime); var result = await _eventRepository.CountBySearchAsync(systemFilter, $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error}", "terms:(is_first_occurrence @include:true) cardinality:stack_id").AnyContext(); bool hasSubmittedEvents = result.Total > 0; if (!hasSubmittedEvents) { hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id, true).AnyContext() > 0; } double newTotal = result.Aggregations.Terms <double>("terms_is_first_occurrence")?.Buckets.FirstOrDefault()?.Total ?? 0; double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0; bool isFreePlan = organization.PlanId == BillingManager.FreePlan.Id; foreach (var user in users) { _logger.Info().Project(project.Id).Message("Queueing \"{0}\" daily summary email ({1}-{2}) for user {3}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress); await _mailer.SendProjectDailySummaryAsync(user, project, data.UtcStartTime, hasSubmittedEvents, result.Total, uniqueTotal, newTotal, isFreePlan).AnyContext(); } _logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, result.Total); return(true); }
public Task <FindResults <PersistentEvent> > GetByFilterAsync(ExceptionlessSystemFilter systemFilter, string userFilter, string sort, string field, DateTime utcStart, DateTime utcEnd, CommandOptionsDescriptor <PersistentEvent> options = null) { IRepositoryQuery <PersistentEvent> query = new RepositoryQuery <PersistentEvent>() .DateRange(utcStart, utcEnd, field ?? InferField(e => e.Date)) .Index(utcStart, utcEnd) .SystemFilter(systemFilter) .FilterExpression(userFilter); query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortDescending(e => e.Date); return(FindAsync(q => query, options)); }
public async Task <ActionResult <IReadOnlyCollection <ViewProject> > > GetByOrganizationAsync(string organizationId, string filter = null, string sort = null, int page = 1, int limit = 10, string mode = null) { var organization = await GetOrganizationAsync(organizationId); if (organization == null) { return(NotFound()); } page = GetPage(page); limit = GetLimit(limit, 1000); var sf = new ExceptionlessSystemFilter(organization); var projects = await _repository.GetByFilterAsync(sf, filter, sort, o => o.PageNumber(page).PageLimit(limit)); var viewProjects = (await MapCollectionAsync <ViewProject>(projects.Documents, true)).ToList(); if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "stats", StringComparison.OrdinalIgnoreCase)) { return(OkWithResourceLinks(await PopulateProjectStatsAsync(viewProjects), projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total)); } return(OkWithResourceLinks(viewProjects, projects.HasMore && !NextPageExceedsSkipLimit(page, limit), page, projects.Total)); }
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data) { // TODO: Add slack daily summaries var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name); return(false); } var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext(); var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList(); if (users.Count == 0) { _logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name); return(false); } // TODO: What should we do about suspended organizations. var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { _logger.LogInformation("The organization {organization} for project {ProjectName} may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name); return(false); } _logger.LogInformation("Sending daily summary: users={UserCount} project={project}", users.Count, project.Id); var sf = new ExceptionlessSystemFilter(project, organization); var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime); string filter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:false"; var result = await _eventRepository.CountBySearchAsync(systemFilter, filter, "terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1").AnyContext(); double total = result.Aggregations.Sum("sum_count")?.Value ?? result.Total; double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0; double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0; bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault(); bool isFreePlan = organization.PlanId == _plans.FreePlan.Id; string fixedFilter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:true"; var fixedResult = await _eventRepository.CountBySearchAsync(systemFilter, fixedFilter, "sum:count~1").AnyContext(); double fixedTotal = fixedResult.Aggregations.Sum("sum_count")?.Value ?? fixedResult.Total; var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime); var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList(); int blockedTotal = usages.Sum(u => u.Blocked); int tooBigTotal = usages.Sum(u => u.TooBig); IReadOnlyCollection <Stack> mostFrequent = null; var stackTerms = result.Aggregations.Terms <string>("terms_stack_id"); if (stackTerms?.Buckets.Count > 0) { mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext(); } IReadOnlyCollection <Stack> newest = null; if (newTotal > 0) { newest = (await _stackRepository.GetByFilterAsync(sf, filter, "-first", "first", data.UtcStartTime, data.UtcEndTime, o => o.PageLimit(3)).AnyContext()).Documents; } foreach (var user in users) { _logger.LogInformation("Queuing {ProjectName} daily summary email ({UtcStartTime}-{UtcEndTime}) for user {EmailAddress}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress); await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext(); } _logger.LogInformation("Done sending daily summary: users={UserCount} project={ProjectName} events={EventCount}", users.Count, project.Name, total); return(true); }
public async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(PersistentEvent ev, ExceptionlessSystemFilter systemFilter, string userFilter, DateTime?utcStart, DateTime?utcEnd) { var previous = GetPreviousEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd); var next = GetNextEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd); await Task.WhenAll(previous, next).AnyContext(); return(new PreviousAndNextEventIdResult { Previous = previous.Result, Next = next.Result }); }
public async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(PersistentEvent ev, ExceptionlessSystemFilter systemFilter, string userFilter, DateTime?utcStart, DateTime?utcEnd) { string previous = await GetPreviousEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext(); string next = await GetNextEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext(); return(new PreviousAndNextEventIdResult { Previous = previous, Next = next }); }
public static async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(this IEventRepository repository, string id, ExceptionlessSystemFilter systemFilter = null, string userFilter = null, DateTime?utcStart = null, DateTime?utcEnd = null) { var ev = await repository.GetByIdAsync(id, o => o.Cache()).AnyContext(); return(await repository.GetPreviousAndNextEventIdsAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext()); }