protected override Task <JobResult> RunInternalAsync(CancellationToken token) { Log.Info().Message("Daily Notification job starting").Write(); if (!Settings.Current.EnableSummaryNotifications) { return(Task.FromResult(new JobResult { Message = "Summary Notifications are disabled.", IsCancelled = true })); } const int BATCH_SIZE = 25; // Get all project id's that should be sent at 9:00am in the projects local time. var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); while (projects.Count > 0) { var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList()); Log.Info().Message("Daily Notification job processing {0} projects. Successfully updated {1} projects. ", projects.Count, documentsUpdated); Debug.Assert(projects.Count == documentsUpdated); foreach (var project in projects) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) { Log.Info().Message("Skipping Summary Notification older than two days for Project: {0} with a start time of {1}.", project.Id, utcStartTime); continue; } if (_mailer != null) { var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; Log.Info().Message("Publishing Summary Notification for Project: {0}, with a start time of {1} and an end time of {2}", notification.Id, notification.UtcStartTime, notification.UtcEndTime); ProcessSummaryNotification(notification); } else { Log.Error().Message("Mailer is null").Write(); } } projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); } return(Task.FromResult(new JobResult { Message = "Successfully sent summary notifications." })); }
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data) { var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write(); return(false); } var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext(); var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList(); if (users.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name); return(false); } // TODO: What should we do about suspended organizations. var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { _logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name); return(false); } _logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id); var sf = new ExceptionlessSystemFilter(project, organization); var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime); var result = await _eventRepository.CountBySearchAsync(systemFilter, $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error}", "terms:(is_first_occurrence @include:true) cardinality:stack_id").AnyContext(); bool hasSubmittedEvents = result.Total > 0; if (!hasSubmittedEvents) { hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id, true).AnyContext() > 0; } double newTotal = result.Aggregations.Terms <double>("terms_is_first_occurrence")?.Buckets.FirstOrDefault()?.Total ?? 0; double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0; bool isFreePlan = organization.PlanId == BillingManager.FreePlan.Id; foreach (var user in users) { _logger.Info().Project(project.Id).Message("Queueing \"{0}\" daily summary email ({1}-{2}) for user {3}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress); await _mailer.SendProjectDailySummaryAsync(user, project, data.UtcStartTime, hasSubmittedEvents, result.Total, uniqueTotal, newTotal, isFreePlan).AnyContext(); } _logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, result.Total); return(true); }
protected override async Task <JobResult> RunInternalAsync(JobContext context) { if (!Settings.Current.EnableDailySummary) { return(JobResult.SuccessWithMessage("Summary notifications are disabled.")); } if (_mailer == null) { return(JobResult.SuccessWithMessage("Summary notifications are disabled due to null mailer.")); } const int BATCH_SIZE = 25; var projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents; while (projects.Count > 0 && !context.CancellationToken.IsCancellationRequested) { var documentsUpdated = await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projects).AnyContext(); _logger.Info("Got {0} projects to process. ", projects.Count); Debug.Assert(projects.Count == documentsUpdated); foreach (var project in projects) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) { _logger.Info("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime); continue; } var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; await ProcessSummaryNotificationAsync(notification).AnyContext(); // Sleep so were not hammering the database. await Task.Delay(TimeSpan.FromSeconds(1)); } projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents; if (projects.Count > 0) { await context.RenewLockAsync().AnyContext(); } } return(JobResult.SuccessWithMessage("Successfully sent summary notifications.")); }
private async Task ProcessSummaryNotificationAsync(SummaryNotification data) { var project = await _projectRepository.GetByIdAsync(data.Id, true).AnyContext(); var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext(); var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write(); return; } var users = (await _userRepository.GetByIdsAsync(userIds).AnyContext()).Documents.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList(); if (users.Count == 0) { _logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write(); return; } _logger.Info().Message("Sending daily summary: users={0} project={1}", users.Count, project.Id).Write(); var result = await _stats.GetTermsStatsAsync(data.UtcStartTime, data.UtcEndTime, "stack_id", "type:error project:" + data.Id, max : 5).AnyContext(); bool hasSubmittedErrors = result.Total > 0; if (!hasSubmittedErrors) { hasSubmittedErrors = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0; } var notification = new DailySummaryModel { ProjectId = project.Id, ProjectName = project.Name, StartDate = data.UtcStartTime, EndDate = data.UtcEndTime, Total = result.Total, PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours, NewTotal = result.New, UniqueTotal = result.Unique, HasSubmittedEvents = hasSubmittedErrors, IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id }; foreach (var user in users) { await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext(); } _logger.Info().Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total).Write(); }
protected override Task <JobResult> RunInternalAsync(CancellationToken token) { if (!Settings.Current.EnableDailySummary) { return(Task.FromResult(JobResult.SuccessWithMessage("Summary notifications are disabled."))); } const int BATCH_SIZE = 25; var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); while (projects.Count > 0 && !token.IsCancellationRequested) { var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList()); Log.Info().Message("Got {0} projects to process. ", projects.Count).Write(); Debug.Assert(projects.Count == documentsUpdated); foreach (var project in projects) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) { Log.Info().Message("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime).Write(); continue; } if (_mailer != null) { var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; ProcessSummaryNotification(notification); } else { Log.Error().Message("Mailer is null").Write(); } } projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); } return(Task.FromResult(new JobResult { Message = "Successfully sent summary notifications." })); }
protected override Task<JobResult> RunInternalAsync() { Log.Info().Message("Daily Notification job starting").Write(); if (!Settings.Current.EnableSummaryNotifications) return Task.FromResult(new JobResult { Message = "Summary Notifications are disabled.", IsCancelled = true }); const int BATCH_SIZE = 25; // Get all project id's that should be sent at 9:00am in the projects local time. var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); while (projects.Count > 0) { var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList()); Log.Info().Message("Daily Notification job processing {0} projects. Successfully updated {1} projects. ", projects.Count, documentsUpdated); Debug.Assert(projects.Count == documentsUpdated); foreach (var project in projects) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) { Log.Info().Message("Skipping Summary Notification older than two days for Project: {0} with a start time of {1}.", project.Id, utcStartTime); continue; } if (_summaryNotificationQueue != null) { var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; Log.Info().Message("Publishing Summary Notification for Project: {0}, with a start time of {1} and an end time of {2}", notification.Id, notification.UtcStartTime, notification.UtcEndTime); _summaryNotificationQueue.EnqueueAsync(notification); } else Log.Error().Message("Message Factory is null").Write(); } projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); } return Task.FromResult(new JobResult { Message = "Successfully enforced all retention limits." }); }
protected override async Task <JobResult> RunInternalAsync(JobContext context) { _lastRun = SystemClock.UtcNow; if (!_emailOptions.EnableDailySummary || _mailer == null) { return(JobResult.SuccessWithMessage("Summary notifications are disabled.")); } var results = await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { _logger.LogTrace("Got {Count} projects to process. ", results.Documents.Count); var projectsToBulkUpdate = new List <Project>(results.Documents.Count); var processSummariesNewerThan = SystemClock.UtcNow.Date.SubtractDays(2); foreach (var project in results.Documents) { using (_logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id))) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < processSummariesNewerThan) { _logger.LogInformation("Skipping daily summary older than two days for project: {Name}", project.Name); projectsToBulkUpdate.Add(project); continue; } var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; bool summarySent = await SendSummaryNotificationAsync(project, notification).AnyContext(); if (summarySent) { await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(new[] { project }).AnyContext(); // Sleep so we are not hammering the backend as we just generated a report. await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext(); } else { projectsToBulkUpdate.Add(project); } } } if (projectsToBulkUpdate.Count > 0) { await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projectsToBulkUpdate).AnyContext(); // Sleep so we are not hammering the backend await SystemClock.SleepAsync(TimeSpan.FromSeconds(1)).AnyContext(); } if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext()) { break; } if (results.Documents.Count > 0) { await context.RenewLockAsync().AnyContext(); _lastRun = SystemClock.UtcNow; } } return(JobResult.SuccessWithMessage("Successfully sent summary notifications.")); }
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data) { // TODO: Add slack daily summaries var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name); return(false); } var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext(); var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList(); if (users.Count == 0) { _logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name); return(false); } // TODO: What should we do about suspended organizations. var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { _logger.LogInformation("The organization {organization} for project {ProjectName} may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name); return(false); } _logger.LogInformation("Sending daily summary: users={UserCount} project={project}", users.Count, project.Id); var sf = new AppFilter(project, organization); var systemFilter = new RepositoryQuery <PersistentEvent>().AppFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime); string filter = "type:error (status:open OR status:regressed)"; var result = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression("terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1")).AnyContext(); double total = result.Aggregations.Sum("sum_count")?.Value ?? result.Total; double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0; double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0; bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault(); bool isFreePlan = organization.PlanId == _plans.FreePlan.Id; string fixedFilter = "type:error status:fixed"; var fixedResult = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(fixedFilter).EnforceEventStackFilter().AggregationsExpression("sum:count~1")).AnyContext(); double fixedTotal = fixedResult.Aggregations.Sum("sum_count")?.Value ?? fixedResult.Total; var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime); var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList(); int blockedTotal = usages.Sum(u => u.Blocked); int tooBigTotal = usages.Sum(u => u.TooBig); IReadOnlyCollection <Stack> mostFrequent = null; var stackTerms = result.Aggregations.Terms <string>("terms_stack_id"); if (stackTerms?.Buckets.Count > 0) { mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext(); } IReadOnlyCollection <Stack> newest = null; if (newTotal > 0) { newest = (await _stackRepository.FindAsync(q => q.AppFilter(sf).FilterExpression(filter).SortExpression("-first").DateRange(data.UtcStartTime, data.UtcEndTime, "first"), o => o.PageLimit(3)).AnyContext()).Documents; } foreach (var user in users) { _logger.LogInformation("Queuing {ProjectName} daily summary email ({UtcStartTime}-{UtcEndTime}) for user {EmailAddress}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress); await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext(); } _logger.LogInformation("Done sending daily summary: users={UserCount} project={ProjectName} events={EventCount}", users.Count, project.Name, total); return(true); }
private async Task ProcessSummaryNotificationAsync(SummaryNotification data) { var project = await _projectRepository.GetByIdAsync(data.Id, true).AnyContext(); var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext(); var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.Info("Project \"{0}\" has no users to send summary to.", project.Id); return; } var users = (await _userRepository.GetByIdsAsync(userIds).AnyContext()).Documents.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList(); if (users.Count == 0) { _logger.Info("Project \"{0}\" has no users to send summary to.", project.Id); return; } _logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id); var fields = new List <FieldAggregation> { new FieldAggregation { Type = FieldAggregationType.Distinct, Field = "stack_id" }, new TermFieldAggregation { Field = "is_first_occurrence", ExcludePattern = "F" } }; var result = await _stats.GetNumbersStatsAsync(fields, data.UtcStartTime, data.UtcEndTime, $"project:{data.Id} type:error").AnyContext(); bool hasSubmittedEvents = result.Total > 0; if (!hasSubmittedEvents) { hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0; } var notification = new DailySummaryModel { ProjectId = project.Id, ProjectName = project.Name, StartDate = data.UtcStartTime, EndDate = data.UtcEndTime, Total = result.Total, PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours, NewTotal = result.Numbers[1], UniqueTotal = result.Numbers[0], HasSubmittedEvents = hasSubmittedEvents, IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id }; foreach (var user in users) { await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext(); } _logger.Info("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total); }
private void ProcessSummaryNotification(SummaryNotification data) { var project = _projectRepository.GetById(data.Id, true); var organization = _organizationRepository.GetById(project.OrganizationId, true); var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList(); if (userIds.Count == 0) { return; } var users = _userRepository.GetByIds(userIds).Where(u => u.IsEmailAddressVerified).ToList(); if (users.Count == 0) { return; } long count; var paging = new PagingOptions { Limit = 5 }; List <Stack> newest = _stackRepository.GetNew(project.Id, data.UtcStartTime, data.UtcEndTime, paging).ToList(); var result = _stats.GetTermsStats(data.UtcStartTime, data.UtcEndTime, "stack_id", "project:" + data.Id, max: 5); var mostFrequent = result.Terms.Take(5).ToList(); var stacks = _stackRepository.GetByIds(mostFrequent.Select(s => s.Term).ToList()); foreach (var frequent in mostFrequent) { var stack = stacks.SingleOrDefault(s => s.Id == frequent.Term); if (stack == null) { mostFrequent.RemoveAll(r => r.Term == frequent.Term); continue; } // Stat's Id and Total properties are already calculated in the Results. //frequent.Type = stack.SignatureInfo.ContainsKey("ExceptionType") ? stack.SignatureInfo["ExceptionType"] : null; //frequent.Method = stack.SignatureInfo.ContainsKey("Method") ? stack.SignatureInfo["Method"] : null; //frequent.Path = stack.SignatureInfo.ContainsKey("Path") ? stack.SignatureInfo["Path"] : null; //frequent.Is404 = stack.SignatureInfo.ContainsKey("Path"); //frequent.Title = stack.Title; //frequent.First = stack.FirstOccurrence; //frequent.Last = stack.LastOccurrence; } var notification = new SummaryNotificationModel { ProjectId = project.Id, ProjectName = project.Name, StartDate = data.UtcStartTime, EndDate = data.UtcEndTime, //Total = result.Total, //PerHourAverage = result.PerHourAverage, //NewTotal = result.NewTotal, //New = newest, //UniqueTotal = result.UniqueTotal, //MostFrequent = mostFrequent, //HasSubmittedErrors = project.TotalErrorCount > 0, IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id }; foreach (var user in users.Where(u => u.EmailNotificationsEnabled)) { _mailer.SendSummaryNotification(user.EmailAddress, notification); } }
private void ProcessSummaryNotification(SummaryNotification data) { var project = _projectRepository.GetById(data.Id, true); var organization = _organizationRepository.GetById(project.OrganizationId, true); var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList(); if (userIds.Count == 0) { Log.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write(); return; } var users = _userRepository.GetByIds(userIds).Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList(); if (users.Count == 0) { Log.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write(); return; } Log.Info().Message("Sending daily summary: users={0} project={1}", users.Count, project.Id).Write(); var paging = new PagingOptions { Limit = 5 }; List <Stack> newest = _stackRepository.GetNew(project.Id, data.UtcStartTime, data.UtcEndTime, paging).ToList(); var result = _stats.GetTermsStats(data.UtcStartTime, data.UtcEndTime, "stack_id", "type:error project:" + data.Id, max: 5); //var termStatsList = result.Terms.Take(5).ToList(); //var stacks = _stackRepository.GetByIds(termStatsList.Select(s => s.Term).ToList()); bool hasSubmittedErrors = result.Total > 0; if (!hasSubmittedErrors) { hasSubmittedErrors = _eventRepository.GetCountByProjectId(project.Id) > 0; } var mostFrequent = new List <EventStackResult>(); //foreach (var termStats in termStatsList) { // var stack = stacks.SingleOrDefault(s => s.Id == termStats.Term); // if (stack == null) // continue; // mostFrequent.Add(new EventStackResult { // First = termStats.FirstOccurrence, // Last = termStats.LastOccurrence, // Id = stack.Id, // Title = stack.Title, // Total = termStats.Total, // Type = stack.SignatureInfo.ContainsKey("ExceptionType") ? stack.SignatureInfo["ExceptionType"] : null, // Method = stack.SignatureInfo.ContainsKey("Method") ? stack.SignatureInfo["Method"] : null, // Path = stack.SignatureInfo.ContainsKey("Source") ? stack.SignatureInfo["Source"] : null, // Is404 = stack.SignatureInfo.ContainsKey("Type") && stack.SignatureInfo["Type"] == "404" // }); //} var notification = new DailySummaryModel { ProjectId = project.Id, ProjectName = project.Name, StartDate = data.UtcStartTime, EndDate = data.UtcEndTime, Total = result.Total, PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours, NewTotal = result.New, New = newest, UniqueTotal = result.Unique, MostFrequent = mostFrequent, HasSubmittedEvents = hasSubmittedErrors, IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id }; foreach (var user in users) { _mailer.SendDailySummary(user.EmailAddress, notification); } Log.Info().Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total).Write(); }
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data) { var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write(); return(false); } var results = await _userRepository.GetByIdsAsync(userIds, true).AnyContext(); var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList(); if (users.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name); return(false); } // TODO: What should we do about suspended organizations. var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext(); if (organization == null) { _logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name); return(false); } _logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id); var fields = new List <FieldAggregation> { new FieldAggregation { Type = FieldAggregationType.Distinct, Field = "stack_id" }, new TermFieldAggregation { Field = "is_first_occurrence", ExcludePattern = "F" } }; var sf = new ExceptionlessSystemFilterQuery(project, organization); var result = await _stats.GetNumbersStatsAsync(fields, data.UtcStartTime, data.UtcEndTime, sf, $"{EventIndexType.Fields.Type}:{Event.KnownTypes.Error}").AnyContext(); bool hasSubmittedEvents = result.Total > 0; if (!hasSubmittedEvents) { hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0; } var notification = new DailySummaryModel { ProjectId = project.Id, ProjectName = project.Name, StartDate = data.UtcStartTime, EndDate = data.UtcEndTime, Total = result.Total, PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours, NewTotal = result.Numbers[1], UniqueTotal = result.Numbers[0], HasSubmittedEvents = hasSubmittedEvents, IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id }; foreach (var user in users) { await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext(); } _logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, notification.Total); return(true); }
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data) { // TODO: Add slack daily summaries var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write(); return(false); } var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext(); var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList(); if (users.Count == 0) { _logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name); return(false); } // TODO: What should we do about suspended organizations. var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { _logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name); return(false); } _logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id); var sf = new ExceptionlessSystemFilter(project, organization); var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime); string filter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:false"; var result = await _eventRepository.CountBySearchAsync(systemFilter, filter, "terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1").AnyContext(); double total = result.Aggregations.Sum("sum_count").Value ?? result.Total; double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0; double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0; bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault(); bool isFreePlan = organization.PlanId == BillingManager.FreePlan.Id; string fixedFilter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:true"; var fixedResult = await _eventRepository.CountBySearchAsync(systemFilter, fixedFilter, "sum:count~1").AnyContext(); double fixedTotal = fixedResult.Aggregations.Sum("sum_count").Value ?? fixedResult.Total; var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime); var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList(); int blockedTotal = usages.Sum(u => u.Blocked); int tooBigTotal = usages.Sum(u => u.TooBig); IReadOnlyCollection <Stack> mostFrequent = null; var stackTerms = result.Aggregations.Terms <string>("terms_stack_id"); if (stackTerms?.Buckets.Count > 0) { mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext(); } IReadOnlyCollection <Stack> newest = null; if (newTotal > 0) { newest = (await _stackRepository.GetByFilterAsync(sf, filter, "-first", "first", data.UtcStartTime, data.UtcEndTime, o => o.PageLimit(3)).AnyContext()).Documents; } foreach (var user in users) { _logger.Info().Project(project.Id).Message("Queuing \"{0}\" daily summary email ({1}-{2}) for user {3}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress); await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext(); } _logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, total); return(true); }
public override JobResult Run(JobContext context) { Log.Info().Message("Daily Notification job starting").Write(); if (!Settings.Current.EnableSummaryNotifications) { return(new JobResult { Result = "Summary Notifications are disabled.", Cancelled = true }); } const int BATCH_SIZE = 25; // Send an email at 9:00am in the projects local time. IMongoQuery query = Query.LT(ProjectRepository.FieldNames.NextSummaryEndOfDayTicks, new BsonInt64(DateTime.UtcNow.Ticks - (TimeSpan.TicksPerHour * 9))); UpdateBuilder update = Update.Inc(ProjectRepository.FieldNames.NextSummaryEndOfDayTicks, TimeSpan.TicksPerDay); var projects = _projectRepository.Collection.FindAs <Project>(query).SetFields(ProjectRepository.FieldNames.Id, ProjectRepository.FieldNames.NextSummaryEndOfDayTicks).SetLimit(BATCH_SIZE).ToList(); while (projects.Count > 0) { IMongoQuery queryWithProjectIds = Query.And(Query.In(ProjectRepository.FieldNames.Id, projects.Select(p => new BsonObjectId(new ObjectId(p.Id)))), query); var result = _projectRepository.Collection.Update(queryWithProjectIds, update, UpdateFlags.Multi); Log.Info().Message("Daily Notification job processing {0} projects. Successfully updated {1} projects. ", projects.Count, result.DocumentsAffected); Debug.Assert(projects.Count == result.DocumentsAffected); foreach (var project in projects) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) { Log.Info().Message("Skipping Summary Notification older than two days for Project: {0} with a start time of {1}.", project.Id, utcStartTime); continue; } if (_messageFactory != null) { using (IMessageProducer messageProducer = _messageFactory.CreateMessageProducer()) { var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; Log.Info().Message("Publishing Summary Notification for Project: {0}, with a start time of {1} and an end time of {2}", notification.Id, notification.UtcStartTime, notification.UtcEndTime); messageProducer.Publish(notification); } } else { Log.Error().Message("Message Factory is null").Write(); } } projects = _projectRepository.Collection.FindAs <Project>(query).SetFields(ProjectRepository.FieldNames.Id, ProjectRepository.FieldNames.NextSummaryEndOfDayTicks).SetLimit(BATCH_SIZE).ToList(); } return(new JobResult { Result = "Successfully enforced all retention limits." }); }