protected override Task <JobResult> RunInternalAsync(CancellationToken token)
{
Log.Info().Message("Daily Notification job starting").Write();
if (!Settings.Current.EnableSummaryNotifications)
{
return(Task.FromResult(new JobResult {
Message = "Summary Notifications are disabled.", IsCancelled = true
}));
}
const int BATCH_SIZE = 25;
// Get all project id's that should be sent at 9:00am in the projects local time.
var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
while (projects.Count > 0)
{
var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList());
Log.Info().Message("Daily Notification job processing {0} projects. Successfully updated {1} projects. ", projects.Count, documentsUpdated);
Debug.Assert(projects.Count == documentsUpdated);
foreach (var project in projects)
{
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2))
{
Log.Info().Message("Skipping Summary Notification older than two days for Project: {0} with a start time of {1}.", project.Id, utcStartTime);
continue;
}
if (_mailer != null)
{
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
Log.Info().Message("Publishing Summary Notification for Project: {0}, with a start time of {1} and an end time of {2}", notification.Id, notification.UtcStartTime, notification.UtcEndTime);
ProcessSummaryNotification(notification);
}
else
{
Log.Error().Message("Mailer is null").Write();
}
}
projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
}
return(Task.FromResult(new JobResult {
Message = "Successfully sent summary notifications."
}));
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write();
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext();
if (organization == null)
{
_logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id);
var sf = new ExceptionlessSystemFilter(project, organization);
var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime);
var result = await _eventRepository.CountBySearchAsync(systemFilter, $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error}", "terms:(is_first_occurrence @include:true) cardinality:stack_id").AnyContext();
bool hasSubmittedEvents = result.Total > 0;
if (!hasSubmittedEvents)
{
hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id, true).AnyContext() > 0;
}
double newTotal = result.Aggregations.Terms <double>("terms_is_first_occurrence")?.Buckets.FirstOrDefault()?.Total ?? 0;
double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
bool isFreePlan = organization.PlanId == BillingManager.FreePlan.Id;
foreach (var user in users)
{
_logger.Info().Project(project.Id).Message("Queueing \"{0}\" daily summary email ({1}-{2}) for user {3}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress);
await _mailer.SendProjectDailySummaryAsync(user, project, data.UtcStartTime, hasSubmittedEvents, result.Total, uniqueTotal, newTotal, isFreePlan).AnyContext();
}
_logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, result.Total);
return(true);
}
protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
if (!Settings.Current.EnableDailySummary)
{
return(JobResult.SuccessWithMessage("Summary notifications are disabled."));
}
if (_mailer == null)
{
return(JobResult.SuccessWithMessage("Summary notifications are disabled due to null mailer."));
}
const int BATCH_SIZE = 25;
var projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents;
while (projects.Count > 0 && !context.CancellationToken.IsCancellationRequested)
{
var documentsUpdated = await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projects).AnyContext();
_logger.Info("Got {0} projects to process. ", projects.Count);
Debug.Assert(projects.Count == documentsUpdated);
foreach (var project in projects)
{
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2))
{
_logger.Info("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime);
continue;
}
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
await ProcessSummaryNotificationAsync(notification).AnyContext();
// Sleep so were not hammering the database.
await Task.Delay(TimeSpan.FromSeconds(1));
}
projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents;
if (projects.Count > 0)
{
await context.RenewLockAsync().AnyContext();
}
}
return(JobResult.SuccessWithMessage("Successfully sent summary notifications."));
}
private async Task ProcessSummaryNotificationAsync(SummaryNotification data)
{
var project = await _projectRepository.GetByIdAsync(data.Id, true).AnyContext();
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
var users = (await _userRepository.GetByIdsAsync(userIds).AnyContext()).Documents.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList();
if (users.Count == 0)
{
_logger.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
_logger.Info().Message("Sending daily summary: users={0} project={1}", users.Count, project.Id).Write();
var result = await _stats.GetTermsStatsAsync(data.UtcStartTime, data.UtcEndTime, "stack_id", "type:error project:" + data.Id, max : 5).AnyContext();
bool hasSubmittedErrors = result.Total > 0;
if (!hasSubmittedErrors)
{
hasSubmittedErrors = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;
}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.New,
UniqueTotal = result.Unique,
HasSubmittedEvents = hasSubmittedErrors,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext();
}
_logger.Info().Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total).Write();
}
protected override Task <JobResult> RunInternalAsync(CancellationToken token)
{
if (!Settings.Current.EnableDailySummary)
{
return(Task.FromResult(JobResult.SuccessWithMessage("Summary notifications are disabled.")));
}
const int BATCH_SIZE = 25;
var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
while (projects.Count > 0 && !token.IsCancellationRequested)
{
var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList());
Log.Info().Message("Got {0} projects to process. ", projects.Count).Write();
Debug.Assert(projects.Count == documentsUpdated);
foreach (var project in projects)
{
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2))
{
Log.Info().Message("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime).Write();
continue;
}
if (_mailer != null)
{
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
ProcessSummaryNotification(notification);
}
else
{
Log.Error().Message("Mailer is null").Write();
}
}
projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
}
return(Task.FromResult(new JobResult {
Message = "Successfully sent summary notifications."
}));
}
protected override Task<JobResult> RunInternalAsync()
{
Log.Info().Message("Daily Notification job starting").Write();
if (!Settings.Current.EnableSummaryNotifications)
return Task.FromResult(new JobResult { Message = "Summary Notifications are disabled.", IsCancelled = true });
const int BATCH_SIZE = 25;
// Get all project id's that should be sent at 9:00am in the projects local time.
var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
while (projects.Count > 0) {
var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList());
Log.Info().Message("Daily Notification job processing {0} projects. Successfully updated {1} projects. ", projects.Count, documentsUpdated);
Debug.Assert(projects.Count == documentsUpdated);
foreach (var project in projects) {
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) {
Log.Info().Message("Skipping Summary Notification older than two days for Project: {0} with a start time of {1}.", project.Id, utcStartTime);
continue;
}
if (_summaryNotificationQueue != null) {
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
Log.Info().Message("Publishing Summary Notification for Project: {0}, with a start time of {1} and an end time of {2}", notification.Id, notification.UtcStartTime, notification.UtcEndTime);
_summaryNotificationQueue.EnqueueAsync(notification);
} else
Log.Error().Message("Message Factory is null").Write();
}
projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
}
return Task.FromResult(new JobResult { Message = "Successfully enforced all retention limits." });
}
protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
_lastRun = SystemClock.UtcNow;
if (!_emailOptions.EnableDailySummary || _mailer == null)
{
return(JobResult.SuccessWithMessage("Summary notifications are disabled."));
}
var results = await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9).AnyContext();
while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
{
_logger.LogTrace("Got {Count} projects to process. ", results.Documents.Count);
var projectsToBulkUpdate = new List <Project>(results.Documents.Count);
var processSummariesNewerThan = SystemClock.UtcNow.Date.SubtractDays(2);
foreach (var project in results.Documents)
{
using (_logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id))) {
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < processSummariesNewerThan)
{
_logger.LogInformation("Skipping daily summary older than two days for project: {Name}", project.Name);
projectsToBulkUpdate.Add(project);
continue;
}
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
bool summarySent = await SendSummaryNotificationAsync(project, notification).AnyContext();
if (summarySent)
{
await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(new[] { project }).AnyContext();
// Sleep so we are not hammering the backend as we just generated a report.
await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext();
}
else
{
projectsToBulkUpdate.Add(project);
}
}
}
if (projectsToBulkUpdate.Count > 0)
{
await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projectsToBulkUpdate).AnyContext();
// Sleep so we are not hammering the backend
await SystemClock.SleepAsync(TimeSpan.FromSeconds(1)).AnyContext();
}
if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext())
{
break;
}
if (results.Documents.Count > 0)
{
await context.RenewLockAsync().AnyContext();
_lastRun = SystemClock.UtcNow;
}
}
return(JobResult.SuccessWithMessage("Successfully sent summary notifications."));
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
// TODO: Add slack daily summaries
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name);
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext();
if (organization == null)
{
_logger.LogInformation("The organization {organization} for project {ProjectName} may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.LogInformation("Sending daily summary: users={UserCount} project={project}", users.Count, project.Id);
var sf = new AppFilter(project, organization);
var systemFilter = new RepositoryQuery <PersistentEvent>().AppFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime);
string filter = "type:error (status:open OR status:regressed)";
var result = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression("terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1")).AnyContext();
double total = result.Aggregations.Sum("sum_count")?.Value ?? result.Total;
double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0;
double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault();
bool isFreePlan = organization.PlanId == _plans.FreePlan.Id;
string fixedFilter = "type:error status:fixed";
var fixedResult = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(fixedFilter).EnforceEventStackFilter().AggregationsExpression("sum:count~1")).AnyContext();
double fixedTotal = fixedResult.Aggregations.Sum("sum_count")?.Value ?? fixedResult.Total;
var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime);
var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList();
int blockedTotal = usages.Sum(u => u.Blocked);
int tooBigTotal = usages.Sum(u => u.TooBig);
IReadOnlyCollection <Stack> mostFrequent = null;
var stackTerms = result.Aggregations.Terms <string>("terms_stack_id");
if (stackTerms?.Buckets.Count > 0)
{
mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext();
}
IReadOnlyCollection <Stack> newest = null;
if (newTotal > 0)
{
newest = (await _stackRepository.FindAsync(q => q.AppFilter(sf).FilterExpression(filter).SortExpression("-first").DateRange(data.UtcStartTime, data.UtcEndTime, "first"), o => o.PageLimit(3)).AnyContext()).Documents;
}
foreach (var user in users)
{
_logger.LogInformation("Queuing {ProjectName} daily summary email ({UtcStartTime}-{UtcEndTime}) for user {EmailAddress}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress);
await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext();
}
_logger.LogInformation("Done sending daily summary: users={UserCount} project={ProjectName} events={EventCount}", users.Count, project.Name, total);
return(true);
}
private async Task ProcessSummaryNotificationAsync(SummaryNotification data)
{
var project = await _projectRepository.GetByIdAsync(data.Id, true).AnyContext();
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info("Project \"{0}\" has no users to send summary to.", project.Id);
return;
}
var users = (await _userRepository.GetByIdsAsync(userIds).AnyContext()).Documents.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList();
if (users.Count == 0)
{
_logger.Info("Project \"{0}\" has no users to send summary to.", project.Id);
return;
}
_logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id);
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
},
new TermFieldAggregation {
Field = "is_first_occurrence", ExcludePattern = "F"
}
};
var result = await _stats.GetNumbersStatsAsync(fields, data.UtcStartTime, data.UtcEndTime, $"project:{data.Id} type:error").AnyContext();
bool hasSubmittedEvents = result.Total > 0;
if (!hasSubmittedEvents)
{
hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;
}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.Numbers[1],
UniqueTotal = result.Numbers[0],
HasSubmittedEvents = hasSubmittedEvents,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext();
}
_logger.Info("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total);
}
private void ProcessSummaryNotification(SummaryNotification data)
{
var project = _projectRepository.GetById(data.Id, true);
var organization = _organizationRepository.GetById(project.OrganizationId, true);
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
return;
}
var users = _userRepository.GetByIds(userIds).Where(u => u.IsEmailAddressVerified).ToList();
if (users.Count == 0)
{
return;
}
long count;
var paging = new PagingOptions {
Limit = 5
};
List <Stack> newest = _stackRepository.GetNew(project.Id, data.UtcStartTime, data.UtcEndTime, paging).ToList();
var result = _stats.GetTermsStats(data.UtcStartTime, data.UtcEndTime, "stack_id", "project:" + data.Id, max: 5);
var mostFrequent = result.Terms.Take(5).ToList();
var stacks = _stackRepository.GetByIds(mostFrequent.Select(s => s.Term).ToList());
foreach (var frequent in mostFrequent)
{
var stack = stacks.SingleOrDefault(s => s.Id == frequent.Term);
if (stack == null)
{
mostFrequent.RemoveAll(r => r.Term == frequent.Term);
continue;
}
// Stat's Id and Total properties are already calculated in the Results.
//frequent.Type = stack.SignatureInfo.ContainsKey("ExceptionType") ? stack.SignatureInfo["ExceptionType"] : null;
//frequent.Method = stack.SignatureInfo.ContainsKey("Method") ? stack.SignatureInfo["Method"] : null;
//frequent.Path = stack.SignatureInfo.ContainsKey("Path") ? stack.SignatureInfo["Path"] : null;
//frequent.Is404 = stack.SignatureInfo.ContainsKey("Path");
//frequent.Title = stack.Title;
//frequent.First = stack.FirstOccurrence;
//frequent.Last = stack.LastOccurrence;
}
var notification = new SummaryNotificationModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
//Total = result.Total,
//PerHourAverage = result.PerHourAverage,
//NewTotal = result.NewTotal,
//New = newest,
//UniqueTotal = result.UniqueTotal,
//MostFrequent = mostFrequent,
//HasSubmittedErrors = project.TotalErrorCount > 0,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users.Where(u => u.EmailNotificationsEnabled))
{
_mailer.SendSummaryNotification(user.EmailAddress, notification);
}
}
private void ProcessSummaryNotification(SummaryNotification data)
{
var project = _projectRepository.GetById(data.Id, true);
var organization = _organizationRepository.GetById(project.OrganizationId, true);
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
Log.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
var users = _userRepository.GetByIds(userIds).Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList();
if (users.Count == 0)
{
Log.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
Log.Info().Message("Sending daily summary: users={0} project={1}", users.Count, project.Id).Write();
var paging = new PagingOptions {
Limit = 5
};
List <Stack> newest = _stackRepository.GetNew(project.Id, data.UtcStartTime, data.UtcEndTime, paging).ToList();
var result = _stats.GetTermsStats(data.UtcStartTime, data.UtcEndTime, "stack_id", "type:error project:" + data.Id, max: 5);
//var termStatsList = result.Terms.Take(5).ToList();
//var stacks = _stackRepository.GetByIds(termStatsList.Select(s => s.Term).ToList());
bool hasSubmittedErrors = result.Total > 0;
if (!hasSubmittedErrors)
{
hasSubmittedErrors = _eventRepository.GetCountByProjectId(project.Id) > 0;
}
var mostFrequent = new List <EventStackResult>();
//foreach (var termStats in termStatsList) {
// var stack = stacks.SingleOrDefault(s => s.Id == termStats.Term);
// if (stack == null)
// continue;
// mostFrequent.Add(new EventStackResult {
// First = termStats.FirstOccurrence,
// Last = termStats.LastOccurrence,
// Id = stack.Id,
// Title = stack.Title,
// Total = termStats.Total,
// Type = stack.SignatureInfo.ContainsKey("ExceptionType") ? stack.SignatureInfo["ExceptionType"] : null,
// Method = stack.SignatureInfo.ContainsKey("Method") ? stack.SignatureInfo["Method"] : null,
// Path = stack.SignatureInfo.ContainsKey("Source") ? stack.SignatureInfo["Source"] : null,
// Is404 = stack.SignatureInfo.ContainsKey("Type") && stack.SignatureInfo["Type"] == "404"
// });
//}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.New,
New = newest,
UniqueTotal = result.Unique,
MostFrequent = mostFrequent,
HasSubmittedEvents = hasSubmittedErrors,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
_mailer.SendDailySummary(user.EmailAddress, notification);
}
Log.Info().Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total).Write();
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write();
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, true).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
if (organization == null)
{
_logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id);
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
},
new TermFieldAggregation {
Field = "is_first_occurrence", ExcludePattern = "F"
}
};
var sf = new ExceptionlessSystemFilterQuery(project, organization);
var result = await _stats.GetNumbersStatsAsync(fields, data.UtcStartTime, data.UtcEndTime, sf, $"{EventIndexType.Fields.Type}:{Event.KnownTypes.Error}").AnyContext();
bool hasSubmittedEvents = result.Total > 0;
if (!hasSubmittedEvents)
{
hasSubmittedEvents = await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;
}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.Numbers[1],
UniqueTotal = result.Numbers[0],
HasSubmittedEvents = hasSubmittedEvents,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
await _mailer.SendDailySummaryAsync(user.EmailAddress, notification).AnyContext();
}
_logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, notification.Total);
return(true);
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
// TODO: Add slack daily summaries
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name).Write();
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.Info().Project(project.Id).Message("Project \"{0}\" has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext();
if (organization == null)
{
_logger.Info().Project(project.Id).Message("The organization \"{0}\" for project \"{1}\" may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.Info("Sending daily summary: users={0} project={1}", users.Count, project.Id);
var sf = new ExceptionlessSystemFilter(project, organization);
var systemFilter = new RepositoryQuery <PersistentEvent>().SystemFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime);
string filter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:false";
var result = await _eventRepository.CountBySearchAsync(systemFilter, filter, "terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1").AnyContext();
double total = result.Aggregations.Sum("sum_count").Value ?? result.Total;
double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0;
double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault();
bool isFreePlan = organization.PlanId == BillingManager.FreePlan.Id;
string fixedFilter = $"{EventIndexType.Alias.Type}:{Event.KnownTypes.Error} {EventIndexType.Alias.IsHidden}:false {EventIndexType.Alias.IsFixed}:true";
var fixedResult = await _eventRepository.CountBySearchAsync(systemFilter, fixedFilter, "sum:count~1").AnyContext();
double fixedTotal = fixedResult.Aggregations.Sum("sum_count").Value ?? fixedResult.Total;
var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime);
var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList();
int blockedTotal = usages.Sum(u => u.Blocked);
int tooBigTotal = usages.Sum(u => u.TooBig);
IReadOnlyCollection <Stack> mostFrequent = null;
var stackTerms = result.Aggregations.Terms <string>("terms_stack_id");
if (stackTerms?.Buckets.Count > 0)
{
mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext();
}
IReadOnlyCollection <Stack> newest = null;
if (newTotal > 0)
{
newest = (await _stackRepository.GetByFilterAsync(sf, filter, "-first", "first", data.UtcStartTime, data.UtcEndTime, o => o.PageLimit(3)).AnyContext()).Documents;
}
foreach (var user in users)
{
_logger.Info().Project(project.Id).Message("Queuing \"{0}\" daily summary email ({1}-{2}) for user {3}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress);
await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext();
}
_logger.Info().Project(project.Id).Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Name, total);
return(true);
}
public override JobResult Run(JobContext context)
{
Log.Info().Message("Daily Notification job starting").Write();
if (!Settings.Current.EnableSummaryNotifications)
{
return(new JobResult {
Result = "Summary Notifications are disabled.",
Cancelled = true
});
}
const int BATCH_SIZE = 25;
// Send an email at 9:00am in the projects local time.
IMongoQuery query = Query.LT(ProjectRepository.FieldNames.NextSummaryEndOfDayTicks, new BsonInt64(DateTime.UtcNow.Ticks - (TimeSpan.TicksPerHour * 9)));
UpdateBuilder update = Update.Inc(ProjectRepository.FieldNames.NextSummaryEndOfDayTicks, TimeSpan.TicksPerDay);
var projects = _projectRepository.Collection.FindAs <Project>(query).SetFields(ProjectRepository.FieldNames.Id, ProjectRepository.FieldNames.NextSummaryEndOfDayTicks).SetLimit(BATCH_SIZE).ToList();
while (projects.Count > 0)
{
IMongoQuery queryWithProjectIds = Query.And(Query.In(ProjectRepository.FieldNames.Id, projects.Select(p => new BsonObjectId(new ObjectId(p.Id)))), query);
var result = _projectRepository.Collection.Update(queryWithProjectIds, update, UpdateFlags.Multi);
Log.Info().Message("Daily Notification job processing {0} projects. Successfully updated {1} projects. ", projects.Count, result.DocumentsAffected);
Debug.Assert(projects.Count == result.DocumentsAffected);
foreach (var project in projects)
{
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2))
{
Log.Info().Message("Skipping Summary Notification older than two days for Project: {0} with a start time of {1}.", project.Id, utcStartTime);
continue;
}
if (_messageFactory != null)
{
using (IMessageProducer messageProducer = _messageFactory.CreateMessageProducer()) {
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
Log.Info().Message("Publishing Summary Notification for Project: {0}, with a start time of {1} and an end time of {2}", notification.Id, notification.UtcStartTime, notification.UtcEndTime);
messageProducer.Publish(notification);
}
}
else
{
Log.Error().Message("Message Factory is null").Write();
}
}
projects = _projectRepository.Collection.FindAs <Project>(query).SetFields(ProjectRepository.FieldNames.Id, ProjectRepository.FieldNames.NextSummaryEndOfDayTicks).SetLimit(BATCH_SIZE).ToList();
}
return(new JobResult {
Result = "Successfully enforced all retention limits."
});
}