protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
if (!Settings.Current.EnableDailySummary)
{
return(JobResult.SuccessWithMessage("Summary notifications are disabled."));
}
if (_mailer == null)
{
return(JobResult.SuccessWithMessage("Summary notifications are disabled due to null mailer."));
}
const int BATCH_SIZE = 25;
var projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents;
while (projects.Count > 0 && !context.CancellationToken.IsCancellationRequested)
{
var documentsUpdated = await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projects).AnyContext();
_logger.Info("Got {0} projects to process. ", projects.Count);
Debug.Assert(projects.Count == documentsUpdated);
foreach (var project in projects)
{
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2))
{
_logger.Info("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime);
continue;
}
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
await ProcessSummaryNotificationAsync(notification).AnyContext();
// Sleep so were not hammering the database.
await Task.Delay(TimeSpan.FromSeconds(1));
}
projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents;
if (projects.Count > 0)
{
await context.RenewLockAsync().AnyContext();
}
}
return(JobResult.SuccessWithMessage("Successfully sent summary notifications."));
}
protected override Task <JobResult> RunInternalAsync(CancellationToken token)
{
if (!Settings.Current.EnableDailySummary)
{
return(Task.FromResult(JobResult.SuccessWithMessage("Summary notifications are disabled.")));
}
const int BATCH_SIZE = 25;
var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
while (projects.Count > 0 && !token.IsCancellationRequested)
{
var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList());
Log.Info().Message("Got {0} projects to process. ", projects.Count).Write();
Debug.Assert(projects.Count == documentsUpdated);
foreach (var project in projects)
{
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2))
{
Log.Info().Message("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime).Write();
continue;
}
if (_mailer != null)
{
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
ProcessSummaryNotification(notification);
}
else
{
Log.Error().Message("Mailer is null").Write();
}
}
projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE);
}
return(Task.FromResult(new JobResult {
Message = "Successfully sent summary notifications."
}));
}
protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
_lastRun = SystemClock.UtcNow;
if (!_emailOptions.EnableDailySummary || _mailer == null)
{
return(JobResult.SuccessWithMessage("Summary notifications are disabled."));
}
var results = await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9).AnyContext();
while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
{
_logger.LogTrace("Got {Count} projects to process. ", results.Documents.Count);
var projectsToBulkUpdate = new List <Project>(results.Documents.Count);
var processSummariesNewerThan = SystemClock.UtcNow.Date.SubtractDays(2);
foreach (var project in results.Documents)
{
using (_logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id))) {
var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay);
if (utcStartTime < processSummariesNewerThan)
{
_logger.LogInformation("Skipping daily summary older than two days for project: {Name}", project.Name);
projectsToBulkUpdate.Add(project);
continue;
}
var notification = new SummaryNotification {
Id = project.Id,
UtcStartTime = utcStartTime,
UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond)
};
bool summarySent = await SendSummaryNotificationAsync(project, notification).AnyContext();
if (summarySent)
{
await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(new[] { project }).AnyContext();
// Sleep so we are not hammering the backend as we just generated a report.
await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext();
}
else
{
projectsToBulkUpdate.Add(project);
}
}
}
if (projectsToBulkUpdate.Count > 0)
{
await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projectsToBulkUpdate).AnyContext();
// Sleep so we are not hammering the backend
await SystemClock.SleepAsync(TimeSpan.FromSeconds(1)).AnyContext();
}
if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext())
{
break;
}
if (results.Documents.Count > 0)
{
await context.RenewLockAsync().AnyContext();
_lastRun = SystemClock.UtcNow;
}
}
return(JobResult.SuccessWithMessage("Successfully sent summary notifications."));
}
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <EventNotification> context)
{
var wi = context.QueueEntry.Value;
var ev = await _eventRepository.GetByIdAsync(wi.EventId).AnyContext();
if (ev == null)
{
return(JobResult.SuccessWithMessage($"Could not load event: {wi.EventId}"));
}
bool shouldLog = ev.ProjectId != _appOptions.InternalProjectId;
int sent = 0;
if (shouldLog)
{
_logger.LogTrace("Process notification: project={project} event={id} stack={stack}", ev.ProjectId, ev.Id, ev.StackId);
}
var project = await _projectRepository.GetByIdAsync(ev.ProjectId, o => o.Cache()).AnyContext();
if (project == null)
{
return(JobResult.SuccessWithMessage($"Could not load project: {ev.ProjectId}."));
}
using (_logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id))) {
if (shouldLog)
{
_logger.LogTrace("Loaded project: name={ProjectName}", project.Name);
}
// after the first 2 occurrences, don't send a notification for the same stack more then once every 30 minutes
var lastTimeSentUtc = await _cache.GetAsync <DateTime>(String.Concat("notify:stack-throttle:", ev.StackId), DateTime.MinValue).AnyContext();
if (wi.TotalOccurrences > 2 && !wi.IsRegression && lastTimeSentUtc != DateTime.MinValue && lastTimeSentUtc > SystemClock.UtcNow.AddMinutes(-30))
{
if (shouldLog)
{
_logger.LogInformation("Skipping message because of stack throttling: last sent={LastSentUtc} occurrences={TotalOccurrences}", lastTimeSentUtc, wi.TotalOccurrences);
}
return(JobResult.Success);
}
if (context.CancellationToken.IsCancellationRequested)
{
return(JobResult.Cancelled);
}
// don't send more than 10 notifications for a given project every 30 minutes
var projectTimeWindow = TimeSpan.FromMinutes(30);
string cacheKey = String.Concat("notify:project-throttle:", ev.ProjectId, "-", SystemClock.UtcNow.Floor(projectTimeWindow).Ticks);
double notificationCount = await _cache.IncrementAsync(cacheKey, 1, projectTimeWindow).AnyContext();
if (notificationCount > 10 && !wi.IsRegression)
{
if (shouldLog)
{
_logger.LogInformation("Skipping message because of project throttling: count={NotificationCount}", notificationCount);
}
return(JobResult.Success);
}
foreach (var kv in project.NotificationSettings)
{
var settings = kv.Value;
if (shouldLog)
{
_logger.LogTrace("Processing notification: {Key}", kv.Key);
}
bool isCritical = ev.IsCritical();
bool shouldReportNewError = settings.ReportNewErrors && wi.IsNew && ev.IsError();
bool shouldReportCriticalError = settings.ReportCriticalErrors && isCritical && ev.IsError();
bool shouldReportRegression = settings.ReportEventRegressions && wi.IsRegression;
bool shouldReportNewEvent = settings.ReportNewEvents && wi.IsNew;
bool shouldReportCriticalEvent = settings.ReportCriticalEvents && isCritical;
bool shouldReport = shouldReportNewError || shouldReportCriticalError || shouldReportRegression || shouldReportNewEvent || shouldReportCriticalEvent;
if (shouldLog)
{
_logger.LogTrace("Settings: new error={ReportNewErrors} critical error={ReportCriticalErrors} regression={ReportEventRegressions} new={ReportNewEvents} critical={ReportCriticalEvents}", settings.ReportNewErrors, settings.ReportCriticalErrors, settings.ReportEventRegressions, settings.ReportNewEvents, settings.ReportCriticalEvents);
_logger.LogTrace("Should process: new error={ShouldReportNewError} critical error={ShouldReportCriticalError} regression={ShouldReportRegression} new={ShouldReportNewEvent} critical={ShouldReportCriticalEvent}", shouldReportNewError, shouldReportCriticalError, shouldReportRegression, shouldReportNewEvent, shouldReportCriticalEvent);
}
var request = ev.GetRequestInfo();
// check for known bots if the user has elected to not report them
if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent))
{
var botPatterns = project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList();
var info = await _parser.ParseAsync(request.UserAgent).AnyContext();
if (info != null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns))
{
shouldReport = false;
if (shouldLog)
{
_logger.LogInformation("Skipping because event is from a bot {UserAgent}.", request.UserAgent);
}
}
}
if (!shouldReport)
{
continue;
}
bool processed;
switch (kv.Key)
{
case Project.NotificationIntegrations.Slack:
processed = await _slackService.SendEventNoticeAsync(ev, project, wi.IsNew, wi.IsRegression).AnyContext();
break;
default:
processed = await SendEmailNotificationAsync(kv.Key, project, ev, wi, shouldLog).AnyContext();
break;
}
if (shouldLog)
{
_logger.LogTrace("Finished processing notification: {Key}", kv.Key);
}
if (processed)
{
sent++;
}
}
// if we sent any notifications, mark the last time a notification for this stack was sent.
if (sent > 0)
{
await _cache.SetAsync(String.Concat("notify:stack-throttle:", ev.StackId), SystemClock.UtcNow, SystemClock.UtcNow.AddMinutes(15)).AnyContext();
if (shouldLog)
{
_logger.LogInformation("Notifications sent: event={id} stack={stack} count={SentCount}", ev.Id, ev.StackId, sent);
}
}
}
return(JobResult.Success);
}
