protected override async Task <JobResult> RunInternalAsync(JobContext context) { if (!Settings.Current.EnableDailySummary) { return(JobResult.SuccessWithMessage("Summary notifications are disabled.")); } if (_mailer == null) { return(JobResult.SuccessWithMessage("Summary notifications are disabled due to null mailer.")); } const int BATCH_SIZE = 25; var projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents; while (projects.Count > 0 && !context.CancellationToken.IsCancellationRequested) { var documentsUpdated = await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projects).AnyContext(); _logger.Info("Got {0} projects to process. ", projects.Count); Debug.Assert(projects.Count == documentsUpdated); foreach (var project in projects) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) { _logger.Info("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime); continue; } var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; await ProcessSummaryNotificationAsync(notification).AnyContext(); // Sleep so were not hammering the database. await Task.Delay(TimeSpan.FromSeconds(1)); } projects = (await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9, BATCH_SIZE).AnyContext()).Documents; if (projects.Count > 0) { await context.RenewLockAsync().AnyContext(); } } return(JobResult.SuccessWithMessage("Successfully sent summary notifications.")); }
protected override Task <JobResult> RunInternalAsync(CancellationToken token) { if (!Settings.Current.EnableDailySummary) { return(Task.FromResult(JobResult.SuccessWithMessage("Summary notifications are disabled."))); } const int BATCH_SIZE = 25; var projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); while (projects.Count > 0 && !token.IsCancellationRequested) { var documentsUpdated = _projectRepository.IncrementNextSummaryEndOfDayTicks(projects.Select(p => p.Id).ToList()); Log.Info().Message("Got {0} projects to process. ", projects.Count).Write(); Debug.Assert(projects.Count == documentsUpdated); foreach (var project in projects) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < DateTime.UtcNow.Date.SubtractDays(2)) { Log.Info().Message("Skipping daily summary older than two days for project \"{0}\" with a start time of \"{1}\".", project.Id, utcStartTime).Write(); continue; } if (_mailer != null) { var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; ProcessSummaryNotification(notification); } else { Log.Error().Message("Mailer is null").Write(); } } projects = _projectRepository.GetByNextSummaryNotificationOffset(9, BATCH_SIZE); } return(Task.FromResult(new JobResult { Message = "Successfully sent summary notifications." })); }
protected override async Task <JobResult> RunInternalAsync(JobContext context) { _lastRun = SystemClock.UtcNow; if (!_emailOptions.EnableDailySummary || _mailer == null) { return(JobResult.SuccessWithMessage("Summary notifications are disabled.")); } var results = await _projectRepository.GetByNextSummaryNotificationOffsetAsync(9).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { _logger.LogTrace("Got {Count} projects to process. ", results.Documents.Count); var projectsToBulkUpdate = new List <Project>(results.Documents.Count); var processSummariesNewerThan = SystemClock.UtcNow.Date.SubtractDays(2); foreach (var project in results.Documents) { using (_logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id))) { var utcStartTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerDay); if (utcStartTime < processSummariesNewerThan) { _logger.LogInformation("Skipping daily summary older than two days for project: {Name}", project.Name); projectsToBulkUpdate.Add(project); continue; } var notification = new SummaryNotification { Id = project.Id, UtcStartTime = utcStartTime, UtcEndTime = new DateTime(project.NextSummaryEndOfDayTicks - TimeSpan.TicksPerSecond) }; bool summarySent = await SendSummaryNotificationAsync(project, notification).AnyContext(); if (summarySent) { await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(new[] { project }).AnyContext(); // Sleep so we are not hammering the backend as we just generated a report. await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext(); } else { projectsToBulkUpdate.Add(project); } } } if (projectsToBulkUpdate.Count > 0) { await _projectRepository.IncrementNextSummaryEndOfDayTicksAsync(projectsToBulkUpdate).AnyContext(); // Sleep so we are not hammering the backend await SystemClock.SleepAsync(TimeSpan.FromSeconds(1)).AnyContext(); } if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext()) { break; } if (results.Documents.Count > 0) { await context.RenewLockAsync().AnyContext(); _lastRun = SystemClock.UtcNow; } } return(JobResult.SuccessWithMessage("Successfully sent summary notifications.")); }
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <EventNotification> context) { var wi = context.QueueEntry.Value; var ev = await _eventRepository.GetByIdAsync(wi.EventId).AnyContext(); if (ev == null) { return(JobResult.SuccessWithMessage($"Could not load event: {wi.EventId}")); } bool shouldLog = ev.ProjectId != _appOptions.InternalProjectId; int sent = 0; if (shouldLog) { _logger.LogTrace("Process notification: project={project} event={id} stack={stack}", ev.ProjectId, ev.Id, ev.StackId); } var project = await _projectRepository.GetByIdAsync(ev.ProjectId, o => o.Cache()).AnyContext(); if (project == null) { return(JobResult.SuccessWithMessage($"Could not load project: {ev.ProjectId}.")); } using (_logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id))) { if (shouldLog) { _logger.LogTrace("Loaded project: name={ProjectName}", project.Name); } // after the first 2 occurrences, don't send a notification for the same stack more then once every 30 minutes var lastTimeSentUtc = await _cache.GetAsync <DateTime>(String.Concat("notify:stack-throttle:", ev.StackId), DateTime.MinValue).AnyContext(); if (wi.TotalOccurrences > 2 && !wi.IsRegression && lastTimeSentUtc != DateTime.MinValue && lastTimeSentUtc > SystemClock.UtcNow.AddMinutes(-30)) { if (shouldLog) { _logger.LogInformation("Skipping message because of stack throttling: last sent={LastSentUtc} occurrences={TotalOccurrences}", lastTimeSentUtc, wi.TotalOccurrences); } return(JobResult.Success); } if (context.CancellationToken.IsCancellationRequested) { return(JobResult.Cancelled); } // don't send more than 10 notifications for a given project every 30 minutes var projectTimeWindow = TimeSpan.FromMinutes(30); string cacheKey = String.Concat("notify:project-throttle:", ev.ProjectId, "-", SystemClock.UtcNow.Floor(projectTimeWindow).Ticks); double notificationCount = await _cache.IncrementAsync(cacheKey, 1, projectTimeWindow).AnyContext(); if (notificationCount > 10 && !wi.IsRegression) { if (shouldLog) { _logger.LogInformation("Skipping message because of project throttling: count={NotificationCount}", notificationCount); } return(JobResult.Success); } foreach (var kv in project.NotificationSettings) { var settings = kv.Value; if (shouldLog) { _logger.LogTrace("Processing notification: {Key}", kv.Key); } bool isCritical = ev.IsCritical(); bool shouldReportNewError = settings.ReportNewErrors && wi.IsNew && ev.IsError(); bool shouldReportCriticalError = settings.ReportCriticalErrors && isCritical && ev.IsError(); bool shouldReportRegression = settings.ReportEventRegressions && wi.IsRegression; bool shouldReportNewEvent = settings.ReportNewEvents && wi.IsNew; bool shouldReportCriticalEvent = settings.ReportCriticalEvents && isCritical; bool shouldReport = shouldReportNewError || shouldReportCriticalError || shouldReportRegression || shouldReportNewEvent || shouldReportCriticalEvent; if (shouldLog) { _logger.LogTrace("Settings: new error={ReportNewErrors} critical error={ReportCriticalErrors} regression={ReportEventRegressions} new={ReportNewEvents} critical={ReportCriticalEvents}", settings.ReportNewErrors, settings.ReportCriticalErrors, settings.ReportEventRegressions, settings.ReportNewEvents, settings.ReportCriticalEvents); _logger.LogTrace("Should process: new error={ShouldReportNewError} critical error={ShouldReportCriticalError} regression={ShouldReportRegression} new={ShouldReportNewEvent} critical={ShouldReportCriticalEvent}", shouldReportNewError, shouldReportCriticalError, shouldReportRegression, shouldReportNewEvent, shouldReportCriticalEvent); } var request = ev.GetRequestInfo(); // check for known bots if the user has elected to not report them if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent)) { var botPatterns = project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList(); var info = await _parser.ParseAsync(request.UserAgent).AnyContext(); if (info != null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns)) { shouldReport = false; if (shouldLog) { _logger.LogInformation("Skipping because event is from a bot {UserAgent}.", request.UserAgent); } } } if (!shouldReport) { continue; } bool processed; switch (kv.Key) { case Project.NotificationIntegrations.Slack: processed = await _slackService.SendEventNoticeAsync(ev, project, wi.IsNew, wi.IsRegression).AnyContext(); break; default: processed = await SendEmailNotificationAsync(kv.Key, project, ev, wi, shouldLog).AnyContext(); break; } if (shouldLog) { _logger.LogTrace("Finished processing notification: {Key}", kv.Key); } if (processed) { sent++; } } // if we sent any notifications, mark the last time a notification for this stack was sent. if (sent > 0) { await _cache.SetAsync(String.Concat("notify:stack-throttle:", ev.StackId), SystemClock.UtcNow, SystemClock.UtcNow.AddMinutes(15)).AnyContext(); if (shouldLog) { _logger.LogInformation("Notifications sent: event={id} stack={stack} count={SentCount}", ev.Id, ev.StackId, sent); } } } return(JobResult.Success); }