private async Task SetBrowserOsAndDeviceFromUserAgent(RequestInfo request, EventContext context) { var info = await _parser.ParseAsync(request.UserAgent, context.Project.Id).AnyContext(); if (info != null) { if (!String.Equals(info.UserAgent.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family; if (!String.IsNullOrEmpty(info.UserAgent.Major)) { request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v))); request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major; } } if (!String.Equals(info.Device.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family; } if (!String.Equals(info.OS.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family; if (!String.IsNullOrEmpty(info.OS.Major)) { request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v))); request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major; } } var botPatterns = context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList(); request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns); } }
public override async Task EventBatchProcessingAsync(ICollection <EventContext> contexts) { var project = contexts.First().Project; var exclusions = project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.DataExclusions) ? DefaultExclusions.Union(project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions)).ToList() : DefaultExclusions; foreach (var context in contexts) { var request = context.Event.GetRequestInfo(); if (request == null) { continue; } var info = await _parser.ParseAsync(request.UserAgent, context.Project.Id).AnyContext(); if (info != null) { if (!String.Equals(info.UserAgent.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family; if (!String.IsNullOrEmpty(info.UserAgent.Major)) { request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v))); request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major; } } if (!String.Equals(info.Device.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family; } if (!String.Equals(info.OS.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family; if (!String.IsNullOrEmpty(info.OS.Major)) { request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v))); request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major; } } var botPatterns = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns) ? context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList() : new List <string>(); request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns); } context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); } }
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <EventNotification> context) { var wi = context.QueueEntry.Value; var ev = await _eventRepository.GetByIdAsync(wi.EventId).AnyContext(); if (ev == null) { return(JobResult.SuccessWithMessage($"Could not load event: {wi.EventId}")); } bool shouldLog = ev.ProjectId != _appOptions.InternalProjectId; int sent = 0; if (shouldLog) { _logger.LogTrace("Process notification: project={project} event={id} stack={stack}", ev.ProjectId, ev.Id, ev.StackId); } var project = await _projectRepository.GetByIdAsync(ev.ProjectId, o => o.Cache()).AnyContext(); if (project == null) { return(JobResult.SuccessWithMessage($"Could not load project: {ev.ProjectId}.")); } using (_logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id))) { if (shouldLog) { _logger.LogTrace("Loaded project: name={ProjectName}", project.Name); } // after the first 2 occurrences, don't send a notification for the same stack more then once every 30 minutes var lastTimeSentUtc = await _cache.GetAsync <DateTime>(String.Concat("notify:stack-throttle:", ev.StackId), DateTime.MinValue).AnyContext(); if (wi.TotalOccurrences > 2 && !wi.IsRegression && lastTimeSentUtc != DateTime.MinValue && lastTimeSentUtc > SystemClock.UtcNow.AddMinutes(-30)) { if (shouldLog) { _logger.LogInformation("Skipping message because of stack throttling: last sent={LastSentUtc} occurrences={TotalOccurrences}", lastTimeSentUtc, wi.TotalOccurrences); } return(JobResult.Success); } if (context.CancellationToken.IsCancellationRequested) { return(JobResult.Cancelled); } // don't send more than 10 notifications for a given project every 30 minutes var projectTimeWindow = TimeSpan.FromMinutes(30); string cacheKey = String.Concat("notify:project-throttle:", ev.ProjectId, "-", SystemClock.UtcNow.Floor(projectTimeWindow).Ticks); double notificationCount = await _cache.IncrementAsync(cacheKey, 1, projectTimeWindow).AnyContext(); if (notificationCount > 10 && !wi.IsRegression) { if (shouldLog) { _logger.LogInformation("Skipping message because of project throttling: count={NotificationCount}", notificationCount); } return(JobResult.Success); } foreach (var kv in project.NotificationSettings) { var settings = kv.Value; if (shouldLog) { _logger.LogTrace("Processing notification: {Key}", kv.Key); } bool isCritical = ev.IsCritical(); bool shouldReportNewError = settings.ReportNewErrors && wi.IsNew && ev.IsError(); bool shouldReportCriticalError = settings.ReportCriticalErrors && isCritical && ev.IsError(); bool shouldReportRegression = settings.ReportEventRegressions && wi.IsRegression; bool shouldReportNewEvent = settings.ReportNewEvents && wi.IsNew; bool shouldReportCriticalEvent = settings.ReportCriticalEvents && isCritical; bool shouldReport = shouldReportNewError || shouldReportCriticalError || shouldReportRegression || shouldReportNewEvent || shouldReportCriticalEvent; if (shouldLog) { _logger.LogTrace("Settings: new error={ReportNewErrors} critical error={ReportCriticalErrors} regression={ReportEventRegressions} new={ReportNewEvents} critical={ReportCriticalEvents}", settings.ReportNewErrors, settings.ReportCriticalErrors, settings.ReportEventRegressions, settings.ReportNewEvents, settings.ReportCriticalEvents); _logger.LogTrace("Should process: new error={ShouldReportNewError} critical error={ShouldReportCriticalError} regression={ShouldReportRegression} new={ShouldReportNewEvent} critical={ShouldReportCriticalEvent}", shouldReportNewError, shouldReportCriticalError, shouldReportRegression, shouldReportNewEvent, shouldReportCriticalEvent); } var request = ev.GetRequestInfo(); // check for known bots if the user has elected to not report them if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent)) { var botPatterns = project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList(); var info = await _parser.ParseAsync(request.UserAgent).AnyContext(); if (info != null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns)) { shouldReport = false; if (shouldLog) { _logger.LogInformation("Skipping because event is from a bot {UserAgent}.", request.UserAgent); } } } if (!shouldReport) { continue; } bool processed; switch (kv.Key) { case Project.NotificationIntegrations.Slack: processed = await _slackService.SendEventNoticeAsync(ev, project, wi.IsNew, wi.IsRegression).AnyContext(); break; default: processed = await SendEmailNotificationAsync(kv.Key, project, ev, wi, shouldLog).AnyContext(); break; } if (shouldLog) { _logger.LogTrace("Finished processing notification: {Key}", kv.Key); } if (processed) { sent++; } } // if we sent any notifications, mark the last time a notification for this stack was sent. if (sent > 0) { await _cache.SetAsync(String.Concat("notify:stack-throttle:", ev.StackId), SystemClock.UtcNow, SystemClock.UtcNow.AddMinutes(15)).AnyContext(); if (shouldLog) { _logger.LogInformation("Notifications sent: event={id} stack={stack} count={SentCount}", ev.Id, ev.StackId, sent); } } } return(JobResult.Success); }
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <EventNotificationWorkItem> context) { var eventModel = await _eventRepository.GetByIdAsync(context.QueueEntry.Value.EventId).AnyContext(); if (eventModel == null) { return(JobResult.FailedWithMessage($"Could not load event: {context.QueueEntry.Value.EventId}")); } var eventNotification = new EventNotification(context.QueueEntry.Value, eventModel); bool shouldLog = eventNotification.Event.ProjectId != Settings.Current.InternalProjectId; int emailsSent = 0; Logger.Trace().Message("Process notification: project={0} event={1} stack={2}", eventNotification.Event.ProjectId, eventNotification.Event.Id, eventNotification.Event.StackId).WriteIf(shouldLog); var project = await _projectRepository.GetByIdAsync(eventNotification.Event.ProjectId, true).AnyContext(); if (project == null) { return(JobResult.FailedWithMessage($"Could not load project: {eventNotification.Event.ProjectId}.")); } Logger.Trace().Message($"Loaded project: name={project.Name}").WriteIf(shouldLog); var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext(); if (organization == null) { return(JobResult.FailedWithMessage($"Could not load organization: {project.OrganizationId}")); } Logger.Trace().Message($"Loaded organization: {organization.Name}").WriteIf(shouldLog); var stack = await _stackRepository.GetByIdAsync(eventNotification.Event.StackId).AnyContext(); if (stack == null) { return(JobResult.FailedWithMessage($"Could not load stack: {eventNotification.Event.StackId}")); } if (!organization.HasPremiumFeatures) { Logger.Info().Message("Skipping \"{0}\" because organization \"{1}\" does not have premium features.", eventNotification.Event.Id, eventNotification.Event.OrganizationId).WriteIf(shouldLog); return(JobResult.Success); } if (stack.DisableNotifications || stack.IsHidden) { Logger.Info().Message("Skipping \"{0}\" because stack \"{1}\" notifications are disabled or stack is hidden.", eventNotification.Event.Id, eventNotification.Event.StackId).WriteIf(shouldLog); return(JobResult.Success); } if (context.CancellationToken.IsCancellationRequested) { return(JobResult.Cancelled); } Logger.Trace().Message("Loaded stack: title={0}", stack.Title).WriteIf(shouldLog); int totalOccurrences = stack.TotalOccurrences; // after the first 2 occurrences, don't send a notification for the same stack more then once every 30 minutes var lastTimeSentUtc = await _cacheClient.GetAsync <DateTime>(String.Concat("notify:stack-throttle:", eventNotification.Event.StackId), DateTime.MinValue).AnyContext(); if (totalOccurrences > 2 && !eventNotification.IsRegression && lastTimeSentUtc != DateTime.MinValue && lastTimeSentUtc > DateTime.UtcNow.AddMinutes(-30)) { Logger.Info().Message("Skipping message because of stack throttling: last sent={0} occurrences={1}", lastTimeSentUtc, totalOccurrences).WriteIf(shouldLog); return(JobResult.Success); } // don't send more than 10 notifications for a given project every 30 minutes var projectTimeWindow = TimeSpan.FromMinutes(30); string cacheKey = String.Concat("notify:project-throttle:", eventNotification.Event.ProjectId, "-", DateTime.UtcNow.Floor(projectTimeWindow).Ticks); long notificationCount = await _cacheClient.IncrementAsync(cacheKey, 1, projectTimeWindow).AnyContext(); if (notificationCount > 10 && !eventNotification.IsRegression) { Logger.Info().Project(eventNotification.Event.ProjectId).Message("Skipping message because of project throttling: count={0}", notificationCount).WriteIf(shouldLog); return(JobResult.Success); } if (context.CancellationToken.IsCancellationRequested) { return(JobResult.Cancelled); } foreach (var kv in project.NotificationSettings) { var settings = kv.Value; Logger.Trace().Message("Processing notification: user={0}", kv.Key).WriteIf(shouldLog); var user = await _userRepository.GetByIdAsync(kv.Key).AnyContext(); if (String.IsNullOrEmpty(user?.EmailAddress)) { Logger.Error().Message("Could not load user {0} or blank email address {1}.", kv.Key, user != null ? user.EmailAddress : "").Write(); continue; } if (!user.IsEmailAddressVerified) { Logger.Info().Message("User {0} with email address {1} has not been verified.", kv.Key, user != null ? user.EmailAddress : "").WriteIf(shouldLog); continue; } if (!user.EmailNotificationsEnabled) { Logger.Info().Message("User {0} with email address {1} has email notifications disabled.", kv.Key, user != null ? user.EmailAddress : "").WriteIf(shouldLog); continue; } if (!user.OrganizationIds.Contains(project.OrganizationId)) { Logger.Error().Message("Unauthorized user: project={0} user={1} organization={2} event={3}", project.Id, kv.Key, project.OrganizationId, eventNotification.Event.Id).Write(); continue; } Logger.Trace().Message("Loaded user: email={0}", user.EmailAddress).WriteIf(shouldLog); bool shouldReportNewError = settings.ReportNewErrors && eventNotification.IsNew && eventNotification.Event.IsError(); bool shouldReportCriticalError = settings.ReportCriticalErrors && eventNotification.IsCritical && eventNotification.Event.IsError(); bool shouldReportRegression = settings.ReportEventRegressions && eventNotification.IsRegression; bool shouldReportNewEvent = settings.ReportNewEvents && eventNotification.IsNew; bool shouldReportCriticalEvent = settings.ReportCriticalEvents && eventNotification.IsCritical; bool shouldReport = shouldReportNewError || shouldReportCriticalError || shouldReportRegression || shouldReportNewEvent || shouldReportCriticalEvent; Logger.Trace().Message("Settings: newerror={0} criticalerror={1} regression={2} new={3} critical={4}", settings.ReportNewErrors, settings.ReportCriticalErrors, settings.ReportEventRegressions, settings.ReportNewEvents, settings.ReportCriticalEvents).WriteIf(shouldLog); Logger.Trace().Message("Should process: newerror={0} criticalerror={1} regression={2} new={3} critical={4}", shouldReportNewError, shouldReportCriticalError, shouldReportRegression, shouldReportNewEvent, shouldReportCriticalEvent).WriteIf(shouldLog); var request = eventNotification.Event.GetRequestInfo(); // check for known bots if the user has elected to not report them if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent)) { var botPatterns = project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns) ? project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList() : new List <string>(); var info = await _parser.ParseAsync(request.UserAgent, eventNotification.Event.ProjectId).AnyContext(); if (info != null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns)) { shouldReport = false; Logger.Info().Message("Skipping because event is from a bot \"{0}\".", request.UserAgent).WriteIf(shouldLog); } } if (!shouldReport) { continue; } var model = new EventNotificationModel(eventNotification) { ProjectName = project.Name, TotalOccurrences = totalOccurrences }; // don't send notifications in non-production mode to email addresses that are not on the outbound email list. if (Settings.Current.WebsiteMode != WebsiteMode.Production && !Settings.Current.AllowedOutboundAddresses.Contains(v => user.EmailAddress.ToLowerInvariant().Contains(v))) { Logger.Info().Message("Skipping because email is not on the outbound list and not in production mode.").WriteIf(shouldLog); continue; } Logger.Trace().Message("Sending email to {0}...", user.EmailAddress).Write(); await _mailer.SendNoticeAsync(user.EmailAddress, model).AnyContext(); emailsSent++; Logger.Trace().Message("Done sending email.").WriteIf(shouldLog); } // if we sent any emails, mark the last time a notification for this stack was sent. if (emailsSent > 0) { await _cacheClient.SetAsync(String.Concat("notify:stack-throttle:", eventNotification.Event.StackId), DateTime.UtcNow, DateTime.UtcNow.AddMinutes(15)).AnyContext(); Logger.Info().Message("Notifications sent: event={0} stack={1} count={2}", eventNotification.Event.Id, eventNotification.Event.StackId, emailsSent).WriteIf(shouldLog); } return(JobResult.Success); }
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <EventNotificationWorkItem> context) { var wi = context.QueueEntry.Value; var ev = await _eventRepository.GetByIdAsync(wi.EventId).AnyContext(); if (ev == null) { return(JobResult.FailedWithMessage($"Could not load event: {wi.EventId}")); } bool shouldLog = ev.ProjectId != Settings.Current.InternalProjectId; int emailsSent = 0; _logger.Trace().Message(() => $"Process notification: project={ev.ProjectId} event={ev.Id} stack={ev.StackId}").WriteIf(shouldLog); var project = await _projectRepository.GetByIdAsync(ev.ProjectId, o => o.Cache()).AnyContext(); if (project == null) { return(JobResult.FailedWithMessage($"Could not load project: {ev.ProjectId}.")); } _logger.Trace().Message(() => $"Loaded project: name={project.Name}").WriteIf(shouldLog); if (context.CancellationToken.IsCancellationRequested) { return(JobResult.Cancelled); } // after the first 2 occurrences, don't send a notification for the same stack more then once every 30 minutes var lastTimeSentUtc = await _cache.GetAsync <DateTime>(String.Concat("notify:stack-throttle:", ev.StackId), DateTime.MinValue).AnyContext(); if (wi.TotalOccurrences > 2 && !wi.IsRegression && lastTimeSentUtc != DateTime.MinValue && lastTimeSentUtc > SystemClock.UtcNow.AddMinutes(-30)) { _logger.Info().Message("Skipping message because of stack throttling: last sent={0} occurrences={1}", lastTimeSentUtc, wi.TotalOccurrences).WriteIf(shouldLog); return(JobResult.Success); } // don't send more than 10 notifications for a given project every 30 minutes var projectTimeWindow = TimeSpan.FromMinutes(30); string cacheKey = String.Concat("notify:project-throttle:", ev.ProjectId, "-", SystemClock.UtcNow.Floor(projectTimeWindow).Ticks); double notificationCount = await _cache.IncrementAsync(cacheKey, 1, projectTimeWindow).AnyContext(); if (notificationCount > 10 && !wi.IsRegression) { _logger.Info().Project(ev.ProjectId).Message("Skipping message because of project throttling: count={0}", notificationCount).WriteIf(shouldLog); return(JobResult.Success); } foreach (var kv in project.NotificationSettings) { var settings = kv.Value; _logger.Trace().Message(() => $"Processing notification: user={kv.Key}").WriteIf(shouldLog); var user = await _userRepository.GetByIdAsync(kv.Key).AnyContext(); if (String.IsNullOrEmpty(user?.EmailAddress)) { _logger.Error("Could not load user {0} or blank email address {1}.", kv.Key, user?.EmailAddress ?? ""); continue; } if (!user.IsEmailAddressVerified) { _logger.Info().Message("User {0} with email address {1} has not been verified.", user.Id, user.EmailAddress).WriteIf(shouldLog); continue; } if (!user.EmailNotificationsEnabled) { _logger.Info().Message("User {0} with email address {1} has email notifications disabled.", user.Id, user.EmailAddress).WriteIf(shouldLog); continue; } if (!user.OrganizationIds.Contains(project.OrganizationId)) { _logger.Error().Message("Unauthorized user: project={0} user={1} organization={2} event={3}", project.Id, kv.Key, project.OrganizationId, ev.Id).Write(); continue; } _logger.Trace().Message(() => $"Loaded user: email={user.EmailAddress}").WriteIf(shouldLog); bool isCritical = ev.IsCritical(); bool shouldReportNewError = settings.ReportNewErrors && wi.IsNew && ev.IsError(); bool shouldReportCriticalError = settings.ReportCriticalErrors && isCritical && ev.IsError(); bool shouldReportRegression = settings.ReportEventRegressions && wi.IsRegression; bool shouldReportNewEvent = settings.ReportNewEvents && wi.IsNew; bool shouldReportCriticalEvent = settings.ReportCriticalEvents && isCritical; bool shouldReport = shouldReportNewError || shouldReportCriticalError || shouldReportRegression || shouldReportNewEvent || shouldReportCriticalEvent; _logger.Trace().Message(() => $"Settings: newerror={settings.ReportNewErrors} criticalerror={settings.ReportCriticalErrors} regression={settings.ReportEventRegressions} new={settings.ReportNewEvents} critical={settings.ReportCriticalEvents}").WriteIf(shouldLog); _logger.Trace().Message(() => $"Should process: newerror={shouldReportNewError} criticalerror={shouldReportCriticalError} regression={shouldReportRegression} new={shouldReportNewEvent} critical={shouldReportCriticalEvent}").WriteIf(shouldLog); var request = ev.GetRequestInfo(); // check for known bots if the user has elected to not report them if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent)) { var botPatterns = project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns) ? project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList() : new List <string>(); var info = await _parser.ParseAsync(request.UserAgent, ev.ProjectId).AnyContext(); if (info != null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns)) { shouldReport = false; _logger.Info().Message("Skipping because event is from a bot \"{0}\".", request.UserAgent).WriteIf(shouldLog); } } if (!shouldReport) { continue; } // don't send notifications in non-production mode to email addresses that are not on the outbound email list. if (Settings.Current.WebsiteMode != WebsiteMode.Production && !Settings.Current.AllowedOutboundAddresses.Contains(v => user.EmailAddress.ToLowerInvariant().Contains(v))) { _logger.Info().Message("Skipping because email is not on the outbound list and not in production mode.").WriteIf(shouldLog); continue; } _logger.Trace("Sending email to {0}...", user.EmailAddress); await _mailer.SendEventNoticeAsync(user, ev, project, wi.IsNew, wi.IsRegression, wi.TotalOccurrences).AnyContext(); emailsSent++; _logger.Trace().Message(() => "Done sending email.").WriteIf(shouldLog); } // if we sent any emails, mark the last time a notification for this stack was sent. if (emailsSent > 0) { await _cache.SetAsync(String.Concat("notify:stack-throttle:", ev.StackId), SystemClock.UtcNow, SystemClock.UtcNow.AddMinutes(15)).AnyContext(); _logger.Info().Message("Notifications sent: event={0} stack={1} count={2}", ev.Id, ev.StackId, emailsSent).WriteIf(shouldLog); } return(JobResult.Success); }