protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <WebHookNotification> context)
{
WebHookNotification body = context.QueueEntry.Value;
bool shouldLog = body.ProjectId != Settings.Current.InternalProjectId;
Logger.Trace().Project(body.ProjectId).Message("Process web hook call: id={0} project={1} url={2}", context.QueueEntry.Id, body.ProjectId, body.Url).WriteIf(shouldLog);
var client = new HttpClient();
try {
var response = await client.PostAsJsonAsync(body.Url, body.Data.ToJson(Formatting.Indented, _jsonSerializerSettings), context.CancellationToken).AnyContext();
if (response.StatusCode == HttpStatusCode.Gone)
{
Logger.Warn().Project(body.ProjectId).Message("Deleting web hook: org={0} project={1} url={2}", body.OrganizationId, body.ProjectId, body.Url).Write();
await _webHookRepository.RemoveByUrlAsync(body.Url).AnyContext();
}
Logger.Info().Project(body.ProjectId).Message("Web hook POST complete: status={0} org={1} project={2} url={3}", response.StatusCode, body.OrganizationId, body.ProjectId, body.Url).WriteIf(shouldLog);
} catch (Exception ex) {
return(JobResult.FromException(ex));
}
return(JobResult.Success);
}
protected override Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <PingRequest> context)
{
RunCount++;
_logger.Info(() => $"Got {RunCount.ToOrdinal()} ping. Sending pong!");
if (RandomData.GetBool(1))
{
throw new ApplicationException("Boom!");
}
return(Task.FromResult(JobResult.Success));
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <MailMessage> context)
{
Logger.Trace().Message("Processing message '{0}'.", context.QueueEntry.Id).Write();
try {
await _mailSender.SendAsync(context.QueueEntry.Value).AnyContext();
Logger.Info().Message("Sent message: to={0} subject=\"{1}\"", context.QueueEntry.Value.To, context.QueueEntry.Value.Subject).Write();
} catch (Exception ex) {
return(JobResult.FromException(ex));
}
return(JobResult.Success);
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <PingRequest> context)
{
RunCount++;
Console.WriteLine("Pong!");
if (RandomData.GetBool(80))
{
await context.QueueEntry.CompleteAsync().AnyContext();
}
else if (RandomData.GetBool(80))
{
await context.QueueEntry.AbandonAsync().AnyContext();
}
return(JobResult.Success);
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <EventUserDescription> context)
{
Logger.Trace().Message("Processing user description: id={0}", context.QueueEntry.Id).Write();
try {
await ProcessUserDescriptionAsync(context.QueueEntry.Value).AnyContext();
Logger.Info().Message("Processed user description: id={0}", context.QueueEntry.Id).Write();
} catch (DocumentNotFoundException ex) {
Logger.Error().Exception(ex).Message("An event with this reference id \"{0}\" has not been processed yet or was deleted. Queue Id: {1}", ex.Id, context.QueueEntry.Id).Write();
return(JobResult.FromException(ex));
} catch (Exception ex) {
Logger.Error().Exception(ex).Message("An error occurred while processing the EventUserDescription '{0}': {1}", context.QueueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex));
}
return(JobResult.Success);
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <ValuesPost> context)
{
var result = await _storage.GetFileContentsAsync(context.QueueEntry.Value.FilePath);
Guid guid;
if (!Guid.TryParse(result, out guid))
{
await _metricsClient.CounterAsync("values.errors");
await context.QueueEntry.AbandonAsync();
await _storage.DeleteFileAsync(context.QueueEntry.Value.FilePath, context.CancellationToken);
return(JobResult.FailedWithMessage($"Unable to retrieve values data '{context.QueueEntry.Value.FilePath}'."));
}
await _metricsClient.CounterAsync("values.dequeued");
await _cacheClient.SetAsync(context.QueueEntry.Value.FilePath, guid);
await _metricsClient.CounterAsync("values.processed");
_logger.Info().Message("Processing post: id={0} path={1}", context.QueueEntry.Id, context.QueueEntry.Value.FilePath).Write();
await _publisher.PublishAsync(new EntityChanged {
ChangeType = ChangeType.Added,
Id = context.QueueEntry.Value.FilePath,
Data = new DataDictionary {
{ "Value", guid }
}
});
await context.QueueEntry.CompleteAsync();
await _storage.DeleteFileAsync(context.QueueEntry.Value.FilePath, context.CancellationToken);
return(JobResult.Success);
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <SampleQueueWorkItem> context)
{
await _metrics.CounterAsync("dequeued").AnyContext();
if (RandomData.GetBool(10))
{
await _metrics.CounterAsync("errors").AnyContext();
throw new ApplicationException("Boom!");
}
if (RandomData.GetBool(10))
{
await _metrics.CounterAsync("abandoned").AnyContext();
return(JobResult.FailedWithMessage("Abandoned"));
}
await _metrics.CounterAsync("completed").AnyContext();
return(JobResult.Success);
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <EventNotificationWorkItem> context)
{
var eventModel = await _eventRepository.GetByIdAsync(context.QueueEntry.Value.EventId).AnyContext();
if (eventModel == null)
{
return(JobResult.FailedWithMessage($"Could not load event: {context.QueueEntry.Value.EventId}"));
}
var eventNotification = new EventNotification(context.QueueEntry.Value, eventModel);
bool shouldLog = eventNotification.Event.ProjectId != Settings.Current.InternalProjectId;
int emailsSent = 0;
Logger.Trace().Message("Process notification: project={0} event={1} stack={2}", eventNotification.Event.ProjectId, eventNotification.Event.Id, eventNotification.Event.StackId).WriteIf(shouldLog);
var project = await _projectRepository.GetByIdAsync(eventNotification.Event.ProjectId, true).AnyContext();
if (project == null)
{
return(JobResult.FailedWithMessage($"Could not load project: {eventNotification.Event.ProjectId}."));
}
Logger.Trace().Message($"Loaded project: name={project.Name}").WriteIf(shouldLog);
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, true).AnyContext();
if (organization == null)
{
return(JobResult.FailedWithMessage($"Could not load organization: {project.OrganizationId}"));
}
Logger.Trace().Message($"Loaded organization: {organization.Name}").WriteIf(shouldLog);
var stack = await _stackRepository.GetByIdAsync(eventNotification.Event.StackId).AnyContext();
if (stack == null)
{
return(JobResult.FailedWithMessage($"Could not load stack: {eventNotification.Event.StackId}"));
}
if (!organization.HasPremiumFeatures)
{
Logger.Info().Message("Skipping \"{0}\" because organization \"{1}\" does not have premium features.", eventNotification.Event.Id, eventNotification.Event.OrganizationId).WriteIf(shouldLog);
return(JobResult.Success);
}
if (stack.DisableNotifications || stack.IsHidden)
{
Logger.Info().Message("Skipping \"{0}\" because stack \"{1}\" notifications are disabled or stack is hidden.", eventNotification.Event.Id, eventNotification.Event.StackId).WriteIf(shouldLog);
return(JobResult.Success);
}
if (context.CancellationToken.IsCancellationRequested)
{
return(JobResult.Cancelled);
}
Logger.Trace().Message("Loaded stack: title={0}", stack.Title).WriteIf(shouldLog);
int totalOccurrences = stack.TotalOccurrences;
// after the first 2 occurrences, don't send a notification for the same stack more then once every 30 minutes
var lastTimeSentUtc = await _cacheClient.GetAsync <DateTime>(String.Concat("notify:stack-throttle:", eventNotification.Event.StackId), DateTime.MinValue).AnyContext();
if (totalOccurrences > 2 &&
!eventNotification.IsRegression &&
lastTimeSentUtc != DateTime.MinValue &&
lastTimeSentUtc > DateTime.UtcNow.AddMinutes(-30))
{
Logger.Info().Message("Skipping message because of stack throttling: last sent={0} occurrences={1}", lastTimeSentUtc, totalOccurrences).WriteIf(shouldLog);
return(JobResult.Success);
}
// don't send more than 10 notifications for a given project every 30 minutes
var projectTimeWindow = TimeSpan.FromMinutes(30);
string cacheKey = String.Concat("notify:project-throttle:", eventNotification.Event.ProjectId, "-", DateTime.UtcNow.Floor(projectTimeWindow).Ticks);
long notificationCount = await _cacheClient.IncrementAsync(cacheKey, 1, projectTimeWindow).AnyContext();
if (notificationCount > 10 && !eventNotification.IsRegression)
{
Logger.Info().Project(eventNotification.Event.ProjectId).Message("Skipping message because of project throttling: count={0}", notificationCount).WriteIf(shouldLog);
return(JobResult.Success);
}
if (context.CancellationToken.IsCancellationRequested)
{
return(JobResult.Cancelled);
}
foreach (var kv in project.NotificationSettings)
{
var settings = kv.Value;
Logger.Trace().Message("Processing notification: user={0}", kv.Key).WriteIf(shouldLog);
var user = await _userRepository.GetByIdAsync(kv.Key).AnyContext();
if (String.IsNullOrEmpty(user?.EmailAddress))
{
Logger.Error().Message("Could not load user {0} or blank email address {1}.", kv.Key, user != null ? user.EmailAddress : "").Write();
continue;
}
if (!user.IsEmailAddressVerified)
{
Logger.Info().Message("User {0} with email address {1} has not been verified.", kv.Key, user != null ? user.EmailAddress : "").WriteIf(shouldLog);
continue;
}
if (!user.EmailNotificationsEnabled)
{
Logger.Info().Message("User {0} with email address {1} has email notifications disabled.", kv.Key, user != null ? user.EmailAddress : "").WriteIf(shouldLog);
continue;
}
if (!user.OrganizationIds.Contains(project.OrganizationId))
{
Logger.Error().Message("Unauthorized user: project={0} user={1} organization={2} event={3}", project.Id, kv.Key, project.OrganizationId, eventNotification.Event.Id).Write();
continue;
}
Logger.Trace().Message("Loaded user: email={0}", user.EmailAddress).WriteIf(shouldLog);
bool shouldReportNewError = settings.ReportNewErrors && eventNotification.IsNew && eventNotification.Event.IsError();
bool shouldReportCriticalError = settings.ReportCriticalErrors && eventNotification.IsCritical && eventNotification.Event.IsError();
bool shouldReportRegression = settings.ReportEventRegressions && eventNotification.IsRegression;
bool shouldReportNewEvent = settings.ReportNewEvents && eventNotification.IsNew;
bool shouldReportCriticalEvent = settings.ReportCriticalEvents && eventNotification.IsCritical;
bool shouldReport = shouldReportNewError || shouldReportCriticalError || shouldReportRegression || shouldReportNewEvent || shouldReportCriticalEvent;
Logger.Trace().Message("Settings: newerror={0} criticalerror={1} regression={2} new={3} critical={4}",
settings.ReportNewErrors, settings.ReportCriticalErrors,
settings.ReportEventRegressions, settings.ReportNewEvents, settings.ReportCriticalEvents).WriteIf(shouldLog);
Logger.Trace().Message("Should process: newerror={0} criticalerror={1} regression={2} new={3} critical={4}",
shouldReportNewError, shouldReportCriticalError,
shouldReportRegression, shouldReportNewEvent, shouldReportCriticalEvent).WriteIf(shouldLog);
var request = eventNotification.Event.GetRequestInfo();
// check for known bots if the user has elected to not report them
if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent))
{
var botPatterns = project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns)
? project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList()
: new List <string>();
var info = await _parser.ParseAsync(request.UserAgent, eventNotification.Event.ProjectId).AnyContext();
if (info != null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns))
{
shouldReport = false;
Logger.Info().Message("Skipping because event is from a bot \"{0}\".", request.UserAgent).WriteIf(shouldLog);
}
}
if (!shouldReport)
{
continue;
}
var model = new EventNotificationModel(eventNotification)
{
ProjectName = project.Name,
TotalOccurrences = totalOccurrences
};
// don't send notifications in non-production mode to email addresses that are not on the outbound email list.
if (Settings.Current.WebsiteMode != WebsiteMode.Production &&
!Settings.Current.AllowedOutboundAddresses.Contains(v => user.EmailAddress.ToLowerInvariant().Contains(v)))
{
Logger.Info().Message("Skipping because email is not on the outbound list and not in production mode.").WriteIf(shouldLog);
continue;
}
Logger.Trace().Message("Sending email to {0}...", user.EmailAddress).Write();
await _mailer.SendNoticeAsync(user.EmailAddress, model).AnyContext();
emailsSent++;
Logger.Trace().Message("Done sending email.").WriteIf(shouldLog);
}
// if we sent any emails, mark the last time a notification for this stack was sent.
if (emailsSent > 0)
{
await _cacheClient.SetAsync(String.Concat("notify:stack-throttle:", eventNotification.Event.StackId), DateTime.UtcNow, DateTime.UtcNow.AddMinutes(15)).AnyContext();
Logger.Info().Message("Notifications sent: event={0} stack={1} count={2}", eventNotification.Event.Id, eventNotification.Event.StackId, emailsSent).WriteIf(shouldLog);
}
return(JobResult.Success);
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <SampleQueueWorkItem> context)
{
await _metrics.CounterAsync("completed").AnyContext();
return(JobResult.Success);
}
protected override async Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <EventPost> context)
{
var queueEntry = context.QueueEntry;
EventPostInfo eventPostInfo = await _storage.GetEventPostAndSetActiveAsync(queueEntry.Value.FilePath, context.CancellationToken).AnyContext();
if (eventPostInfo == null)
{
await queueEntry.AbandonAsync().AnyContext();
await _storage.SetNotActiveAsync(queueEntry.Value.FilePath).AnyContext();
return(JobResult.FailedWithMessage($"Unable to retrieve post data '{queueEntry.Value.FilePath}'."));
}
bool isInternalProject = eventPostInfo.ProjectId == Settings.Current.InternalProjectId;
Logger.Info().Message("Processing post: id={0} path={1} project={2} ip={3} v={4} agent={5}", queueEntry.Id, queueEntry.Value.FilePath, eventPostInfo.ProjectId, eventPostInfo.IpAddress, eventPostInfo.ApiVersion, eventPostInfo.UserAgent).WriteIf(!isInternalProject);
List <PersistentEvent> events = null;
try {
_metricsClient.Time(() => {
events = ParseEventPost(eventPostInfo);
Logger.Info().Message("Parsed {0} events for post: id={1}", events.Count, queueEntry.Id).WriteIf(!isInternalProject);
}, MetricNames.PostsParsingTime);
await _metricsClient.CounterAsync(MetricNames.PostsParsed).AnyContext();
await _metricsClient.GaugeAsync(MetricNames.PostsEventCount, events.Count).AnyContext();
} catch (Exception ex) {
await queueEntry.AbandonAsync().AnyContext();
await _metricsClient.CounterAsync(MetricNames.PostsParseErrors).AnyContext();
await _storage.SetNotActiveAsync(queueEntry.Value.FilePath).AnyContext();
Logger.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex, $"An error occurred while processing the EventPost '{queueEntry.Id}': {ex.Message}"));
}
if (!events.Any() || context.CancellationToken.IsCancellationRequested)
{
await queueEntry.AbandonAsync().AnyContext();
await _storage.SetNotActiveAsync(queueEntry.Value.FilePath).AnyContext();
return(!events.Any() ? JobResult.Success : JobResult.Cancelled);
}
int eventsToProcess = events.Count;
bool isSingleEvent = events.Count == 1;
if (!isSingleEvent)
{
var project = await _projectRepository.GetByIdAsync(eventPostInfo.ProjectId, true).AnyContext();
if (project == null)
{
// NOTE: This could archive the data for a project that no longer exists.
Logger.Error().Project(eventPostInfo.ProjectId).Message($"Unable to process EventPost \"{queueEntry.Value.FilePath}\": Unable to load project: {eventPostInfo.ProjectId}").Write();
await CompleteEntryAsync(queueEntry, eventPostInfo, DateTime.UtcNow).AnyContext();
return(JobResult.Success);
}
// Don't process all the events if it will put the account over its limits.
eventsToProcess = await _organizationRepository.GetRemainingEventLimitAsync(project.OrganizationId).AnyContext();
// Add 1 because we already counted 1 against their limit when we received the event post.
if (eventsToProcess < Int32.MaxValue)
{
eventsToProcess += 1;
}
// Increment by count - 1 since we already incremented it by 1 in the OverageHandler.
await _organizationRepository.IncrementUsageAsync(project.OrganizationId, false, events.Count - 1).AnyContext();
}
var errorCount = 0;
var created = DateTime.UtcNow;
try {
events.ForEach(e => e.CreatedUtc = created);
var results = await _eventPipeline.RunAsync(events.Take(eventsToProcess).ToList()).AnyContext();
Logger.Info().Message("Ran {0} events through the pipeline: id={1} project={2} success={3} error={4}", results.Count, queueEntry.Id, eventPostInfo.ProjectId, results.Count(r => r.IsProcessed), results.Count(r => r.HasError)).WriteIf(!isInternalProject);
foreach (var eventContext in results)
{
if (eventContext.IsCancelled)
{
continue;
}
if (!eventContext.HasError)
{
continue;
}
Logger.Error().Exception(eventContext.Exception).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, eventContext.ErrorMessage).Write();
if (eventContext.Exception is ValidationException)
{
continue;
}
errorCount++;
if (!isSingleEvent)
{
// Put this single event back into the queue so we can retry it separately.
await _queue.EnqueueAsync(new EventPostInfo {
ApiVersion = eventPostInfo.ApiVersion,
Data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(eventContext.Event)),
IpAddress = eventPostInfo.IpAddress,
MediaType = eventPostInfo.MediaType,
CharSet = eventPostInfo.CharSet,
ProjectId = eventPostInfo.ProjectId,
UserAgent = eventPostInfo.UserAgent
}, _storage, false, context.CancellationToken).AnyContext();
}
}
} catch (Exception ex) {
Logger.Error().Exception(ex).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, ex.Message).Write();
if (ex is ArgumentException || ex is DocumentNotFoundException)
{
await queueEntry.CompleteAsync().AnyContext();
}
else
{
errorCount++;
}
}
if (isSingleEvent && errorCount > 0)
{
await queueEntry.AbandonAsync().AnyContext();
await _storage.SetNotActiveAsync(queueEntry.Value.FilePath).AnyContext();
}
else
{
await CompleteEntryAsync(queueEntry, eventPostInfo, created).AnyContext();
}
return(JobResult.Success);
}