public override void Process(EventContext ctx) { _stats.Counter(StatNames.EventsProcessed); if (ctx.Organization.PlanId != BillingManager.FreePlan.Id) { _stats.Counter(StatNames.EventsPaidProcessed); } }
protected override void Run(EventContext context, IEnumerable <Type> actionTypes) { _statsClient.Counter(StatNames.EventsSubmitted); try { if (!String.IsNullOrEmpty(context.Event.Id)) { throw new ArgumentException("Event Id should not be populated."); } if (String.IsNullOrEmpty(context.Event.ProjectId)) { throw new ArgumentException("ProjectId must be populated on the Event."); } if (context.Project == null) { context.Project = _projectRepository.GetById(context.Event.ProjectId, true); } if (context.Project == null) { throw new InvalidOperationException(String.Format("Unable to load project \"{0}\"", context.Event.ProjectId)); } if (String.IsNullOrEmpty(context.Event.OrganizationId)) { context.Event.OrganizationId = context.Project.OrganizationId; } if (context.Organization == null) { context.Organization = _organizationRepository.GetById(context.Event.OrganizationId, true); } if (context.Organization == null) { throw new InvalidOperationException(String.Format("Unable to load organization \"{0}\"", context.Event.OrganizationId)); } // load organization settings into the context foreach (var key in context.Organization.Data.Keys) { context.SetProperty(key, context.Organization.Data[key]); } // load project settings into the context, overriding any organization settings with the same name foreach (var key in context.Project.Data.Keys) { context.SetProperty(key, context.Project.Data[key]); } _statsClient.Time(() => base.Run(context, actionTypes), StatNames.EventsProcessingTime); } catch (Exception) { _statsClient.Counter(StatNames.EventsProcessErrors); throw; } }
public async Task <IHttpActionResult> SetUserDescription(string referenceId, UserDescription description, string projectId = null) { _statsClient.Counter(StatNames.EventsUserDescriptionSubmitted); if (String.IsNullOrEmpty(referenceId)) { return(NotFound()); } if (description == null) { return(BadRequest("Description must be specified.")); } var result = _userDescriptionValidator.Validate(description); if (!result.IsValid) { return(BadRequest(result.Errors.ToErrorMessage())); } if (projectId == null) { projectId = User.GetDefaultProjectId(); } // must have a project id if (String.IsNullOrEmpty(projectId)) { return(BadRequest("No project id specified and no default project was found.")); } var project = _projectRepository.GetById(projectId, true); if (project == null || !User.GetOrganizationIds().ToList().Contains(project.OrganizationId)) { return(NotFound()); } var eventUserDescription = Mapper.Map <UserDescription, EventUserDescription>(description); eventUserDescription.ProjectId = projectId; eventUserDescription.ReferenceId = referenceId; await _eventUserDescriptionQueue.EnqueueAsync(eventUserDescription); _statsClient.Counter(StatNames.EventsUserDescriptionQueued); return(StatusCode(HttpStatusCode.Accepted)); }
protected async override Task <JobResult> RunInternalAsync() { Log.Info().Message("Process user description job starting").Write(); int totalUserDescriptionsProcessed = 0; int totalUserDescriptionsToProcess = Context.GetWorkItemLimit(); while (!CancelPending && (totalUserDescriptionsToProcess == -1 || totalUserDescriptionsProcessed < totalUserDescriptionsToProcess)) { QueueEntry <EventUserDescription> queueEntry = null; try { queueEntry = await _queue.DequeueAsync(); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventUserDescription: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { continue; } _statsClient.Counter(StatNames.EventsUserDescriptionDequeued); Log.Info().Message("Processing EventUserDescription '{0}'.", queueEntry.Id).Write(); try { ProcessUserDescription(queueEntry.Value); totalUserDescriptionsProcessed++; _statsClient.Counter(StatNames.EventsUserDescriptionProcessed); } catch (DocumentNotFoundException ex) { _statsClient.Counter(StatNames.EventsUserDescriptionErrors); queueEntry.AbandonAsync().Wait(); Log.Error().Exception(ex).Message("An event with this reference id \"{0}\" has not been processed yet or was deleted. Queue Id: {1}", ex.Id, queueEntry.Id).Write(); continue; } catch (Exception ex) { _statsClient.Counter(StatNames.EventsUserDescriptionErrors); queueEntry.AbandonAsync().Wait(); // TODO: Add the EventUserDescription to the logged exception. Log.Error().Exception(ex).Message("An error occurred while processing the EventUserDescription '{0}': {1}", queueEntry.Id, ex.Message).Write(); return(JobResult.FromException(ex)); } await queueEntry.CompleteAsync(); } return(JobResult.Success); }
protected override Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (!IsEventPost(request)) { return(base.SendAsync(request, cancellationToken)); } if (_cacheClient.TryGet("ApiDisabled", false)) { return(CreateResponse(request, HttpStatusCode.ServiceUnavailable, "Service Unavailable")); } var project = request.GetDefaultProject(); if (project == null) { return(CreateResponse(request, HttpStatusCode.Unauthorized, "Unauthorized")); } bool tooBig = false; if (request.Content != null && request.Content.Headers != null) { long size = request.Content.Headers.ContentLength.GetValueOrDefault(); _statsClient.Gauge(StatNames.PostsSize, size); if (size > Settings.Current.MaximumEventPostSize) { Log.Warn().Message("Event submission discarded for being too large: {0}", size).Project(project.Id).Write(); _statsClient.Counter(StatNames.PostsDiscarded); tooBig = true; } } bool overLimit = _organizationRepository.IncrementUsage(project.OrganizationId, tooBig); // block large submissions, but return success status code so the client doesn't keep sending them if (tooBig) { return(CreateResponse(request, HttpStatusCode.Accepted, "Event submission discarded for being too large.")); } if (overLimit) { _statsClient.Counter(StatNames.PostsBlocked); return(CreateResponse(request, HttpStatusCode.PaymentRequired, "Event limit exceeded.")); } return(base.SendAsync(request, cancellationToken)); }
public override void Process(ErrorPipelineContext ctx) { var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId); if (organization == null) { return; } _stats.Counter(StatNames.ErrorsProcessed); if (organization.PlanId != BillingManager.FreePlan.Id) { _stats.Counter(StatNames.ErrorsPaidProcessed); } }
private void QueueMessage(System.Net.Mail.MailMessage message) { CleanAddresses(message); _queue.Enqueue(message.ToMailMessage()); _statsClient.Counter(StatNames.EmailsQueued); }
private async Task QueueMessage(MailMessage message) { CleanAddresses(message); await _queue.EnqueueAsync(message.ToMailMessage()); _statsClient.Counter(StatNames.EmailsQueued); }
protected async override Task <JobResult> RunInternalAsync(CancellationToken token) { Log.Info().Message("Process user description job starting").Write(); QueueEntry <EventUserDescription> queueEntry = null; try { queueEntry = _queue.Dequeue(); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventUserDescription: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { return(JobResult.Success); } _statsClient.Counter(StatNames.EventsUserDescriptionDequeued); Log.Info().Message("Processing EventUserDescription '{0}'.", queueEntry.Id).Write(); try { ProcessUserDescription(queueEntry.Value); _statsClient.Counter(StatNames.EventsUserDescriptionProcessed); } catch (DocumentNotFoundException ex) { _statsClient.Counter(StatNames.EventsUserDescriptionErrors); queueEntry.Abandon(); Log.Error().Exception(ex).Message("An event with this reference id \"{0}\" has not been processed yet or was deleted. Queue Id: {1}", ex.Id, queueEntry.Id).Write(); return(JobResult.FromException(ex)); } catch (Exception ex) { _statsClient.Counter(StatNames.EventsUserDescriptionErrors); queueEntry.Abandon(); // TODO: Add the EventUserDescription to the logged exception. Log.Error().Exception(ex).Message("An error occurred while processing the EventUserDescription '{0}': {1}", queueEntry.Id, ex.Message).Write(); return(JobResult.FromException(ex)); } queueEntry.Complete(); return(JobResult.Success); }
protected async override Task <JobResult> RunInternalAsync() { Log.Info().Message("Process email message job starting").Write(); int totalEmailsProcessed = 0; int totalEmailsToProcess = Context.GetWorkItemLimit(); while (!CancelPending && (totalEmailsToProcess == -1 || totalEmailsProcessed < totalEmailsToProcess)) { QueueEntry <MailMessage> queueEntry = null; try { queueEntry = await _queue.DequeueAsync(); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next MailMessageNotification: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { continue; } _statsClient.Counter(StatNames.EmailsDequeued); Log.Info().Message("Processing MailMessageNotification '{0}'.", queueEntry.Id).Write(); try { await _mailSender.SendAsync(queueEntry.Value); totalEmailsProcessed++; _statsClient.Counter(StatNames.EmailsSent); } catch (Exception ex) { _statsClient.Counter(StatNames.EmailsSendErrors); queueEntry.AbandonAsync().Wait(); Log.Error().Exception(ex).Message("Error sending message '{0}': {1}", queueEntry.Id, ex.Message).Write(); } await queueEntry.CompleteAsync(); } return(JobResult.Success); }
protected async override Task <JobResult> RunInternalAsync(CancellationToken token) { Log.Info().Message("Process email message job starting").Write(); QueueEntry <MailMessage> queueEntry = null; try { queueEntry = _queue.Dequeue(); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next MailMessageNotification: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { return(JobResult.Success); } _statsClient.Counter(StatNames.EmailsDequeued); Log.Info().Message("Processing MailMessageNotification '{0}'.", queueEntry.Id).Write(); try { await _mailSender.SendAsync(queueEntry.Value); _statsClient.Counter(StatNames.EmailsSent); } catch (Exception ex) { _statsClient.Counter(StatNames.EmailsSendErrors); queueEntry.Abandon(); Log.Error().Exception(ex).Message("Error sending message '{0}': {1}", queueEntry.Id, ex.Message).Write(); } queueEntry.Complete(); return(JobResult.Success); }
protected async override Task <JobResult> RunInternalAsync(CancellationToken token) { QueueEntry <MailMessage> queueEntry = null; try { queueEntry = _queue.Dequeue(); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { return(JobResult.Success); } _statsClient.Counter(StatNames.EmailsDequeued); Log.Trace().Message("Processing message '{0}'.", queueEntry.Id).Write(); try { await _mailSender.SendAsync(queueEntry.Value); Log.Info().Message("Sent message: to={0} subject=\"{1}\"", queueEntry.Value.To, queueEntry.Value.Subject).Write(); _statsClient.Counter(StatNames.EmailsSent); } catch (Exception ex) { _statsClient.Counter(StatNames.EmailsSendErrors); queueEntry.Abandon(); Log.Error().Exception(ex).Message("Error sending message: id={0} error={1}", queueEntry.Id, ex.Message).Write(); } queueEntry.Complete(); return(JobResult.Success); }
public void Run(PersistentEvent ev) { _statsClient.Counter(StatNames.EventsSubmitted); try { _statsClient.Time(() => { if (String.IsNullOrEmpty(ev.ProjectId)) throw new ArgumentException("ProjectId must be populated on the Event."); var project = _projectRepository.GetById(ev.ProjectId, true); if (project == null) throw new InvalidOperationException(String.Format("Unable to load project \"{0}\"", ev.ProjectId)); if (String.IsNullOrEmpty(ev.OrganizationId)) ev.OrganizationId = project.OrganizationId; var ctx = new EventContext(ev) { Organization = _organizationRepository.GetById(ev.OrganizationId, true), Project = project }; if (ctx.Organization == null) throw new InvalidOperationException(String.Format("Unable to load organization \"{0}\"", ev.OrganizationId)); // load organization settings into the context foreach (var key in ctx.Organization.Data.Keys) ctx.SetProperty(key, ctx.Organization.Data[key]); // load project settings into the context, overriding any organization settings with the same name foreach (var key in ctx.Project.Data.Keys) ctx.SetProperty(key, ctx.Project.Data[key]); Run(ctx); }, StatNames.EventsProcessingTime); } catch (Exception ex) { _statsClient.Counter(StatNames.EventsProcessErrors); throw; } }
public async Task <IHttpActionResult> Post([NakedBody] byte[] data, string projectId = null, int version = 1, [UserAgent] string userAgent = null) { _statsClient.Counter(StatNames.PostsSubmitted); if (projectId == null) { projectId = User.GetProjectId(); } // must have a project id if (String.IsNullOrEmpty(projectId)) { return(StatusCode(HttpStatusCode.Unauthorized)); } // TODO: Add a check to see if the project id is over it's project limits. If it is, then turn off the client. bool isCompressed = Request.Content.Headers.ContentEncoding.Contains("gzip"); if (!isCompressed) { data = data.Compress(); } await _eventPostQueue.EnqueueAsync(new EventPost { MediaType = Request.Content.Headers.ContentType.MediaType, CharSet = Request.Content.Headers.ContentType.CharSet, ProjectId = projectId, UserAgent = userAgent, ApiVersion = version, Data = data }); _statsClient.Counter(StatNames.PostsQueued); return(StatusCode(HttpStatusCode.Accepted)); }
public override void EventProcessing(EventContext context) { if (Settings.Current.WebsiteMode == WebsiteMode.Dev) { return; } var project = _projectRepository.GetById(context.Event.ProjectId); if (project == null || !project.DeleteBotDataEnabled) { return; } // Throttle errors by client ip address to no more than X every 5 minutes. var ri = context.Event.GetRequestInfo(); if (ri == null || String.IsNullOrEmpty(ri.ClientIpAddress)) { return; } string throttleCacheKey = String.Concat("bot:", ri.ClientIpAddress, ":", DateTime.Now.Floor(_throttlingPeriod).Ticks); var requestCount = _cacheClient.Get <int?>(throttleCacheKey); if (requestCount != null) { _cacheClient.Increment(throttleCacheKey, 1); requestCount++; } else { _cacheClient.Set(throttleCacheKey, 1, _throttlingPeriod); requestCount = 1; } if (requestCount < Settings.Current.BotThrottleLimit) { return; } _stats.Counter(StatNames.EventsBotThrottleTriggered); Log.Info().Message("Bot throttle triggered. IP: {0} Time: {1} Project: {2}", ri.ClientIpAddress, DateTime.Now.Floor(_throttlingPeriod), context.Event.ProjectId).Project(context.Event.ProjectId).Write(); // the throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system Task.Run(() => _eventRepository.HideAllByClientIpAndDateAsync(context.Event.OrganizationId, ri.ClientIpAddress, DateTime.Now.Floor(_throttlingPeriod), DateTime.Now.Ceiling(_throttlingPeriod))); context.IsCancelled = true; }
protected async override Task <JobResult> RunInternalAsync() { Log.Info().Message("Process events job starting").Write(); int totalEventsProcessed = 0; int totalEventsToProcess = Context.GetWorkItemLimit(); while (!CancelPending && (totalEventsToProcess == -1 || totalEventsProcessed < totalEventsToProcess)) { QueueEntry <EventPost> queueEntry = null; try { queueEntry = await _queue.DequeueAsync(); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { continue; } _statsClient.Counter(StatNames.PostsDequeued); Log.Info().Message("Processing EventPost '{0}'.", queueEntry.Id).Write(); List <PersistentEvent> events = null; try { _statsClient.Time(() => { events = ParseEventPost(queueEntry.Value); }, StatNames.PostsParsingTime); _statsClient.Counter(StatNames.PostsParsed); _statsClient.Gauge(StatNames.PostsBatchSize, events.Count); } catch (Exception ex) { _statsClient.Counter(StatNames.PostsParseErrors); queueEntry.AbandonAsync().Wait(); // TODO: Add the EventPost to the logged exception. Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write(); continue; } if (events == null) { queueEntry.AbandonAsync().Wait(); continue; } int eventsToProcess = events.Count; bool isSingleEvent = events.Count == 1; if (!isSingleEvent) { var project = _projectRepository.GetById(queueEntry.Value.ProjectId, true); // Don't process all the events if it will put the account over its limits. eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId); // Add 1 because we already counted 1 against their limit when we received the event post. if (eventsToProcess < Int32.MaxValue) { eventsToProcess += 1; } // Increment by count - 1 since we already incremented it by 1 in the OverageHandler. _organizationRepository.IncrementUsage(project.OrganizationId, events.Count - 1); } int errorCount = 0; foreach (PersistentEvent ev in events.Take(eventsToProcess)) { try { _eventPipeline.Run(ev); totalEventsProcessed++; if (totalEventsToProcess > 0 && totalEventsProcessed >= totalEventsToProcess) { break; } } catch (ValidationException ex) { Log.Error().Exception(ex).Project(queueEntry.Value.ProjectId).Message("Event validation error occurred: {0}", ex.Message).Write(); } catch (Exception ex) { Log.Error().Exception(ex).Project(queueEntry.Value.ProjectId).Message("Error while processing event: {0}", ex.Message).Write(); if (!isSingleEvent) { // Put this single event back into the queue so we can retry it separately. _queue.EnqueueAsync(new EventPost { Data = Encoding.UTF8.GetBytes(ev.ToJson()).Compress(), ContentEncoding = "gzip", ProjectId = ev.ProjectId, CharSet = "utf-8", MediaType = "application/json", }).Wait(); } errorCount++; } } if (isSingleEvent && errorCount > 0) { queueEntry.AbandonAsync().Wait(); } else { queueEntry.CompleteAsync().Wait(); } } return(JobResult.Success); }
protected async override Task <JobResult> RunInternalAsync(CancellationToken token) { QueueEntry <EventPost> queueEntry = null; try { queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1)); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { return(JobResult.Success); } if (token.IsCancellationRequested) { queueEntry.Abandon(); return(JobResult.Cancelled); } EventPostInfo eventPostInfo = _storage.GetEventPostAndSetActive(queueEntry.Value.FilePath); if (eventPostInfo == null) { queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); return(JobResult.FailedWithMessage(String.Format("Unable to retrieve post data '{0}'.", queueEntry.Value.FilePath))); } bool isInternalProject = eventPostInfo.ProjectId == Settings.Current.InternalProjectId; _statsClient.Counter(StatNames.PostsDequeued); Log.Info().Message("Processing post: id={0} path={1} project={2} ip={3} v={4} agent={5}", queueEntry.Id, queueEntry.Value.FilePath, eventPostInfo.ProjectId, eventPostInfo.IpAddress, eventPostInfo.ApiVersion, eventPostInfo.UserAgent).WriteIf(!isInternalProject); List <PersistentEvent> events = null; try { _statsClient.Time(() => { events = ParseEventPost(eventPostInfo); Log.Info().Message("Parsed {0} events for post: id={1}", events.Count, queueEntry.Id).WriteIf(!isInternalProject); }, StatNames.PostsParsingTime); _statsClient.Counter(StatNames.PostsParsed); _statsClient.Gauge(StatNames.PostsEventCount, events.Count); } catch (Exception ex) { _statsClient.Counter(StatNames.PostsParseErrors); queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write(); return(JobResult.FromException(ex, String.Format("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message))); } if (token.IsCancellationRequested) { queueEntry.Abandon(); return(JobResult.Cancelled); } if (events == null) { queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); return(JobResult.Success); } int eventsToProcess = events.Count; bool isSingleEvent = events.Count == 1; if (!isSingleEvent) { var project = _projectRepository.GetById(eventPostInfo.ProjectId, true); // Don't process all the events if it will put the account over its limits. eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId); // Add 1 because we already counted 1 against their limit when we received the event post. if (eventsToProcess < Int32.MaxValue) { eventsToProcess += 1; } // Increment by count - 1 since we already incremented it by 1 in the OverageHandler. _organizationRepository.IncrementUsage(project.OrganizationId, false, events.Count - 1); } if (events == null) { queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); return(JobResult.Success); } var errorCount = 0; var created = DateTime.UtcNow; try { events.ForEach(e => e.CreatedUtc = created); var results = _eventPipeline.Run(events.Take(eventsToProcess).ToList()); Log.Info().Message("Ran {0} events through the pipeline: id={1} project={2} success={3} error={4}", results.Count, queueEntry.Id, eventPostInfo.ProjectId, results.Count(r => r.IsProcessed), results.Count(r => r.HasError)).WriteIf(!isInternalProject); foreach (var eventContext in results) { if (eventContext.IsCancelled) { continue; } if (!eventContext.HasError) { continue; } Log.Error().Exception(eventContext.Exception).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, eventContext.ErrorMessage).Write(); if (eventContext.Exception is ValidationException) { continue; } errorCount++; if (!isSingleEvent) { // Put this single event back into the queue so we can retry it separately. _queue.Enqueue(new EventPostInfo { ApiVersion = eventPostInfo.ApiVersion, CharSet = eventPostInfo.CharSet, ContentEncoding = "application/json", Data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(eventContext.Event)), IpAddress = eventPostInfo.IpAddress, MediaType = eventPostInfo.MediaType, ProjectId = eventPostInfo.ProjectId, UserAgent = eventPostInfo.UserAgent }, _storage, false); } } } catch (ArgumentException ex) { Log.Error().Exception(ex).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, ex.Message).Write(); queueEntry.Complete(); } catch (Exception ex) { Log.Error().Exception(ex).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, ex.Message).Write(); errorCount++; } if (isSingleEvent && errorCount > 0) { queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); } else { queueEntry.Complete(); if (queueEntry.Value.ShouldArchive) { _storage.CompleteEventPost(queueEntry.Value.FilePath, eventPostInfo.ProjectId, created, queueEntry.Value.ShouldArchive); } else { _storage.DeleteFile(queueEntry.Value.FilePath); _storage.SetNotActive(queueEntry.Value.FilePath); } } return(JobResult.Success); }
public override HttpResponseMessage Post(Error value) { if (value == null) { return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Invalid error posted.")); } _stats.Counter(StatNames.ErrorsSubmitted); if (User != null && User.Project != null) { value.ProjectId = User.Project.Id; value.OrganizationId = User.Project.OrganizationId; } if (value.OccurrenceDate == DateTimeOffset.MinValue) { value.OccurrenceDate = DateTimeOffset.UtcNow; } string message = User == null?String.Format("Inserting error '{0}'.", value.Id) : String.Format("Inserting error '{0}' with API key '{1}'.", value.Id, User.Identity.Name); if (value.RequestInfo != null) { message += String.Format(" IP Address: {0}.", value.RequestInfo.ClientIpAddress); } if (value.ExceptionlessClientInfo != null) { message += String.Format(" Client Version: {0}.", value.ExceptionlessClientInfo.Version); } Log.Debug().Message(message).Write(); if (String.IsNullOrWhiteSpace(value.OrganizationId) || !User.IsInOrganization(value.OrganizationId)) { return(InvalidOrganizationErrorResponseMessage()); } string id = value.Id; if (String.IsNullOrEmpty(id)) { value.Id = ObjectId.GenerateNewId().ToString(); id = value.Id; } if (_messageFactory != null) { using (IMessageProducer messageProducer = _messageFactory.CreateMessageProducer()) { RetryUtil.Retry(() => messageProducer.Publish(value)); _stats.Counter(StatNames.ErrorsQueued); } } else { Log.Error().Message("Message Factory is null").Write(); } if (Request == null) { return(CreatedResponseMessage()); } HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.Created); response.Headers.Location = new Uri(Url.Link("DefaultApi", new { id })); return(response); }
private void ProcessErrorException(IMessage <Error> message, Exception exception) { exception.ToExceptionless().AddDefaultInformation().MarkAsCritical().AddObject(message.GetBody()).AddTags("ErrorMQ").Submit(); Log.Error().Project(message.GetBody().ProjectId).Exception(exception).Message("Error processing error.").Write(); _stats.Counter(StatNames.ErrorsProcessingFailed); }
protected async override Task <JobResult> RunInternalAsync(CancellationToken token) { Log.Info().Message("Process events job starting").Write(); QueueEntry <EventPostFileInfo> queueEntry = null; try { queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1)); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { return(JobResult.Success); } EventPost eventPost = _storage.GetEventPostAndSetActive(queueEntry.Value.FilePath); if (eventPost == null) { queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); return(JobResult.FailedWithMessage(String.Format("Unable to retrieve post data '{0}'.", queueEntry.Value.FilePath))); } _statsClient.Counter(StatNames.PostsDequeued); Log.Info().Message("Processing EventPost '{0}'.", queueEntry.Id).Write(); List <PersistentEvent> events = null; try { _statsClient.Time(() => { events = ParseEventPost(eventPost); }, StatNames.PostsParsingTime); _statsClient.Counter(StatNames.PostsParsed); _statsClient.Gauge(StatNames.PostsBatchSize, events.Count); } catch (Exception ex) { _statsClient.Counter(StatNames.PostsParseErrors); queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); // TODO: Add the EventPost to the logged exception. Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write(); return(JobResult.FromException(ex, String.Format("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message))); } if (events == null) { queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); return(JobResult.Success); } int eventsToProcess = events.Count; bool isSingleEvent = events.Count == 1; if (!isSingleEvent) { var project = _projectRepository.GetById(eventPost.ProjectId, true); // Don't process all the events if it will put the account over its limits. eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId); // Add 1 because we already counted 1 against their limit when we received the event post. if (eventsToProcess < Int32.MaxValue) { eventsToProcess += 1; } // Increment by count - 1 since we already incremented it by 1 in the OverageHandler. _organizationRepository.IncrementUsage(project.OrganizationId, events.Count - 1); } int errorCount = 0; DateTime created = DateTime.UtcNow; foreach (PersistentEvent ev in events.Take(eventsToProcess)) { try { ev.CreatedUtc = created; _eventPipeline.Run(ev); } catch (ValidationException ex) { Log.Error().Exception(ex).Project(eventPost.ProjectId).Message("Event validation error occurred: {0}", ex.Message).Write(); } catch (Exception ex) { Log.Error().Exception(ex).Project(eventPost.ProjectId).Message("Error while processing event: {0}", ex.Message).Write(); if (!isSingleEvent) { // Put this single event back into the queue so we can retry it separately. _queue.Enqueue(new EventPost { ApiVersion = eventPost.ApiVersion, CharSet = eventPost.CharSet, ContentEncoding = "application/json", Data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(ev)), IpAddress = eventPost.IpAddress, MediaType = eventPost.MediaType, ProjectId = eventPost.ProjectId, UserAgent = eventPost.UserAgent }, _storage, false); } errorCount++; } } if (isSingleEvent && errorCount > 0) { queueEntry.Abandon(); _storage.SetNotActive(queueEntry.Value.FilePath); } else { queueEntry.Complete(); if (queueEntry.Value.ShouldArchive) { _storage.CompleteEventPost(queueEntry.Value.FilePath, eventPost.ProjectId, created, queueEntry.Value.ShouldArchive); } else { _storage.DeleteFile(queueEntry.Value.FilePath); _storage.SetNotActive(queueEntry.Value.FilePath); } } return(JobResult.Success); }
public override ICollection <EventContext> Run(ICollection <EventContext> contexts) { if (contexts == null || contexts.Count == 0) { return(contexts ?? new List <EventContext>()); } _statsClient.Counter(StatNames.EventsSubmitted, contexts.Count); try { if (contexts.Any(c => !String.IsNullOrEmpty(c.Event.Id))) { throw new ArgumentException("All Event Ids should not be populated."); } string projectId = contexts.First().Event.ProjectId; if (String.IsNullOrEmpty(projectId)) { throw new ArgumentException("All Project Ids must be populated."); } if (contexts.Any(c => c.Event.ProjectId != projectId)) { throw new ArgumentException("All Project Ids must be the same for a batch of events."); } var project = _projectRepository.GetById(projectId, true); if (project == null) { throw new InvalidOperationException(String.Format("Unable to load project \"{0}\"", projectId)); } contexts.ForEach(c => c.Project = project); var organization = _organizationRepository.GetById(project.OrganizationId, true); if (organization == null) { throw new InvalidOperationException(String.Format("Unable to load organization \"{0}\"", project.OrganizationId)); } contexts.ForEach(c => { c.Organization = organization; c.Event.OrganizationId = organization.Id; }); // load organization settings into the context foreach (var key in organization.Data.Keys) { contexts.ForEach(c => c.SetProperty(key, organization.Data[key])); } // load project settings into the context, overriding any organization settings with the same name foreach (var key in project.Data.Keys) { contexts.ForEach(c => c.SetProperty(key, project.Data[key])); } _statsClient.Time(() => base.Run(contexts), StatNames.EventsProcessingTime); var cancelled = contexts.Count(c => c.IsCancelled); if (cancelled > 0) { _statsClient.Counter(StatNames.EventsProcessCancelled, cancelled); } // TODO: Log the errors out to the events proejct id. var errors = contexts.Count(c => c.HasError); if (errors > 0) { _statsClient.Counter(StatNames.EventsProcessErrors, errors); } } catch (Exception) { _statsClient.Counter(StatNames.EventsProcessErrors, contexts.Count); throw; } return(contexts); }