public override void Process(EventContext ctx)
{
_stats.Counter(StatNames.EventsProcessed);
if (ctx.Organization.PlanId != BillingManager.FreePlan.Id)
{
_stats.Counter(StatNames.EventsPaidProcessed);
}
}
protected override void Run(EventContext context, IEnumerable <Type> actionTypes)
{
_statsClient.Counter(StatNames.EventsSubmitted);
try {
if (!String.IsNullOrEmpty(context.Event.Id))
{
throw new ArgumentException("Event Id should not be populated.");
}
if (String.IsNullOrEmpty(context.Event.ProjectId))
{
throw new ArgumentException("ProjectId must be populated on the Event.");
}
if (context.Project == null)
{
context.Project = _projectRepository.GetById(context.Event.ProjectId, true);
}
if (context.Project == null)
{
throw new InvalidOperationException(String.Format("Unable to load project \"{0}\"", context.Event.ProjectId));
}
if (String.IsNullOrEmpty(context.Event.OrganizationId))
{
context.Event.OrganizationId = context.Project.OrganizationId;
}
if (context.Organization == null)
{
context.Organization = _organizationRepository.GetById(context.Event.OrganizationId, true);
}
if (context.Organization == null)
{
throw new InvalidOperationException(String.Format("Unable to load organization \"{0}\"", context.Event.OrganizationId));
}
// load organization settings into the context
foreach (var key in context.Organization.Data.Keys)
{
context.SetProperty(key, context.Organization.Data[key]);
}
// load project settings into the context, overriding any organization settings with the same name
foreach (var key in context.Project.Data.Keys)
{
context.SetProperty(key, context.Project.Data[key]);
}
_statsClient.Time(() => base.Run(context, actionTypes), StatNames.EventsProcessingTime);
} catch (Exception) {
_statsClient.Counter(StatNames.EventsProcessErrors);
throw;
}
}
public async Task <IHttpActionResult> SetUserDescription(string referenceId, UserDescription description, string projectId = null)
{
_statsClient.Counter(StatNames.EventsUserDescriptionSubmitted);
if (String.IsNullOrEmpty(referenceId))
{
return(NotFound());
}
if (description == null)
{
return(BadRequest("Description must be specified."));
}
var result = _userDescriptionValidator.Validate(description);
if (!result.IsValid)
{
return(BadRequest(result.Errors.ToErrorMessage()));
}
if (projectId == null)
{
projectId = User.GetDefaultProjectId();
}
// must have a project id
if (String.IsNullOrEmpty(projectId))
{
return(BadRequest("No project id specified and no default project was found."));
}
var project = _projectRepository.GetById(projectId, true);
if (project == null || !User.GetOrganizationIds().ToList().Contains(project.OrganizationId))
{
return(NotFound());
}
var eventUserDescription = Mapper.Map <UserDescription, EventUserDescription>(description);
eventUserDescription.ProjectId = projectId;
eventUserDescription.ReferenceId = referenceId;
await _eventUserDescriptionQueue.EnqueueAsync(eventUserDescription);
_statsClient.Counter(StatNames.EventsUserDescriptionQueued);
return(StatusCode(HttpStatusCode.Accepted));
}
protected async override Task <JobResult> RunInternalAsync()
{
Log.Info().Message("Process user description job starting").Write();
int totalUserDescriptionsProcessed = 0;
int totalUserDescriptionsToProcess = Context.GetWorkItemLimit();
while (!CancelPending && (totalUserDescriptionsToProcess == -1 || totalUserDescriptionsProcessed < totalUserDescriptionsToProcess))
{
QueueEntry <EventUserDescription> queueEntry = null;
try {
queueEntry = await _queue.DequeueAsync();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventUserDescription: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
continue;
}
_statsClient.Counter(StatNames.EventsUserDescriptionDequeued);
Log.Info().Message("Processing EventUserDescription '{0}'.", queueEntry.Id).Write();
try {
ProcessUserDescription(queueEntry.Value);
totalUserDescriptionsProcessed++;
_statsClient.Counter(StatNames.EventsUserDescriptionProcessed);
} catch (DocumentNotFoundException ex) {
_statsClient.Counter(StatNames.EventsUserDescriptionErrors);
queueEntry.AbandonAsync().Wait();
Log.Error().Exception(ex).Message("An event with this reference id \"{0}\" has not been processed yet or was deleted. Queue Id: {1}", ex.Id, queueEntry.Id).Write();
continue;
} catch (Exception ex) {
_statsClient.Counter(StatNames.EventsUserDescriptionErrors);
queueEntry.AbandonAsync().Wait();
// TODO: Add the EventUserDescription to the logged exception.
Log.Error().Exception(ex).Message("An error occurred while processing the EventUserDescription '{0}': {1}", queueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex));
}
await queueEntry.CompleteAsync();
}
return(JobResult.Success);
}
protected override Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (!IsEventPost(request))
{
return(base.SendAsync(request, cancellationToken));
}
if (_cacheClient.TryGet("ApiDisabled", false))
{
return(CreateResponse(request, HttpStatusCode.ServiceUnavailable, "Service Unavailable"));
}
var project = request.GetDefaultProject();
if (project == null)
{
return(CreateResponse(request, HttpStatusCode.Unauthorized, "Unauthorized"));
}
bool tooBig = false;
if (request.Content != null && request.Content.Headers != null)
{
long size = request.Content.Headers.ContentLength.GetValueOrDefault();
_statsClient.Gauge(StatNames.PostsSize, size);
if (size > Settings.Current.MaximumEventPostSize)
{
Log.Warn().Message("Event submission discarded for being too large: {0}", size).Project(project.Id).Write();
_statsClient.Counter(StatNames.PostsDiscarded);
tooBig = true;
}
}
bool overLimit = _organizationRepository.IncrementUsage(project.OrganizationId, tooBig);
// block large submissions, but return success status code so the client doesn't keep sending them
if (tooBig)
{
return(CreateResponse(request, HttpStatusCode.Accepted, "Event submission discarded for being too large."));
}
if (overLimit)
{
_statsClient.Counter(StatNames.PostsBlocked);
return(CreateResponse(request, HttpStatusCode.PaymentRequired, "Event limit exceeded."));
}
return(base.SendAsync(request, cancellationToken));
}
public override void Process(ErrorPipelineContext ctx)
{
var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId);
if (organization == null)
{
return;
}
_stats.Counter(StatNames.ErrorsProcessed);
if (organization.PlanId != BillingManager.FreePlan.Id)
{
_stats.Counter(StatNames.ErrorsPaidProcessed);
}
}
private void QueueMessage(System.Net.Mail.MailMessage message)
{
CleanAddresses(message);
_queue.Enqueue(message.ToMailMessage());
_statsClient.Counter(StatNames.EmailsQueued);
}
private async Task QueueMessage(MailMessage message)
{
CleanAddresses(message);
await _queue.EnqueueAsync(message.ToMailMessage());
_statsClient.Counter(StatNames.EmailsQueued);
}
protected async override Task <JobResult> RunInternalAsync(CancellationToken token)
{
Log.Info().Message("Process user description job starting").Write();
QueueEntry <EventUserDescription> queueEntry = null;
try {
queueEntry = _queue.Dequeue();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventUserDescription: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
_statsClient.Counter(StatNames.EventsUserDescriptionDequeued);
Log.Info().Message("Processing EventUserDescription '{0}'.", queueEntry.Id).Write();
try {
ProcessUserDescription(queueEntry.Value);
_statsClient.Counter(StatNames.EventsUserDescriptionProcessed);
} catch (DocumentNotFoundException ex) {
_statsClient.Counter(StatNames.EventsUserDescriptionErrors);
queueEntry.Abandon();
Log.Error().Exception(ex).Message("An event with this reference id \"{0}\" has not been processed yet or was deleted. Queue Id: {1}", ex.Id, queueEntry.Id).Write();
return(JobResult.FromException(ex));
} catch (Exception ex) {
_statsClient.Counter(StatNames.EventsUserDescriptionErrors);
queueEntry.Abandon();
// TODO: Add the EventUserDescription to the logged exception.
Log.Error().Exception(ex).Message("An error occurred while processing the EventUserDescription '{0}': {1}", queueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex));
}
queueEntry.Complete();
return(JobResult.Success);
}
protected async override Task <JobResult> RunInternalAsync()
{
Log.Info().Message("Process email message job starting").Write();
int totalEmailsProcessed = 0;
int totalEmailsToProcess = Context.GetWorkItemLimit();
while (!CancelPending && (totalEmailsToProcess == -1 || totalEmailsProcessed < totalEmailsToProcess))
{
QueueEntry <MailMessage> queueEntry = null;
try {
queueEntry = await _queue.DequeueAsync();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next MailMessageNotification: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
continue;
}
_statsClient.Counter(StatNames.EmailsDequeued);
Log.Info().Message("Processing MailMessageNotification '{0}'.", queueEntry.Id).Write();
try {
await _mailSender.SendAsync(queueEntry.Value);
totalEmailsProcessed++;
_statsClient.Counter(StatNames.EmailsSent);
} catch (Exception ex) {
_statsClient.Counter(StatNames.EmailsSendErrors);
queueEntry.AbandonAsync().Wait();
Log.Error().Exception(ex).Message("Error sending message '{0}': {1}", queueEntry.Id, ex.Message).Write();
}
await queueEntry.CompleteAsync();
}
return(JobResult.Success);
}
protected async override Task <JobResult> RunInternalAsync(CancellationToken token)
{
Log.Info().Message("Process email message job starting").Write();
QueueEntry <MailMessage> queueEntry = null;
try {
queueEntry = _queue.Dequeue();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next MailMessageNotification: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
_statsClient.Counter(StatNames.EmailsDequeued);
Log.Info().Message("Processing MailMessageNotification '{0}'.", queueEntry.Id).Write();
try {
await _mailSender.SendAsync(queueEntry.Value);
_statsClient.Counter(StatNames.EmailsSent);
} catch (Exception ex) {
_statsClient.Counter(StatNames.EmailsSendErrors);
queueEntry.Abandon();
Log.Error().Exception(ex).Message("Error sending message '{0}': {1}", queueEntry.Id, ex.Message).Write();
}
queueEntry.Complete();
return(JobResult.Success);
}
protected async override Task <JobResult> RunInternalAsync(CancellationToken token)
{
QueueEntry <MailMessage> queueEntry = null;
try {
queueEntry = _queue.Dequeue();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
_statsClient.Counter(StatNames.EmailsDequeued);
Log.Trace().Message("Processing message '{0}'.", queueEntry.Id).Write();
try {
await _mailSender.SendAsync(queueEntry.Value);
Log.Info().Message("Sent message: to={0} subject=\"{1}\"", queueEntry.Value.To, queueEntry.Value.Subject).Write();
_statsClient.Counter(StatNames.EmailsSent);
} catch (Exception ex) {
_statsClient.Counter(StatNames.EmailsSendErrors);
queueEntry.Abandon();
Log.Error().Exception(ex).Message("Error sending message: id={0} error={1}", queueEntry.Id, ex.Message).Write();
}
queueEntry.Complete();
return(JobResult.Success);
}
public void Run(PersistentEvent ev) {
_statsClient.Counter(StatNames.EventsSubmitted);
try {
_statsClient.Time(() => {
if (String.IsNullOrEmpty(ev.ProjectId))
throw new ArgumentException("ProjectId must be populated on the Event.");
var project = _projectRepository.GetById(ev.ProjectId, true);
if (project == null)
throw new InvalidOperationException(String.Format("Unable to load project \"{0}\"", ev.ProjectId));
if (String.IsNullOrEmpty(ev.OrganizationId))
ev.OrganizationId = project.OrganizationId;
var ctx = new EventContext(ev) {
Organization = _organizationRepository.GetById(ev.OrganizationId, true),
Project = project
};
if (ctx.Organization == null)
throw new InvalidOperationException(String.Format("Unable to load organization \"{0}\"", ev.OrganizationId));
// load organization settings into the context
foreach (var key in ctx.Organization.Data.Keys)
ctx.SetProperty(key, ctx.Organization.Data[key]);
// load project settings into the context, overriding any organization settings with the same name
foreach (var key in ctx.Project.Data.Keys)
ctx.SetProperty(key, ctx.Project.Data[key]);
Run(ctx);
}, StatNames.EventsProcessingTime);
} catch (Exception ex) {
_statsClient.Counter(StatNames.EventsProcessErrors);
throw;
}
}
public async Task <IHttpActionResult> Post([NakedBody] byte[] data, string projectId = null, int version = 1, [UserAgent] string userAgent = null)
{
_statsClient.Counter(StatNames.PostsSubmitted);
if (projectId == null)
{
projectId = User.GetProjectId();
}
// must have a project id
if (String.IsNullOrEmpty(projectId))
{
return(StatusCode(HttpStatusCode.Unauthorized));
}
// TODO: Add a check to see if the project id is over it's project limits. If it is, then turn off the client.
bool isCompressed = Request.Content.Headers.ContentEncoding.Contains("gzip");
if (!isCompressed)
{
data = data.Compress();
}
await _eventPostQueue.EnqueueAsync(new EventPost {
MediaType = Request.Content.Headers.ContentType.MediaType,
CharSet = Request.Content.Headers.ContentType.CharSet,
ProjectId = projectId,
UserAgent = userAgent,
ApiVersion = version,
Data = data
});
_statsClient.Counter(StatNames.PostsQueued);
return(StatusCode(HttpStatusCode.Accepted));
}
public override void EventProcessing(EventContext context)
{
if (Settings.Current.WebsiteMode == WebsiteMode.Dev)
{
return;
}
var project = _projectRepository.GetById(context.Event.ProjectId);
if (project == null || !project.DeleteBotDataEnabled)
{
return;
}
// Throttle errors by client ip address to no more than X every 5 minutes.
var ri = context.Event.GetRequestInfo();
if (ri == null || String.IsNullOrEmpty(ri.ClientIpAddress))
{
return;
}
string throttleCacheKey = String.Concat("bot:", ri.ClientIpAddress, ":", DateTime.Now.Floor(_throttlingPeriod).Ticks);
var requestCount = _cacheClient.Get <int?>(throttleCacheKey);
if (requestCount != null)
{
_cacheClient.Increment(throttleCacheKey, 1);
requestCount++;
}
else
{
_cacheClient.Set(throttleCacheKey, 1, _throttlingPeriod);
requestCount = 1;
}
if (requestCount < Settings.Current.BotThrottleLimit)
{
return;
}
_stats.Counter(StatNames.EventsBotThrottleTriggered);
Log.Info().Message("Bot throttle triggered. IP: {0} Time: {1} Project: {2}", ri.ClientIpAddress, DateTime.Now.Floor(_throttlingPeriod), context.Event.ProjectId).Project(context.Event.ProjectId).Write();
// the throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system
Task.Run(() => _eventRepository.HideAllByClientIpAndDateAsync(context.Event.OrganizationId, ri.ClientIpAddress, DateTime.Now.Floor(_throttlingPeriod), DateTime.Now.Ceiling(_throttlingPeriod)));
context.IsCancelled = true;
}
protected async override Task <JobResult> RunInternalAsync()
{
Log.Info().Message("Process events job starting").Write();
int totalEventsProcessed = 0;
int totalEventsToProcess = Context.GetWorkItemLimit();
while (!CancelPending && (totalEventsToProcess == -1 || totalEventsProcessed < totalEventsToProcess))
{
QueueEntry <EventPost> queueEntry = null;
try {
queueEntry = await _queue.DequeueAsync();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
continue;
}
_statsClient.Counter(StatNames.PostsDequeued);
Log.Info().Message("Processing EventPost '{0}'.", queueEntry.Id).Write();
List <PersistentEvent> events = null;
try {
_statsClient.Time(() => {
events = ParseEventPost(queueEntry.Value);
}, StatNames.PostsParsingTime);
_statsClient.Counter(StatNames.PostsParsed);
_statsClient.Gauge(StatNames.PostsBatchSize, events.Count);
} catch (Exception ex) {
_statsClient.Counter(StatNames.PostsParseErrors);
queueEntry.AbandonAsync().Wait();
// TODO: Add the EventPost to the logged exception.
Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write();
continue;
}
if (events == null)
{
queueEntry.AbandonAsync().Wait();
continue;
}
int eventsToProcess = events.Count;
bool isSingleEvent = events.Count == 1;
if (!isSingleEvent)
{
var project = _projectRepository.GetById(queueEntry.Value.ProjectId, true);
// Don't process all the events if it will put the account over its limits.
eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId);
// Add 1 because we already counted 1 against their limit when we received the event post.
if (eventsToProcess < Int32.MaxValue)
{
eventsToProcess += 1;
}
// Increment by count - 1 since we already incremented it by 1 in the OverageHandler.
_organizationRepository.IncrementUsage(project.OrganizationId, events.Count - 1);
}
int errorCount = 0;
foreach (PersistentEvent ev in events.Take(eventsToProcess))
{
try {
_eventPipeline.Run(ev);
totalEventsProcessed++;
if (totalEventsToProcess > 0 && totalEventsProcessed >= totalEventsToProcess)
{
break;
}
} catch (ValidationException ex) {
Log.Error().Exception(ex).Project(queueEntry.Value.ProjectId).Message("Event validation error occurred: {0}", ex.Message).Write();
} catch (Exception ex) {
Log.Error().Exception(ex).Project(queueEntry.Value.ProjectId).Message("Error while processing event: {0}", ex.Message).Write();
if (!isSingleEvent)
{
// Put this single event back into the queue so we can retry it separately.
_queue.EnqueueAsync(new EventPost {
Data = Encoding.UTF8.GetBytes(ev.ToJson()).Compress(),
ContentEncoding = "gzip",
ProjectId = ev.ProjectId,
CharSet = "utf-8",
MediaType = "application/json",
}).Wait();
}
errorCount++;
}
}
if (isSingleEvent && errorCount > 0)
{
queueEntry.AbandonAsync().Wait();
}
else
{
queueEntry.CompleteAsync().Wait();
}
}
return(JobResult.Success);
}
protected async override Task <JobResult> RunInternalAsync(CancellationToken token)
{
QueueEntry <EventPost> queueEntry = null;
try {
queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1));
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
if (token.IsCancellationRequested)
{
queueEntry.Abandon();
return(JobResult.Cancelled);
}
EventPostInfo eventPostInfo = _storage.GetEventPostAndSetActive(queueEntry.Value.FilePath);
if (eventPostInfo == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.FailedWithMessage(String.Format("Unable to retrieve post data '{0}'.", queueEntry.Value.FilePath)));
}
bool isInternalProject = eventPostInfo.ProjectId == Settings.Current.InternalProjectId;
_statsClient.Counter(StatNames.PostsDequeued);
Log.Info().Message("Processing post: id={0} path={1} project={2} ip={3} v={4} agent={5}", queueEntry.Id, queueEntry.Value.FilePath, eventPostInfo.ProjectId, eventPostInfo.IpAddress, eventPostInfo.ApiVersion, eventPostInfo.UserAgent).WriteIf(!isInternalProject);
List <PersistentEvent> events = null;
try {
_statsClient.Time(() => {
events = ParseEventPost(eventPostInfo);
Log.Info().Message("Parsed {0} events for post: id={1}", events.Count, queueEntry.Id).WriteIf(!isInternalProject);
}, StatNames.PostsParsingTime);
_statsClient.Counter(StatNames.PostsParsed);
_statsClient.Gauge(StatNames.PostsEventCount, events.Count);
} catch (Exception ex) {
_statsClient.Counter(StatNames.PostsParseErrors);
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex, String.Format("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message)));
}
if (token.IsCancellationRequested)
{
queueEntry.Abandon();
return(JobResult.Cancelled);
}
if (events == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.Success);
}
int eventsToProcess = events.Count;
bool isSingleEvent = events.Count == 1;
if (!isSingleEvent)
{
var project = _projectRepository.GetById(eventPostInfo.ProjectId, true);
// Don't process all the events if it will put the account over its limits.
eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId);
// Add 1 because we already counted 1 against their limit when we received the event post.
if (eventsToProcess < Int32.MaxValue)
{
eventsToProcess += 1;
}
// Increment by count - 1 since we already incremented it by 1 in the OverageHandler.
_organizationRepository.IncrementUsage(project.OrganizationId, false, events.Count - 1);
}
if (events == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.Success);
}
var errorCount = 0;
var created = DateTime.UtcNow;
try {
events.ForEach(e => e.CreatedUtc = created);
var results = _eventPipeline.Run(events.Take(eventsToProcess).ToList());
Log.Info().Message("Ran {0} events through the pipeline: id={1} project={2} success={3} error={4}", results.Count, queueEntry.Id, eventPostInfo.ProjectId, results.Count(r => r.IsProcessed), results.Count(r => r.HasError)).WriteIf(!isInternalProject);
foreach (var eventContext in results)
{
if (eventContext.IsCancelled)
{
continue;
}
if (!eventContext.HasError)
{
continue;
}
Log.Error().Exception(eventContext.Exception).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, eventContext.ErrorMessage).Write();
if (eventContext.Exception is ValidationException)
{
continue;
}
errorCount++;
if (!isSingleEvent)
{
// Put this single event back into the queue so we can retry it separately.
_queue.Enqueue(new EventPostInfo {
ApiVersion = eventPostInfo.ApiVersion,
CharSet = eventPostInfo.CharSet,
ContentEncoding = "application/json",
Data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(eventContext.Event)),
IpAddress = eventPostInfo.IpAddress,
MediaType = eventPostInfo.MediaType,
ProjectId = eventPostInfo.ProjectId,
UserAgent = eventPostInfo.UserAgent
}, _storage, false);
}
}
} catch (ArgumentException ex) {
Log.Error().Exception(ex).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, ex.Message).Write();
queueEntry.Complete();
} catch (Exception ex) {
Log.Error().Exception(ex).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, ex.Message).Write();
errorCount++;
}
if (isSingleEvent && errorCount > 0)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
}
else
{
queueEntry.Complete();
if (queueEntry.Value.ShouldArchive)
{
_storage.CompleteEventPost(queueEntry.Value.FilePath, eventPostInfo.ProjectId, created, queueEntry.Value.ShouldArchive);
}
else
{
_storage.DeleteFile(queueEntry.Value.FilePath);
_storage.SetNotActive(queueEntry.Value.FilePath);
}
}
return(JobResult.Success);
}
public override HttpResponseMessage Post(Error value)
{
if (value == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Invalid error posted."));
}
_stats.Counter(StatNames.ErrorsSubmitted);
if (User != null && User.Project != null)
{
value.ProjectId = User.Project.Id;
value.OrganizationId = User.Project.OrganizationId;
}
if (value.OccurrenceDate == DateTimeOffset.MinValue)
{
value.OccurrenceDate = DateTimeOffset.UtcNow;
}
string message = User == null?String.Format("Inserting error '{0}'.", value.Id) : String.Format("Inserting error '{0}' with API key '{1}'.", value.Id, User.Identity.Name);
if (value.RequestInfo != null)
{
message += String.Format(" IP Address: {0}.", value.RequestInfo.ClientIpAddress);
}
if (value.ExceptionlessClientInfo != null)
{
message += String.Format(" Client Version: {0}.", value.ExceptionlessClientInfo.Version);
}
Log.Debug().Message(message).Write();
if (String.IsNullOrWhiteSpace(value.OrganizationId) || !User.IsInOrganization(value.OrganizationId))
{
return(InvalidOrganizationErrorResponseMessage());
}
string id = value.Id;
if (String.IsNullOrEmpty(id))
{
value.Id = ObjectId.GenerateNewId().ToString();
id = value.Id;
}
if (_messageFactory != null)
{
using (IMessageProducer messageProducer = _messageFactory.CreateMessageProducer()) {
RetryUtil.Retry(() => messageProducer.Publish(value));
_stats.Counter(StatNames.ErrorsQueued);
}
}
else
{
Log.Error().Message("Message Factory is null").Write();
}
if (Request == null)
{
return(CreatedResponseMessage());
}
HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.Created);
response.Headers.Location = new Uri(Url.Link("DefaultApi", new { id }));
return(response);
}
private void ProcessErrorException(IMessage <Error> message, Exception exception)
{
exception.ToExceptionless().AddDefaultInformation().MarkAsCritical().AddObject(message.GetBody()).AddTags("ErrorMQ").Submit();
Log.Error().Project(message.GetBody().ProjectId).Exception(exception).Message("Error processing error.").Write();
_stats.Counter(StatNames.ErrorsProcessingFailed);
}
protected async override Task <JobResult> RunInternalAsync(CancellationToken token)
{
Log.Info().Message("Process events job starting").Write();
QueueEntry <EventPostFileInfo> queueEntry = null;
try {
queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1));
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
EventPost eventPost = _storage.GetEventPostAndSetActive(queueEntry.Value.FilePath);
if (eventPost == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.FailedWithMessage(String.Format("Unable to retrieve post data '{0}'.", queueEntry.Value.FilePath)));
}
_statsClient.Counter(StatNames.PostsDequeued);
Log.Info().Message("Processing EventPost '{0}'.", queueEntry.Id).Write();
List <PersistentEvent> events = null;
try {
_statsClient.Time(() => {
events = ParseEventPost(eventPost);
}, StatNames.PostsParsingTime);
_statsClient.Counter(StatNames.PostsParsed);
_statsClient.Gauge(StatNames.PostsBatchSize, events.Count);
} catch (Exception ex) {
_statsClient.Counter(StatNames.PostsParseErrors);
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
// TODO: Add the EventPost to the logged exception.
Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex, String.Format("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message)));
}
if (events == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.Success);
}
int eventsToProcess = events.Count;
bool isSingleEvent = events.Count == 1;
if (!isSingleEvent)
{
var project = _projectRepository.GetById(eventPost.ProjectId, true);
// Don't process all the events if it will put the account over its limits.
eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId);
// Add 1 because we already counted 1 against their limit when we received the event post.
if (eventsToProcess < Int32.MaxValue)
{
eventsToProcess += 1;
}
// Increment by count - 1 since we already incremented it by 1 in the OverageHandler.
_organizationRepository.IncrementUsage(project.OrganizationId, events.Count - 1);
}
int errorCount = 0;
DateTime created = DateTime.UtcNow;
foreach (PersistentEvent ev in events.Take(eventsToProcess))
{
try {
ev.CreatedUtc = created;
_eventPipeline.Run(ev);
} catch (ValidationException ex) {
Log.Error().Exception(ex).Project(eventPost.ProjectId).Message("Event validation error occurred: {0}", ex.Message).Write();
} catch (Exception ex) {
Log.Error().Exception(ex).Project(eventPost.ProjectId).Message("Error while processing event: {0}", ex.Message).Write();
if (!isSingleEvent)
{
// Put this single event back into the queue so we can retry it separately.
_queue.Enqueue(new EventPost {
ApiVersion = eventPost.ApiVersion,
CharSet = eventPost.CharSet,
ContentEncoding = "application/json",
Data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(ev)),
IpAddress = eventPost.IpAddress,
MediaType = eventPost.MediaType,
ProjectId = eventPost.ProjectId,
UserAgent = eventPost.UserAgent
}, _storage, false);
}
errorCount++;
}
}
if (isSingleEvent && errorCount > 0)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
}
else
{
queueEntry.Complete();
if (queueEntry.Value.ShouldArchive)
{
_storage.CompleteEventPost(queueEntry.Value.FilePath, eventPost.ProjectId, created, queueEntry.Value.ShouldArchive);
}
else
{
_storage.DeleteFile(queueEntry.Value.FilePath);
_storage.SetNotActive(queueEntry.Value.FilePath);
}
}
return(JobResult.Success);
}
public override ICollection <EventContext> Run(ICollection <EventContext> contexts)
{
if (contexts == null || contexts.Count == 0)
{
return(contexts ?? new List <EventContext>());
}
_statsClient.Counter(StatNames.EventsSubmitted, contexts.Count);
try {
if (contexts.Any(c => !String.IsNullOrEmpty(c.Event.Id)))
{
throw new ArgumentException("All Event Ids should not be populated.");
}
string projectId = contexts.First().Event.ProjectId;
if (String.IsNullOrEmpty(projectId))
{
throw new ArgumentException("All Project Ids must be populated.");
}
if (contexts.Any(c => c.Event.ProjectId != projectId))
{
throw new ArgumentException("All Project Ids must be the same for a batch of events.");
}
var project = _projectRepository.GetById(projectId, true);
if (project == null)
{
throw new InvalidOperationException(String.Format("Unable to load project \"{0}\"", projectId));
}
contexts.ForEach(c => c.Project = project);
var organization = _organizationRepository.GetById(project.OrganizationId, true);
if (organization == null)
{
throw new InvalidOperationException(String.Format("Unable to load organization \"{0}\"", project.OrganizationId));
}
contexts.ForEach(c => {
c.Organization = organization;
c.Event.OrganizationId = organization.Id;
});
// load organization settings into the context
foreach (var key in organization.Data.Keys)
{
contexts.ForEach(c => c.SetProperty(key, organization.Data[key]));
}
// load project settings into the context, overriding any organization settings with the same name
foreach (var key in project.Data.Keys)
{
contexts.ForEach(c => c.SetProperty(key, project.Data[key]));
}
_statsClient.Time(() => base.Run(contexts), StatNames.EventsProcessingTime);
var cancelled = contexts.Count(c => c.IsCancelled);
if (cancelled > 0)
{
_statsClient.Counter(StatNames.EventsProcessCancelled, cancelled);
}
// TODO: Log the errors out to the events proejct id.
var errors = contexts.Count(c => c.HasError);
if (errors > 0)
{
_statsClient.Counter(StatNames.EventsProcessErrors, errors);
}
} catch (Exception) {
_statsClient.Counter(StatNames.EventsProcessErrors, contexts.Count);
throw;
}
return(contexts);
}