protected async override Task <JobResult> RunInternalAsync()
{
Log.Info().Message("Process events job starting").Write();
int totalEventsProcessed = 0;
int totalEventsToProcess = Context.GetWorkItemLimit();
while (!CancelPending && (totalEventsToProcess == -1 || totalEventsProcessed < totalEventsToProcess))
{
QueueEntry <EventPost> queueEntry = null;
try {
queueEntry = await _queue.DequeueAsync();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
continue;
}
_statsClient.Counter(StatNames.PostsDequeued);
Log.Info().Message("Processing EventPost '{0}'.", queueEntry.Id).Write();
List <PersistentEvent> events = null;
try {
_statsClient.Time(() => {
events = ParseEventPost(queueEntry.Value);
}, StatNames.PostsParsingTime);
_statsClient.Counter(StatNames.PostsParsed);
_statsClient.Gauge(StatNames.PostsBatchSize, events.Count);
} catch (Exception ex) {
_statsClient.Counter(StatNames.PostsParseErrors);
queueEntry.AbandonAsync().Wait();
// TODO: Add the EventPost to the logged exception.
Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write();
continue;
}
if (events == null)
{
queueEntry.AbandonAsync().Wait();
continue;
}
int eventsToProcess = events.Count;
bool isSingleEvent = events.Count == 1;
if (!isSingleEvent)
{
var project = _projectRepository.GetById(queueEntry.Value.ProjectId, true);
// Don't process all the events if it will put the account over its limits.
eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId);
// Add 1 because we already counted 1 against their limit when we received the event post.
if (eventsToProcess < Int32.MaxValue)
{
eventsToProcess += 1;
}
// Increment by count - 1 since we already incremented it by 1 in the OverageHandler.
_organizationRepository.IncrementUsage(project.OrganizationId, events.Count - 1);
}
int errorCount = 0;
foreach (PersistentEvent ev in events.Take(eventsToProcess))
{
try {
_eventPipeline.Run(ev);
totalEventsProcessed++;
if (totalEventsToProcess > 0 && totalEventsProcessed >= totalEventsToProcess)
{
break;
}
} catch (ValidationException ex) {
Log.Error().Exception(ex).Project(queueEntry.Value.ProjectId).Message("Event validation error occurred: {0}", ex.Message).Write();
} catch (Exception ex) {
Log.Error().Exception(ex).Project(queueEntry.Value.ProjectId).Message("Error while processing event: {0}", ex.Message).Write();
if (!isSingleEvent)
{
// Put this single event back into the queue so we can retry it separately.
_queue.EnqueueAsync(new EventPost {
Data = Encoding.UTF8.GetBytes(ev.ToJson()).Compress(),
ContentEncoding = "gzip",
ProjectId = ev.ProjectId,
CharSet = "utf-8",
MediaType = "application/json",
}).Wait();
}
errorCount++;
}
}
if (isSingleEvent && errorCount > 0)
{
queueEntry.AbandonAsync().Wait();
}
else
{
queueEntry.CompleteAsync().Wait();
}
}
return(JobResult.Success);
}
protected async override Task <JobResult> RunInternalAsync(CancellationToken token)
{
QueueEntry <EventPost> queueEntry = null;
try {
queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1));
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
if (token.IsCancellationRequested)
{
queueEntry.Abandon();
return(JobResult.Cancelled);
}
EventPostInfo eventPostInfo = _storage.GetEventPostAndSetActive(queueEntry.Value.FilePath);
if (eventPostInfo == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.FailedWithMessage(String.Format("Unable to retrieve post data '{0}'.", queueEntry.Value.FilePath)));
}
bool isInternalProject = eventPostInfo.ProjectId == Settings.Current.InternalProjectId;
_statsClient.Counter(StatNames.PostsDequeued);
Log.Info().Message("Processing post: id={0} path={1} project={2} ip={3} v={4} agent={5}", queueEntry.Id, queueEntry.Value.FilePath, eventPostInfo.ProjectId, eventPostInfo.IpAddress, eventPostInfo.ApiVersion, eventPostInfo.UserAgent).WriteIf(!isInternalProject);
List <PersistentEvent> events = null;
try {
_statsClient.Time(() => {
events = ParseEventPost(eventPostInfo);
Log.Info().Message("Parsed {0} events for post: id={1}", events.Count, queueEntry.Id).WriteIf(!isInternalProject);
}, StatNames.PostsParsingTime);
_statsClient.Counter(StatNames.PostsParsed);
_statsClient.Gauge(StatNames.PostsEventCount, events.Count);
} catch (Exception ex) {
_statsClient.Counter(StatNames.PostsParseErrors);
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex, String.Format("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message)));
}
if (token.IsCancellationRequested)
{
queueEntry.Abandon();
return(JobResult.Cancelled);
}
if (events == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.Success);
}
int eventsToProcess = events.Count;
bool isSingleEvent = events.Count == 1;
if (!isSingleEvent)
{
var project = _projectRepository.GetById(eventPostInfo.ProjectId, true);
// Don't process all the events if it will put the account over its limits.
eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId);
// Add 1 because we already counted 1 against their limit when we received the event post.
if (eventsToProcess < Int32.MaxValue)
{
eventsToProcess += 1;
}
// Increment by count - 1 since we already incremented it by 1 in the OverageHandler.
_organizationRepository.IncrementUsage(project.OrganizationId, false, events.Count - 1);
}
if (events == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.Success);
}
var errorCount = 0;
var created = DateTime.UtcNow;
try {
events.ForEach(e => e.CreatedUtc = created);
var results = _eventPipeline.Run(events.Take(eventsToProcess).ToList());
Log.Info().Message("Ran {0} events through the pipeline: id={1} project={2} success={3} error={4}", results.Count, queueEntry.Id, eventPostInfo.ProjectId, results.Count(r => r.IsProcessed), results.Count(r => r.HasError)).WriteIf(!isInternalProject);
foreach (var eventContext in results)
{
if (eventContext.IsCancelled)
{
continue;
}
if (!eventContext.HasError)
{
continue;
}
Log.Error().Exception(eventContext.Exception).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, eventContext.ErrorMessage).Write();
if (eventContext.Exception is ValidationException)
{
continue;
}
errorCount++;
if (!isSingleEvent)
{
// Put this single event back into the queue so we can retry it separately.
_queue.Enqueue(new EventPostInfo {
ApiVersion = eventPostInfo.ApiVersion,
CharSet = eventPostInfo.CharSet,
ContentEncoding = "application/json",
Data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(eventContext.Event)),
IpAddress = eventPostInfo.IpAddress,
MediaType = eventPostInfo.MediaType,
ProjectId = eventPostInfo.ProjectId,
UserAgent = eventPostInfo.UserAgent
}, _storage, false);
}
}
} catch (ArgumentException ex) {
Log.Error().Exception(ex).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, ex.Message).Write();
queueEntry.Complete();
} catch (Exception ex) {
Log.Error().Exception(ex).Project(eventPostInfo.ProjectId).Message("Error while processing event post \"{0}\": {1}", queueEntry.Value.FilePath, ex.Message).Write();
errorCount++;
}
if (isSingleEvent && errorCount > 0)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
}
else
{
queueEntry.Complete();
if (queueEntry.Value.ShouldArchive)
{
_storage.CompleteEventPost(queueEntry.Value.FilePath, eventPostInfo.ProjectId, created, queueEntry.Value.ShouldArchive);
}
else
{
_storage.DeleteFile(queueEntry.Value.FilePath);
_storage.SetNotActive(queueEntry.Value.FilePath);
}
}
return(JobResult.Success);
}
protected async override Task <JobResult> RunInternalAsync(CancellationToken token)
{
Log.Trace().Message("Process events job starting").Write();
QueueEntry <EventPostFileInfo> queueEntry = null;
try {
queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1));
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("An error occurred while trying to dequeue the next EventPost: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
EventPost eventPost = _storage.GetEventPostAndSetActive(queueEntry.Value.FilePath);
if (eventPost == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.FailedWithMessage(String.Format("Unable to retrieve post data '{0}'.", queueEntry.Value.FilePath)));
}
_statsClient.Counter(StatNames.PostsDequeued);
Log.Info().Message("Processing EventPost '{0}'.", queueEntry.Id).Write();
List <PersistentEvent> events = null;
try {
_statsClient.Time(() => {
events = ParseEventPost(eventPost);
}, StatNames.PostsParsingTime);
_statsClient.Counter(StatNames.PostsParsed);
_statsClient.Gauge(StatNames.PostsBatchSize, events.Count);
} catch (Exception ex) {
_statsClient.Counter(StatNames.PostsParseErrors);
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
// TODO: Add the EventPost to the logged exception.
Log.Error().Exception(ex).Message("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message).Write();
return(JobResult.FromException(ex, String.Format("An error occurred while processing the EventPost '{0}': {1}", queueEntry.Id, ex.Message)));
}
if (events == null)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
return(JobResult.Success);
}
int eventsToProcess = events.Count;
bool isSingleEvent = events.Count == 1;
if (!isSingleEvent)
{
var project = _projectRepository.GetById(eventPost.ProjectId, true);
// Don't process all the events if it will put the account over its limits.
eventsToProcess = _organizationRepository.GetRemainingEventLimit(project.OrganizationId);
// Add 1 because we already counted 1 against their limit when we received the event post.
if (eventsToProcess < Int32.MaxValue)
{
eventsToProcess += 1;
}
// Increment by count - 1 since we already incremented it by 1 in the OverageHandler.
_organizationRepository.IncrementUsage(project.OrganizationId, events.Count - 1);
}
int errorCount = 0;
DateTime created = DateTime.UtcNow;
foreach (PersistentEvent ev in events.Take(eventsToProcess))
{
try {
ev.CreatedUtc = created;
_eventPipeline.Run(ev);
} catch (ValidationException ex) {
Log.Error().Exception(ex).Project(eventPost.ProjectId).Message("Event validation error occurred: {0}", ex.Message).Write();
} catch (Exception ex) {
Log.Error().Exception(ex).Project(eventPost.ProjectId).Message("Error while processing event: {0}", ex.Message).Write();
if (!isSingleEvent)
{
// Put this single event back into the queue so we can retry it separately.
_queue.Enqueue(new EventPost {
ApiVersion = eventPost.ApiVersion,
CharSet = eventPost.CharSet,
ContentEncoding = "application/json",
Data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(ev)),
IpAddress = eventPost.IpAddress,
MediaType = eventPost.MediaType,
ProjectId = eventPost.ProjectId,
UserAgent = eventPost.UserAgent
}, _storage, false);
}
errorCount++;
}
}
if (isSingleEvent && errorCount > 0)
{
queueEntry.Abandon();
_storage.SetNotActive(queueEntry.Value.FilePath);
}
else
{
queueEntry.Complete();
if (queueEntry.Value.ShouldArchive)
{
_storage.CompleteEventPost(queueEntry.Value.FilePath, eventPost.ProjectId, created, queueEntry.Value.ShouldArchive);
}
else
{
_storage.DeleteFile(queueEntry.Value.FilePath);
_storage.SetNotActive(queueEntry.Value.FilePath);
}
}
return(JobResult.Success);
}
public override ICollection <EventContext> Run(ICollection <EventContext> contexts)
{
if (contexts == null || contexts.Count == 0)
{
return(contexts ?? new List <EventContext>());
}
_statsClient.Counter(StatNames.EventsSubmitted, contexts.Count);
try {
if (contexts.Any(c => !String.IsNullOrEmpty(c.Event.Id)))
{
throw new ArgumentException("All Event Ids should not be populated.");
}
string projectId = contexts.First().Event.ProjectId;
if (String.IsNullOrEmpty(projectId))
{
throw new ArgumentException("All Project Ids must be populated.");
}
if (contexts.Any(c => c.Event.ProjectId != projectId))
{
throw new ArgumentException("All Project Ids must be the same for a batch of events.");
}
var project = _projectRepository.GetById(projectId, true);
if (project == null)
{
throw new InvalidOperationException(String.Format("Unable to load project \"{0}\"", projectId));
}
contexts.ForEach(c => c.Project = project);
var organization = _organizationRepository.GetById(project.OrganizationId, true);
if (organization == null)
{
throw new InvalidOperationException(String.Format("Unable to load organization \"{0}\"", project.OrganizationId));
}
contexts.ForEach(c => {
c.Organization = organization;
c.Event.OrganizationId = organization.Id;
});
// load organization settings into the context
foreach (var key in organization.Data.Keys)
{
contexts.ForEach(c => c.SetProperty(key, organization.Data[key]));
}
// load project settings into the context, overriding any organization settings with the same name
foreach (var key in project.Data.Keys)
{
contexts.ForEach(c => c.SetProperty(key, project.Data[key]));
}
_statsClient.Time(() => base.Run(contexts), StatNames.EventsProcessingTime);
var cancelled = contexts.Count(c => c.IsCancelled);
if (cancelled > 0)
{
_statsClient.Counter(StatNames.EventsProcessCancelled, cancelled);
}
// TODO: Log the errors out to the events proejct id.
var errors = contexts.Count(c => c.HasError);
if (errors > 0)
{
_statsClient.Counter(StatNames.EventsProcessErrors, errors);
}
} catch (Exception) {
_statsClient.Counter(StatNames.EventsProcessErrors, contexts.Count);
throw;
}
return(contexts);
}