public override void Process(ErrorPipelineContext ctx)
{
if (ctx.IsNew)
{
return;
}
int maxErrorsPerStack = 50;
Organization organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId);
if (organization != null)
{
maxErrorsPerStack = organization.MaxErrorsPerDay > 0 ? organization.MaxErrorsPerDay + Math.Min(50, organization.MaxErrorsPerDay * 2) : Int32.MaxValue;
}
// get a list of oldest ids that exceed our desired max errors
var errors = _errorRepository.Collection.Find(Query.EQ(ErrorRepository.FieldNames.ErrorStackId, new BsonObjectId(new ObjectId(ctx.Error.ErrorStackId))))
.SetSortOrder(SortBy.Descending(ErrorRepository.FieldNames.OccurrenceDate_UTC))
.SetFields(ErrorRepository.FieldNames.Id)
.Select(e => new Error {
Id = e.Id,
OrganizationId = ctx.Error.OrganizationId,
ProjectId = ctx.Error.ProjectId,
ErrorStackId = ctx.Error.ErrorStackId
})
.Skip(maxErrorsPerStack)
.Take(150).ToArray();
if (errors.Length > 0)
{
_errorRepository.Delete(errors);
}
}
public override void Process(ErrorPipelineContext ctx) {
if (ctx.IsNew)
return;
int maxErrorsPerStack = 50;
Organization organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId);
if (organization != null)
maxErrorsPerStack = organization.MaxErrorsPerDay > 0 ? organization.MaxErrorsPerDay + Math.Min(50, organization.MaxErrorsPerDay * 2) : Int32.MaxValue;
// Get a list of oldest ids that exceed our desired max errors.
var errors = _errorRepository.Collection.Find(
Query.EQ(ErrorRepository.FieldNames.ErrorStackId, new BsonObjectId(new ObjectId(ctx.Error.ErrorStackId))))
.SetSortOrder(SortBy.Descending(ErrorRepository.FieldNames.OccurrenceDate_UTC))
.SetFields(ErrorRepository.FieldNames.Id)
.SetSkip(maxErrorsPerStack)
.SetLimit(150)
.Select(e => new Error {
Id = e.Id,
OrganizationId = ctx.Error.OrganizationId,
ProjectId = ctx.Error.ProjectId,
ErrorStackId = ctx.Error.ErrorStackId
})
.ToArray();
if (errors.Length > 0)
_errorRepository.Delete(errors);
}
public override void Process(ErrorPipelineContext ctx) {
if (Settings.Current.WebsiteMode == WebsiteMode.Dev)
return;
// Throttle errors by client ip address to no more than X every 5 minutes.
string clientIp = null;
if (ctx.Error.RequestInfo != null && !String.IsNullOrEmpty(ctx.Error.RequestInfo.ClientIpAddress))
clientIp = ctx.Error.RequestInfo.ClientIpAddress;
if (String.IsNullOrEmpty(clientIp))
return;
string throttleCacheKey = String.Concat("bot:", clientIp, ":", DateTime.Now.Floor(_throttlingPeriod).Ticks);
var requestCount = _cacheClient.Get<int?>(throttleCacheKey);
if (requestCount != null) {
_cacheClient.Increment(throttleCacheKey, 1);
requestCount++;
} else {
_cacheClient.Set(throttleCacheKey, 1, _throttlingPeriod);
requestCount = 1;
}
if (requestCount < Settings.Current.BotThrottleLimit)
return;
Log.Info().Message("Bot throttle triggered. IP: {0} Time: {1} Project: {2}", clientIp, DateTime.Now.Floor(_throttlingPeriod), ctx.Error.ProjectId).Project(ctx.Error.ProjectId).Write();
// the throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system
Task.Run(() => _errorRepository.RemoveAllByClientIpAndDateAsync(clientIp, DateTime.Now.Floor(_throttlingPeriod), DateTime.Now.Ceiling(_throttlingPeriod)));
ctx.IsCancelled = true;
}
public override void Process(ErrorPipelineContext ctx) {
if (ctx.StackInfo == null || !ctx.StackInfo.OccurrencesAreCritical)
return;
Log.Trace().Message("Marking error as critical.").Write();
ctx.Error.MarkAsCritical();
}
public override void Process(ErrorPipelineContext ctx) {
_organizationRepository.IncrementStats(ctx.Error.OrganizationId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0);
_projectRepository.IncrementStats(ctx.Error.ProjectId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0);
if (!ctx.IsNew)
_errorStackRepository.IncrementStats(ctx.Error.ErrorStackId, ctx.Error.OccurrenceDate.UtcDateTime);
IEnumerable<TimeSpan> offsets = _projectRepository.GetTargetTimeOffsetsForStats(ctx.Error.ProjectId);
_statsHelper.Process(ctx.Error, ctx.IsNew, offsets);
}
public override void Process(ErrorPipelineContext ctx)
{
if (ctx.StackInfo == null || !ctx.StackInfo.OccurrencesAreCritical)
{
return;
}
Log.Trace().Message("Marking error as critical.").Write();
ctx.Error.MarkAsCritical();
}
public override void Process(ErrorPipelineContext ctx) {
var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId);
if (organization == null)
return;
_stats.Counter(StatNames.ErrorsProcessed);
if (organization.PlanId != BillingManager.FreePlan.Id)
_stats.Counter(StatNames.ErrorsPaidProcessed);
}
public override void Process(ErrorPipelineContext ctx) {
try {
ctx.Error = _errorRepository.Add(ctx.Error);
} catch (WriteConcernException ex) {
// ignore errors being submitted multiple times
if (ex.Message.Contains("E11000")) {
Log.Info().Project(ctx.Error.ProjectId).Message("Ignoring duplicate error submission: {0}", ctx.Error.Id).Write();
ctx.IsCancelled = true;
} else
throw;
}
}
public static WebHookError FromError(ErrorPipelineContext ctx, IProjectRepository projectRepository, IErrorStackRepository errorStackRepository, IOrganizationRepository organizationRepository) {
if (ctx == null || ctx.Error == null)
throw new ArgumentNullException("ctx");
if (projectRepository == null)
throw new ArgumentNullException("projectRepository");
if (errorStackRepository == null)
throw new ArgumentNullException("errorStackRepository");
if (organizationRepository == null)
throw new ArgumentNullException("organizationRepository");
var project = projectRepository.GetByIdCached(ctx.Error.ProjectId);
if (project == null)
throw new ArgumentException("ProjectId not found.");
var organization = organizationRepository.GetByIdCached(ctx.Error.OrganizationId);
if (organization == null)
throw new ArgumentException("OrganizationId not found.");
var errorStack = errorStackRepository.GetByIdCached(ctx.Error.ErrorStackId);
if (errorStack == null)
throw new ArgumentException("ErrorStackId not found.");
return new WebHookError {
Id = ctx.Error.Id,
OccurrenceDate = ctx.Error.OccurrenceDate,
Tags = ctx.Error.Tags,
MachineName = ctx.Error.EnvironmentInfo != null ? ctx.Error.EnvironmentInfo.MachineName : null,
RequestPath = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.GetFullPath() : null,
IpAddress = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.ClientIpAddress : ctx.Error.EnvironmentInfo != null ? ctx.Error.EnvironmentInfo.IpAddress : null,
Message = ctx.Error.Message,
Type = ctx.Error.Type,
Code = ctx.Error.Code,
TargetMethod = ctx.Error.TargetMethod != null ? ctx.Error.TargetMethod.FullName : null,
ProjectId = ctx.Error.ProjectId,
ProjectName = project.Name,
OrganizationId = ctx.Error.OrganizationId,
OrganizationName = organization.Name,
ErrorStackId = ctx.Error.ErrorStackId,
ErrorStackTitle = errorStack.Title,
ErrorStackDescription = errorStack.Description,
ErrorStackTags = errorStack.Tags,
TotalOccurrences = errorStack.TotalOccurrences,
FirstOccurrence = errorStack.FirstOccurrence,
LastOccurrence = errorStack.LastOccurrence,
DateFixed = errorStack.DateFixed,
IsRegression = ctx.IsRegression,
IsNew = ctx.IsNew
};
}
public override void Process(ErrorPipelineContext ctx)
{
_organizationRepository.IncrementStats(ctx.Error.OrganizationId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0);
_projectRepository.IncrementStats(ctx.Error.ProjectId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0);
if (!ctx.IsNew)
{
_errorStackRepository.IncrementStats(ctx.Error.ErrorStackId, ctx.Error.OccurrenceDate.UtcDateTime);
}
IEnumerable <TimeSpan> offsets = _projectRepository.GetTargetTimeOffsetsForStats(ctx.Error.ProjectId);
_statsHelper.Process(ctx.Error, ctx.IsNew, offsets);
}
public override void Process(ErrorPipelineContext ctx) {
if (ctx.StackInfo == null || !ctx.StackInfo.DateFixed.HasValue || ctx.StackInfo.DateFixed.Value >= ctx.Error.OccurrenceDate.UtcDateTime)
return;
Log.Trace().Message("Marking error as an regression.").Write();
_errorStackRepository.Collection.Update(
Query.EQ(ErrorStackRepository.FieldNames.Id, new BsonObjectId(new ObjectId(ctx.StackInfo.Id))),
Update
.Unset(ErrorStackRepository.FieldNames.DateFixed)
.Set(ErrorStackRepository.FieldNames.IsRegressed, true));
_errorStackRepository.InvalidateCache(ctx.Error.ErrorStackId, ctx.StackInfo.SignatureHash, ctx.Error.ProjectId);
ctx.IsRegression = true;
}
public override void Process(ErrorPipelineContext ctx)
{
var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId);
if (organization == null)
{
return;
}
_stats.Counter(StatNames.ErrorsProcessed);
if (organization.PlanId != BillingManager.FreePlan.Id)
{
_stats.Counter(StatNames.ErrorsPaidProcessed);
}
}
public override void Process(ErrorPipelineContext ctx)
{
try {
ctx.Error = _errorRepository.Add(ctx.Error);
} catch (WriteConcernException ex) {
// ignore errors being submitted multiple times
if (ex.Message.Contains("E11000"))
{
Log.Info().Project(ctx.Error.ProjectId).Message("Ignoring duplicate error submission: {0}", ctx.Error.Id).Write();
ctx.IsCancelled = true;
}
else
{
throw;
}
}
}
public override void Process(ErrorPipelineContext ctx)
{
var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId);
// if they don't have premium features, then we don't need to queue notifications
if (organization != null && !organization.HasPremiumFeatures)
{
return;
}
using (IMessageProducer messageProducer = _messageFactory.CreateMessageProducer()) {
messageProducer.Publish(new ErrorNotification {
ErrorId = ctx.Error.Id,
ErrorStackId = ctx.Error.ErrorStackId,
FullTypeName = ctx.StackingInfo.FullTypeName,
IsNew = ctx.IsNew,
IsCritical = ctx.Error.Tags != null && ctx.Error.Tags.Contains("Critical"),
IsRegression = ctx.IsRegression,
Message = ctx.StackingInfo.Message,
ProjectId = ctx.Error.ProjectId,
Code = ctx.Error.Code,
UserAgent = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.UserAgent : null,
Url = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.GetFullPath(true, true) : null
});
foreach (ProjectHook hook in _projectHookRepository.GetByProjectId(ctx.Error.ProjectId))
{
bool shouldCall = hook.EventTypes.Contains(ProjectHookRepository.EventTypes.NewError) && ctx.IsNew ||
hook.EventTypes.Contains(ProjectHookRepository.EventTypes.ErrorRegression) && ctx.IsRegression ||
hook.EventTypes.Contains(ProjectHookRepository.EventTypes.CriticalError) && ctx.Error.Tags != null && ctx.Error.Tags.Contains("Critical");
if (!shouldCall)
{
continue;
}
Log.Trace().Project(ctx.Error.ProjectId).Message("Web hook queued: project={0} url={1}", ctx.Error.ProjectId, hook.Url).Write();
messageProducer.Publish(new WebHookNotification {
ProjectId = ctx.Error.ProjectId,
Url = hook.Url,
Data = WebHookError.FromError(ctx, _projectRepository, _errorStackRepository, _organizationRepository)
});
}
}
}
public override void Process(ErrorPipelineContext ctx)
{
if (ctx.StackInfo == null || !ctx.StackInfo.DateFixed.HasValue || ctx.StackInfo.DateFixed.Value >= ctx.Error.OccurrenceDate.UtcDateTime)
{
return;
}
Log.Trace().Message("Marking error as an regression.").Write();
_errorStackRepository.Collection.Update(
Query.EQ(ErrorStackRepository.FieldNames.Id, new BsonObjectId(new ObjectId(ctx.StackInfo.Id))),
Update
.Unset(ErrorStackRepository.FieldNames.DateFixed)
.Set(ErrorStackRepository.FieldNames.IsRegressed, true));
_errorStackRepository.InvalidateCache(ctx.Error.ErrorStackId, ctx.StackInfo.SignatureHash, ctx.Error.ProjectId);
ctx.IsRegression = true;
}
public override void Process(ErrorPipelineContext ctx)
{
if (Settings.Current.WebsiteMode == WebsiteMode.Dev)
{
return;
}
// Throttle errors by client ip address to no more than X every 5 minutes.
string clientIp = null;
if (ctx.Error.RequestInfo != null && !String.IsNullOrEmpty(ctx.Error.RequestInfo.ClientIpAddress))
{
clientIp = ctx.Error.RequestInfo.ClientIpAddress;
}
if (String.IsNullOrEmpty(clientIp))
{
return;
}
string throttleCacheKey = String.Concat("bot:", clientIp, ":", DateTime.Now.Floor(_throttlingPeriod).Ticks);
var requestCount = _cacheClient.Get <int?>(throttleCacheKey);
if (requestCount != null)
{
_cacheClient.Increment(throttleCacheKey, 1);
requestCount++;
}
else
{
_cacheClient.Set(throttleCacheKey, 1, _throttlingPeriod);
requestCount = 1;
}
if (requestCount < Settings.Current.BotThrottleLimit)
{
return;
}
Log.Info().Message("Bot throttle triggered. IP: {0} Time: {1} Project: {2}", clientIp, DateTime.Now.Floor(_throttlingPeriod), ctx.Error.ProjectId).Project(ctx.Error.ProjectId).Write();
// the throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system
Task.Run(() => _errorRepository.RemoveAllByClientIpAndDateAsync(clientIp, DateTime.Now.Floor(_throttlingPeriod), DateTime.Now.Ceiling(_throttlingPeriod)));
ctx.IsCancelled = true;
}
public override void Process(ErrorPipelineContext ctx) {
ctx.StackingInfo = ctx.Error.GetStackingInfo(_signatureFactory);
if (String.IsNullOrEmpty(ctx.Error.ErrorStackId)) {
if (_stackRepository == null)
throw new InvalidOperationException("You must pass a non-null stackRepository parameter to the constructor.");
Log.Trace().Message("Error did not specify an ErrorStackId.").Write();
var signature = _signatureFactory.GetSignature(ctx.Error);
ctx.StackingInfo = ctx.Error.GetStackingInfo();
// Set Path to be the only thing we stack on for 404 errors
if (ctx.Error.Is404() && ctx.Error.RequestInfo != null) {
Log.Trace().Message("Updating SignatureInfo for 404 error.").Write();
signature.SignatureInfo.Clear();
signature.SignatureInfo.Add("HttpMethod", ctx.Error.RequestInfo.HttpMethod);
signature.SignatureInfo.Add("Path", ctx.Error.RequestInfo.Path);
signature.RecalculateHash();
}
ctx.StackInfo = _stackRepository.GetErrorStackInfoBySignatureHash(ctx.Error.ProjectId, signature.SignatureHash);
if (ctx.StackInfo == null) {
Log.Trace().Message("Creating new error stack.").Write();
ctx.IsNew = true;
var stack = new ErrorStack {
OrganizationId = ctx.Error.OrganizationId,
ProjectId = ctx.Error.ProjectId,
SignatureInfo = signature.SignatureInfo,
SignatureHash = signature.SignatureHash,
Title = ctx.StackingInfo.Message,
Tags = ctx.Error.Tags ?? new TagSet(),
TotalOccurrences = 1,
FirstOccurrence = ctx.Error.OccurrenceDate.UtcDateTime,
LastOccurrence = ctx.Error.OccurrenceDate.UtcDateTime
};
_stackRepository.Add(stack, true);
ctx.StackInfo = new ErrorStackInfo {
Id = stack.Id,
DateFixed = stack.DateFixed,
OccurrencesAreCritical = stack.OccurrencesAreCritical,
SignatureHash = stack.SignatureHash
};
// new 404 stack id added, invalidate 404 id cache
if (signature.SignatureInfo.ContainsKey("Path"))
_stackRepository.InvalidateNotFoundIdsCache(ctx.Error.ProjectId);
}
Log.Trace().Message("Updating error's ErrorStackId to: {0}", ctx.StackInfo.Id).Write();
ctx.Error.ErrorStackId = ctx.StackInfo.Id;
} else {
var stack = _stackRepository.GetByIdCached(ctx.Error.ErrorStackId, true);
// TODO: Update unit tests to work with this check.
//if (stack == null || stack.ProjectId != error.ProjectId)
// throw new InvalidOperationException("Invalid ErrorStackId.");
if (stack == null)
return;
if (ctx.Error.Tags != null && ctx.Error.Tags.Count > 0) {
if(stack.Tags == null)
stack.Tags = new TagSet();
List<string> newTags = ctx.Error.Tags.Where(t => !stack.Tags.Contains(t)).ToList();
if (newTags.Count > 0) {
stack.Tags.AddRange(newTags);
_stackRepository.Update(stack);
}
}
ctx.StackInfo = new ErrorStackInfo {
Id = stack.Id,
DateFixed = stack.DateFixed,
OccurrencesAreCritical = stack.OccurrencesAreCritical,
SignatureHash = stack.SignatureHash,
IsHidden = stack.IsHidden
};
}
// sync the fixed and hidden flags to the error occurrence
ctx.Error.IsFixed = ctx.StackInfo.DateFixed.HasValue;
ctx.Error.IsHidden = ctx.StackInfo.IsHidden;
}
public override void Process(ErrorPipelineContext ctx)
{
ctx.StackingInfo = ctx.Error.GetStackingInfo();
if (String.IsNullOrEmpty(ctx.Error.ErrorStackId))
{
if (_stackRepository == null)
{
throw new InvalidOperationException("You must pass a non-null stackRepository parameter to the constructor.");
}
Log.Trace().Message("Error did not specify an ErrorStackId.").Write();
var signature = _signatureFactory.GetSignature(ctx.Error);
ctx.StackingInfo = ctx.Error.GetStackingInfo();
Log.Trace().Message("Created Error Signature. Error is null: {0}", ctx.Error == null).Write();
// Set Path to be the only thing we stack on for 404 errors
if (ctx.Error.Code == "404" && ctx.Error.RequestInfo != null)
{
Log.Trace().Message("Updating SignatureInfo for 404 error.").Write();
signature.SignatureInfo.Clear();
signature.SignatureInfo.Add("HttpMethod", ctx.Error.RequestInfo.HttpMethod);
signature.SignatureInfo.Add("Path", ctx.Error.RequestInfo.Path);
signature.RecalculateHash();
}
ctx.StackInfo = _stackRepository.GetErrorStackInfoBySignatureHash(ctx.Error.ProjectId, signature.SignatureHash);
if (ctx.StackInfo == null)
{
Log.Trace().Message("Creating new error stack.").Write();
ctx.IsNew = true;
var stack = new ErrorStack {
OrganizationId = ctx.Error.OrganizationId,
ProjectId = ctx.Error.ProjectId,
SignatureInfo = signature.SignatureInfo,
SignatureHash = signature.SignatureHash,
Title = ctx.StackingInfo.Message,
TotalOccurrences = 1,
FirstOccurrence = ctx.Error.OccurrenceDate.UtcDateTime,
LastOccurrence = ctx.Error.OccurrenceDate.UtcDateTime
};
_stackRepository.Add(stack);
ctx.StackInfo = new ErrorStackInfo {
Id = stack.Id,
DateFixed = stack.DateFixed,
OccurrencesAreCritical = stack.OccurrencesAreCritical,
SignatureHash = stack.SignatureHash
};
// new 404 stack id added, invalidate 404 id cache
if (signature.SignatureInfo.ContainsKey("Path"))
{
_stackRepository.InvalidateNotFoundIdsCache(ctx.Error.ProjectId);
}
}
Log.Trace().Message("Updating error's ErrorStackId to: {0}", ctx.StackInfo.Id).Write();
ctx.Error.ErrorStackId = ctx.StackInfo.Id;
}
else
{
var stack = _stackRepository.GetByIdCached(ctx.Error.ErrorStackId);
// TODO: Update unit tests to work with this check.
//if (stack == null || stack.ProjectId != error.ProjectId)
// throw new InvalidOperationException("Invalid ErrorStackId.");
if (stack != null)
{
ctx.StackInfo = new ErrorStackInfo {
Id = stack.Id,
DateFixed = stack.DateFixed,
OccurrencesAreCritical = stack.OccurrencesAreCritical,
SignatureHash = stack.SignatureHash
};
}
}
}
public override void Process(ErrorPipelineContext ctx)
{
using (IRedisClient client = _clientsManager.GetClient())
client.PublishMessage(NOTIFICATION_CHANNEL_KEY, String.Concat(ctx.Error.OrganizationId, ":", ctx.Error.ProjectId));
}
public override void Process(ErrorPipelineContext ctx) {
using (IRedisClient client = _clientsManager.GetClient())
client.PublishMessage(NOTIFICATION_CHANNEL_KEY, String.Concat(ctx.Error.OrganizationId, ":", ctx.Error.ProjectId, ":", ctx.Error.ErrorStackId, ":", ctx.Error.IsHidden, ":", ctx.Error.IsFixed, ":", ctx.Error.Is404()));
}
