public override void Process(ErrorPipelineContext ctx) { if (ctx.IsNew) { return; } int maxErrorsPerStack = 50; Organization organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId); if (organization != null) { maxErrorsPerStack = organization.MaxErrorsPerDay > 0 ? organization.MaxErrorsPerDay + Math.Min(50, organization.MaxErrorsPerDay * 2) : Int32.MaxValue; } // get a list of oldest ids that exceed our desired max errors var errors = _errorRepository.Collection.Find(Query.EQ(ErrorRepository.FieldNames.ErrorStackId, new BsonObjectId(new ObjectId(ctx.Error.ErrorStackId)))) .SetSortOrder(SortBy.Descending(ErrorRepository.FieldNames.OccurrenceDate_UTC)) .SetFields(ErrorRepository.FieldNames.Id) .Select(e => new Error { Id = e.Id, OrganizationId = ctx.Error.OrganizationId, ProjectId = ctx.Error.ProjectId, ErrorStackId = ctx.Error.ErrorStackId }) .Skip(maxErrorsPerStack) .Take(150).ToArray(); if (errors.Length > 0) { _errorRepository.Delete(errors); } }
public override void Process(ErrorPipelineContext ctx) { if (ctx.IsNew) return; int maxErrorsPerStack = 50; Organization organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId); if (organization != null) maxErrorsPerStack = organization.MaxErrorsPerDay > 0 ? organization.MaxErrorsPerDay + Math.Min(50, organization.MaxErrorsPerDay * 2) : Int32.MaxValue; // Get a list of oldest ids that exceed our desired max errors. var errors = _errorRepository.Collection.Find( Query.EQ(ErrorRepository.FieldNames.ErrorStackId, new BsonObjectId(new ObjectId(ctx.Error.ErrorStackId)))) .SetSortOrder(SortBy.Descending(ErrorRepository.FieldNames.OccurrenceDate_UTC)) .SetFields(ErrorRepository.FieldNames.Id) .SetSkip(maxErrorsPerStack) .SetLimit(150) .Select(e => new Error { Id = e.Id, OrganizationId = ctx.Error.OrganizationId, ProjectId = ctx.Error.ProjectId, ErrorStackId = ctx.Error.ErrorStackId }) .ToArray(); if (errors.Length > 0) _errorRepository.Delete(errors); }
public override void Process(ErrorPipelineContext ctx) { if (Settings.Current.WebsiteMode == WebsiteMode.Dev) return; // Throttle errors by client ip address to no more than X every 5 minutes. string clientIp = null; if (ctx.Error.RequestInfo != null && !String.IsNullOrEmpty(ctx.Error.RequestInfo.ClientIpAddress)) clientIp = ctx.Error.RequestInfo.ClientIpAddress; if (String.IsNullOrEmpty(clientIp)) return; string throttleCacheKey = String.Concat("bot:", clientIp, ":", DateTime.Now.Floor(_throttlingPeriod).Ticks); var requestCount = _cacheClient.Get<int?>(throttleCacheKey); if (requestCount != null) { _cacheClient.Increment(throttleCacheKey, 1); requestCount++; } else { _cacheClient.Set(throttleCacheKey, 1, _throttlingPeriod); requestCount = 1; } if (requestCount < Settings.Current.BotThrottleLimit) return; Log.Info().Message("Bot throttle triggered. IP: {0} Time: {1} Project: {2}", clientIp, DateTime.Now.Floor(_throttlingPeriod), ctx.Error.ProjectId).Project(ctx.Error.ProjectId).Write(); // the throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system Task.Run(() => _errorRepository.RemoveAllByClientIpAndDateAsync(clientIp, DateTime.Now.Floor(_throttlingPeriod), DateTime.Now.Ceiling(_throttlingPeriod))); ctx.IsCancelled = true; }
public override void Process(ErrorPipelineContext ctx) { if (ctx.StackInfo == null || !ctx.StackInfo.OccurrencesAreCritical) return; Log.Trace().Message("Marking error as critical.").Write(); ctx.Error.MarkAsCritical(); }
public override void Process(ErrorPipelineContext ctx) { _organizationRepository.IncrementStats(ctx.Error.OrganizationId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0); _projectRepository.IncrementStats(ctx.Error.ProjectId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0); if (!ctx.IsNew) _errorStackRepository.IncrementStats(ctx.Error.ErrorStackId, ctx.Error.OccurrenceDate.UtcDateTime); IEnumerable<TimeSpan> offsets = _projectRepository.GetTargetTimeOffsetsForStats(ctx.Error.ProjectId); _statsHelper.Process(ctx.Error, ctx.IsNew, offsets); }
public override void Process(ErrorPipelineContext ctx) { if (ctx.StackInfo == null || !ctx.StackInfo.OccurrencesAreCritical) { return; } Log.Trace().Message("Marking error as critical.").Write(); ctx.Error.MarkAsCritical(); }
public override void Process(ErrorPipelineContext ctx) { var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId); if (organization == null) return; _stats.Counter(StatNames.ErrorsProcessed); if (organization.PlanId != BillingManager.FreePlan.Id) _stats.Counter(StatNames.ErrorsPaidProcessed); }
public override void Process(ErrorPipelineContext ctx) { try { ctx.Error = _errorRepository.Add(ctx.Error); } catch (WriteConcernException ex) { // ignore errors being submitted multiple times if (ex.Message.Contains("E11000")) { Log.Info().Project(ctx.Error.ProjectId).Message("Ignoring duplicate error submission: {0}", ctx.Error.Id).Write(); ctx.IsCancelled = true; } else throw; } }
public static WebHookError FromError(ErrorPipelineContext ctx, IProjectRepository projectRepository, IErrorStackRepository errorStackRepository, IOrganizationRepository organizationRepository) { if (ctx == null || ctx.Error == null) throw new ArgumentNullException("ctx"); if (projectRepository == null) throw new ArgumentNullException("projectRepository"); if (errorStackRepository == null) throw new ArgumentNullException("errorStackRepository"); if (organizationRepository == null) throw new ArgumentNullException("organizationRepository"); var project = projectRepository.GetByIdCached(ctx.Error.ProjectId); if (project == null) throw new ArgumentException("ProjectId not found."); var organization = organizationRepository.GetByIdCached(ctx.Error.OrganizationId); if (organization == null) throw new ArgumentException("OrganizationId not found."); var errorStack = errorStackRepository.GetByIdCached(ctx.Error.ErrorStackId); if (errorStack == null) throw new ArgumentException("ErrorStackId not found."); return new WebHookError { Id = ctx.Error.Id, OccurrenceDate = ctx.Error.OccurrenceDate, Tags = ctx.Error.Tags, MachineName = ctx.Error.EnvironmentInfo != null ? ctx.Error.EnvironmentInfo.MachineName : null, RequestPath = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.GetFullPath() : null, IpAddress = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.ClientIpAddress : ctx.Error.EnvironmentInfo != null ? ctx.Error.EnvironmentInfo.IpAddress : null, Message = ctx.Error.Message, Type = ctx.Error.Type, Code = ctx.Error.Code, TargetMethod = ctx.Error.TargetMethod != null ? ctx.Error.TargetMethod.FullName : null, ProjectId = ctx.Error.ProjectId, ProjectName = project.Name, OrganizationId = ctx.Error.OrganizationId, OrganizationName = organization.Name, ErrorStackId = ctx.Error.ErrorStackId, ErrorStackTitle = errorStack.Title, ErrorStackDescription = errorStack.Description, ErrorStackTags = errorStack.Tags, TotalOccurrences = errorStack.TotalOccurrences, FirstOccurrence = errorStack.FirstOccurrence, LastOccurrence = errorStack.LastOccurrence, DateFixed = errorStack.DateFixed, IsRegression = ctx.IsRegression, IsNew = ctx.IsNew }; }
public override void Process(ErrorPipelineContext ctx) { _organizationRepository.IncrementStats(ctx.Error.OrganizationId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0); _projectRepository.IncrementStats(ctx.Error.ProjectId, errorCount: 1, stackCount: ctx.IsNew ? 1 : 0); if (!ctx.IsNew) { _errorStackRepository.IncrementStats(ctx.Error.ErrorStackId, ctx.Error.OccurrenceDate.UtcDateTime); } IEnumerable <TimeSpan> offsets = _projectRepository.GetTargetTimeOffsetsForStats(ctx.Error.ProjectId); _statsHelper.Process(ctx.Error, ctx.IsNew, offsets); }
public override void Process(ErrorPipelineContext ctx) { if (ctx.StackInfo == null || !ctx.StackInfo.DateFixed.HasValue || ctx.StackInfo.DateFixed.Value >= ctx.Error.OccurrenceDate.UtcDateTime) return; Log.Trace().Message("Marking error as an regression.").Write(); _errorStackRepository.Collection.Update( Query.EQ(ErrorStackRepository.FieldNames.Id, new BsonObjectId(new ObjectId(ctx.StackInfo.Id))), Update .Unset(ErrorStackRepository.FieldNames.DateFixed) .Set(ErrorStackRepository.FieldNames.IsRegressed, true)); _errorStackRepository.InvalidateCache(ctx.Error.ErrorStackId, ctx.StackInfo.SignatureHash, ctx.Error.ProjectId); ctx.IsRegression = true; }
public override void Process(ErrorPipelineContext ctx) { var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId); if (organization == null) { return; } _stats.Counter(StatNames.ErrorsProcessed); if (organization.PlanId != BillingManager.FreePlan.Id) { _stats.Counter(StatNames.ErrorsPaidProcessed); } }
public override void Process(ErrorPipelineContext ctx) { try { ctx.Error = _errorRepository.Add(ctx.Error); } catch (WriteConcernException ex) { // ignore errors being submitted multiple times if (ex.Message.Contains("E11000")) { Log.Info().Project(ctx.Error.ProjectId).Message("Ignoring duplicate error submission: {0}", ctx.Error.Id).Write(); ctx.IsCancelled = true; } else { throw; } } }
public override void Process(ErrorPipelineContext ctx) { var organization = _organizationRepository.GetByIdCached(ctx.Error.OrganizationId); // if they don't have premium features, then we don't need to queue notifications if (organization != null && !organization.HasPremiumFeatures) { return; } using (IMessageProducer messageProducer = _messageFactory.CreateMessageProducer()) { messageProducer.Publish(new ErrorNotification { ErrorId = ctx.Error.Id, ErrorStackId = ctx.Error.ErrorStackId, FullTypeName = ctx.StackingInfo.FullTypeName, IsNew = ctx.IsNew, IsCritical = ctx.Error.Tags != null && ctx.Error.Tags.Contains("Critical"), IsRegression = ctx.IsRegression, Message = ctx.StackingInfo.Message, ProjectId = ctx.Error.ProjectId, Code = ctx.Error.Code, UserAgent = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.UserAgent : null, Url = ctx.Error.RequestInfo != null ? ctx.Error.RequestInfo.GetFullPath(true, true) : null }); foreach (ProjectHook hook in _projectHookRepository.GetByProjectId(ctx.Error.ProjectId)) { bool shouldCall = hook.EventTypes.Contains(ProjectHookRepository.EventTypes.NewError) && ctx.IsNew || hook.EventTypes.Contains(ProjectHookRepository.EventTypes.ErrorRegression) && ctx.IsRegression || hook.EventTypes.Contains(ProjectHookRepository.EventTypes.CriticalError) && ctx.Error.Tags != null && ctx.Error.Tags.Contains("Critical"); if (!shouldCall) { continue; } Log.Trace().Project(ctx.Error.ProjectId).Message("Web hook queued: project={0} url={1}", ctx.Error.ProjectId, hook.Url).Write(); messageProducer.Publish(new WebHookNotification { ProjectId = ctx.Error.ProjectId, Url = hook.Url, Data = WebHookError.FromError(ctx, _projectRepository, _errorStackRepository, _organizationRepository) }); } } }
public override void Process(ErrorPipelineContext ctx) { if (ctx.StackInfo == null || !ctx.StackInfo.DateFixed.HasValue || ctx.StackInfo.DateFixed.Value >= ctx.Error.OccurrenceDate.UtcDateTime) { return; } Log.Trace().Message("Marking error as an regression.").Write(); _errorStackRepository.Collection.Update( Query.EQ(ErrorStackRepository.FieldNames.Id, new BsonObjectId(new ObjectId(ctx.StackInfo.Id))), Update .Unset(ErrorStackRepository.FieldNames.DateFixed) .Set(ErrorStackRepository.FieldNames.IsRegressed, true)); _errorStackRepository.InvalidateCache(ctx.Error.ErrorStackId, ctx.StackInfo.SignatureHash, ctx.Error.ProjectId); ctx.IsRegression = true; }
public override void Process(ErrorPipelineContext ctx) { if (Settings.Current.WebsiteMode == WebsiteMode.Dev) { return; } // Throttle errors by client ip address to no more than X every 5 minutes. string clientIp = null; if (ctx.Error.RequestInfo != null && !String.IsNullOrEmpty(ctx.Error.RequestInfo.ClientIpAddress)) { clientIp = ctx.Error.RequestInfo.ClientIpAddress; } if (String.IsNullOrEmpty(clientIp)) { return; } string throttleCacheKey = String.Concat("bot:", clientIp, ":", DateTime.Now.Floor(_throttlingPeriod).Ticks); var requestCount = _cacheClient.Get <int?>(throttleCacheKey); if (requestCount != null) { _cacheClient.Increment(throttleCacheKey, 1); requestCount++; } else { _cacheClient.Set(throttleCacheKey, 1, _throttlingPeriod); requestCount = 1; } if (requestCount < Settings.Current.BotThrottleLimit) { return; } Log.Info().Message("Bot throttle triggered. IP: {0} Time: {1} Project: {2}", clientIp, DateTime.Now.Floor(_throttlingPeriod), ctx.Error.ProjectId).Project(ctx.Error.ProjectId).Write(); // the throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system Task.Run(() => _errorRepository.RemoveAllByClientIpAndDateAsync(clientIp, DateTime.Now.Floor(_throttlingPeriod), DateTime.Now.Ceiling(_throttlingPeriod))); ctx.IsCancelled = true; }
public override void Process(ErrorPipelineContext ctx) { ctx.StackingInfo = ctx.Error.GetStackingInfo(_signatureFactory); if (String.IsNullOrEmpty(ctx.Error.ErrorStackId)) { if (_stackRepository == null) throw new InvalidOperationException("You must pass a non-null stackRepository parameter to the constructor."); Log.Trace().Message("Error did not specify an ErrorStackId.").Write(); var signature = _signatureFactory.GetSignature(ctx.Error); ctx.StackingInfo = ctx.Error.GetStackingInfo(); // Set Path to be the only thing we stack on for 404 errors if (ctx.Error.Is404() && ctx.Error.RequestInfo != null) { Log.Trace().Message("Updating SignatureInfo for 404 error.").Write(); signature.SignatureInfo.Clear(); signature.SignatureInfo.Add("HttpMethod", ctx.Error.RequestInfo.HttpMethod); signature.SignatureInfo.Add("Path", ctx.Error.RequestInfo.Path); signature.RecalculateHash(); } ctx.StackInfo = _stackRepository.GetErrorStackInfoBySignatureHash(ctx.Error.ProjectId, signature.SignatureHash); if (ctx.StackInfo == null) { Log.Trace().Message("Creating new error stack.").Write(); ctx.IsNew = true; var stack = new ErrorStack { OrganizationId = ctx.Error.OrganizationId, ProjectId = ctx.Error.ProjectId, SignatureInfo = signature.SignatureInfo, SignatureHash = signature.SignatureHash, Title = ctx.StackingInfo.Message, Tags = ctx.Error.Tags ?? new TagSet(), TotalOccurrences = 1, FirstOccurrence = ctx.Error.OccurrenceDate.UtcDateTime, LastOccurrence = ctx.Error.OccurrenceDate.UtcDateTime }; _stackRepository.Add(stack, true); ctx.StackInfo = new ErrorStackInfo { Id = stack.Id, DateFixed = stack.DateFixed, OccurrencesAreCritical = stack.OccurrencesAreCritical, SignatureHash = stack.SignatureHash }; // new 404 stack id added, invalidate 404 id cache if (signature.SignatureInfo.ContainsKey("Path")) _stackRepository.InvalidateNotFoundIdsCache(ctx.Error.ProjectId); } Log.Trace().Message("Updating error's ErrorStackId to: {0}", ctx.StackInfo.Id).Write(); ctx.Error.ErrorStackId = ctx.StackInfo.Id; } else { var stack = _stackRepository.GetByIdCached(ctx.Error.ErrorStackId, true); // TODO: Update unit tests to work with this check. //if (stack == null || stack.ProjectId != error.ProjectId) // throw new InvalidOperationException("Invalid ErrorStackId."); if (stack == null) return; if (ctx.Error.Tags != null && ctx.Error.Tags.Count > 0) { if(stack.Tags == null) stack.Tags = new TagSet(); List<string> newTags = ctx.Error.Tags.Where(t => !stack.Tags.Contains(t)).ToList(); if (newTags.Count > 0) { stack.Tags.AddRange(newTags); _stackRepository.Update(stack); } } ctx.StackInfo = new ErrorStackInfo { Id = stack.Id, DateFixed = stack.DateFixed, OccurrencesAreCritical = stack.OccurrencesAreCritical, SignatureHash = stack.SignatureHash, IsHidden = stack.IsHidden }; } // sync the fixed and hidden flags to the error occurrence ctx.Error.IsFixed = ctx.StackInfo.DateFixed.HasValue; ctx.Error.IsHidden = ctx.StackInfo.IsHidden; }
public override void Process(ErrorPipelineContext ctx) { ctx.StackingInfo = ctx.Error.GetStackingInfo(); if (String.IsNullOrEmpty(ctx.Error.ErrorStackId)) { if (_stackRepository == null) { throw new InvalidOperationException("You must pass a non-null stackRepository parameter to the constructor."); } Log.Trace().Message("Error did not specify an ErrorStackId.").Write(); var signature = _signatureFactory.GetSignature(ctx.Error); ctx.StackingInfo = ctx.Error.GetStackingInfo(); Log.Trace().Message("Created Error Signature. Error is null: {0}", ctx.Error == null).Write(); // Set Path to be the only thing we stack on for 404 errors if (ctx.Error.Code == "404" && ctx.Error.RequestInfo != null) { Log.Trace().Message("Updating SignatureInfo for 404 error.").Write(); signature.SignatureInfo.Clear(); signature.SignatureInfo.Add("HttpMethod", ctx.Error.RequestInfo.HttpMethod); signature.SignatureInfo.Add("Path", ctx.Error.RequestInfo.Path); signature.RecalculateHash(); } ctx.StackInfo = _stackRepository.GetErrorStackInfoBySignatureHash(ctx.Error.ProjectId, signature.SignatureHash); if (ctx.StackInfo == null) { Log.Trace().Message("Creating new error stack.").Write(); ctx.IsNew = true; var stack = new ErrorStack { OrganizationId = ctx.Error.OrganizationId, ProjectId = ctx.Error.ProjectId, SignatureInfo = signature.SignatureInfo, SignatureHash = signature.SignatureHash, Title = ctx.StackingInfo.Message, TotalOccurrences = 1, FirstOccurrence = ctx.Error.OccurrenceDate.UtcDateTime, LastOccurrence = ctx.Error.OccurrenceDate.UtcDateTime }; _stackRepository.Add(stack); ctx.StackInfo = new ErrorStackInfo { Id = stack.Id, DateFixed = stack.DateFixed, OccurrencesAreCritical = stack.OccurrencesAreCritical, SignatureHash = stack.SignatureHash }; // new 404 stack id added, invalidate 404 id cache if (signature.SignatureInfo.ContainsKey("Path")) { _stackRepository.InvalidateNotFoundIdsCache(ctx.Error.ProjectId); } } Log.Trace().Message("Updating error's ErrorStackId to: {0}", ctx.StackInfo.Id).Write(); ctx.Error.ErrorStackId = ctx.StackInfo.Id; } else { var stack = _stackRepository.GetByIdCached(ctx.Error.ErrorStackId); // TODO: Update unit tests to work with this check. //if (stack == null || stack.ProjectId != error.ProjectId) // throw new InvalidOperationException("Invalid ErrorStackId."); if (stack != null) { ctx.StackInfo = new ErrorStackInfo { Id = stack.Id, DateFixed = stack.DateFixed, OccurrencesAreCritical = stack.OccurrencesAreCritical, SignatureHash = stack.SignatureHash }; } } }
public override void Process(ErrorPipelineContext ctx) { using (IRedisClient client = _clientsManager.GetClient()) client.PublishMessage(NOTIFICATION_CHANNEL_KEY, String.Concat(ctx.Error.OrganizationId, ":", ctx.Error.ProjectId)); }
public override void Process(ErrorPipelineContext ctx) { using (IRedisClient client = _clientsManager.GetClient()) client.PublishMessage(NOTIFICATION_CHANNEL_KEY, String.Concat(ctx.Error.OrganizationId, ":", ctx.Error.ProjectId, ":", ctx.Error.ErrorStackId, ":", ctx.Error.IsHidden, ":", ctx.Error.IsFixed, ":", ctx.Error.Is404())); }