public override void EventProcessing(EventContext context)
{
if (!context.Event.IsError())
return;
Error error = context.Event.GetError();
if (error == null)
return;
if (String.IsNullOrWhiteSpace(context.Event.Message))
context.Event.Message = error.Message;
string[] commonUserMethods = { "DataContext.SubmitChanges", "Entities.SaveChanges" };
if (context.HasProperty("CommonMethods"))
commonUserMethods = context.GetProperty<string>("CommonMethods").SplitAndTrim(',');
string[] userNamespaces = null;
if (context.HasProperty("UserNamespaces"))
userNamespaces = context.GetProperty<string>("UserNamespaces").SplitAndTrim(',');
var signature = new ErrorSignature(error, userCommonMethods: commonUserMethods, userNamespaces: userNamespaces);
if (signature.SignatureInfo.Count <= 0)
return;
foreach (var key in signature.SignatureInfo.Keys)
context.StackSignatureData.Add(key, signature.SignatureInfo[key]);
}
private async Task SetBrowserOsAndDeviceFromUserAgent(RequestInfo request, EventContext context) {
var info = await _parser.ParseAsync(request.UserAgent, context.Project.Id).AnyContext();
if (info != null) {
if (!String.Equals(info.UserAgent.Family, "Other")) {
request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family;
if (!String.IsNullOrEmpty(info.UserAgent.Major)) {
request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major;
}
}
if (!String.Equals(info.Device.Family, "Other"))
request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family;
if (!String.Equals(info.OS.Family, "Other")) {
request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family;
if (!String.IsNullOrEmpty(info.OS.Major)) {
request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major;
}
}
var botPatterns = context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList();
request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns);
}
}
public override void Process(EventContext ctx) {
if (ctx.StackInfo == null || !ctx.StackInfo.OccurrencesAreCritical)
return;
Log.Trace().Message("Marking error as critical.").Write();
ctx.Event.MarkAsCritical();
}
public override void EventProcessing(EventContext context)
{
if (Settings.Current.WebsiteMode == WebsiteMode.Dev)
return;
var project = _projectRepository.GetById(context.Event.ProjectId);
if (project == null || !project.DeleteBotDataEnabled)
return;
// Throttle errors by client ip address to no more than X every 5 minutes.
var ri = context.Event.GetRequestInfo();
if (ri == null || String.IsNullOrEmpty(ri.ClientIpAddress))
return;
string throttleCacheKey = String.Concat("bot:", ri.ClientIpAddress, ":", DateTime.Now.Floor(_throttlingPeriod).Ticks);
var requestCount = _cacheClient.Get<int?>(throttleCacheKey);
if (requestCount != null) {
_cacheClient.Increment(throttleCacheKey, 1);
requestCount++;
} else {
_cacheClient.Set(throttleCacheKey, 1, _throttlingPeriod);
requestCount = 1;
}
if (requestCount < Settings.Current.BotThrottleLimit)
return;
Log.Info().Message("Bot throttle triggered. IP: {0} Time: {1} Project: {2}", ri.ClientIpAddress, DateTime.Now.Floor(_throttlingPeriod), context.Event.ProjectId).Project(context.Event.ProjectId).Write();
// the throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system
Task.Run(() => _eventRepository.HideAllByClientIpAndDateAsync(context.Event.OrganizationId, ri.ClientIpAddress, DateTime.Now.Floor(_throttlingPeriod), DateTime.Now.Ceiling(_throttlingPeriod)));
context.IsCancelled = true;
}
public override void Process(EventContext ctx)
{
// if they don't have premium features, then we don't need to queue notifications
if (!ctx.Organization.HasPremiumFeatures)
return;
_notificationQueue.EnqueueAsync(new EventNotification {
Event = ctx.Event,
IsNew = ctx.IsNew,
IsCritical = ctx.Event.IsCritical(),
IsRegression = ctx.IsRegression,
//TotalOccurrences = ctx.Stack.TotalOccurrences,
ProjectName = ctx.Project.Name
}).Wait();
foreach (WebHook hook in _webHookRepository.GetByOrganizationIdOrProjectId(ctx.Event.OrganizationId, ctx.Event.ProjectId)) {
bool shouldCall = hook.EventTypes.Contains(WebHookRepository.EventTypes.NewError) && ctx.IsNew
|| hook.EventTypes.Contains(WebHookRepository.EventTypes.ErrorRegression) && ctx.IsRegression
|| hook.EventTypes.Contains(WebHookRepository.EventTypes.CriticalError) && ctx.Event.Tags != null && ctx.Event.Tags.Contains("Critical");
if (!shouldCall)
continue;
Log.Trace().Project(ctx.Event.ProjectId).Message("Web hook queued: project={0} url={1}", ctx.Event.ProjectId, hook.Url).Write();
// TODO: Should we be using the hook's project id and organization id?
var context = new WebHookDataContext(hook.Version, ctx.Event, ctx.Organization, ctx.Project, ctx.Stack, ctx.IsNew, ctx.IsRegression);
_webHookNotificationQueue.EnqueueAsync(new WebHookNotification {
OrganizationId = ctx.Event.OrganizationId,
ProjectId = ctx.Event.ProjectId,
Url = hook.Url,
Data = _webHookDataPluginManager.CreateFromEvent(context)
}).Wait();
}
}
public override Task EventProcessingAsync(EventContext context) {
var data = context.Event.Data;
if (data.ContainsKey(Event.KnownDataKeys.ManualStackingKey) && data[Event.KnownDataKeys.ManualStackingKey] != null)
context.StackSignatureData.AddItemIfNotEmpty(nameof(Event.KnownDataKeys.ManualStackingKey), data[Event.KnownDataKeys.ManualStackingKey].ToString());
return Task.CompletedTask;
}
public override async Task ProcessAsync(EventContext ctx) {
// if they don't have premium features, then we don't need to queue notifications
if (!ctx.Organization.HasPremiumFeatures)
return;
if (ShouldQueueNotification(ctx))
await _notificationQueue.EnqueueAsync(new EventNotificationWorkItem {
EventId = ctx.Event.Id,
IsNew = ctx.IsNew,
IsCritical = ctx.Event.IsCritical(),
IsRegression = ctx.IsRegression,
TotalOccurrences = ctx.Stack.TotalOccurrences,
ProjectName = ctx.Project.Name
}).AnyContext();
foreach (WebHook hook in (await _webHookRepository.GetByOrganizationIdOrProjectIdAsync(ctx.Event.OrganizationId, ctx.Event.ProjectId).AnyContext()).Documents) {
if (!ShouldCallWebHook(hook, ctx))
continue;
var context = new WebHookDataContext(hook.Version, ctx.Event, ctx.Organization, ctx.Project, ctx.Stack, ctx.IsNew, ctx.IsRegression);
var notification = new WebHookNotification {
OrganizationId = ctx.Event.OrganizationId,
ProjectId = ctx.Event.ProjectId,
Url = hook.Url,
Data = await _webHookDataPluginManager.CreateFromEventAsync(context).AnyContext()
};
await _webHookNotificationQueue.EnqueueAsync(notification).AnyContext();
Logger.Trace().Project(ctx.Event.ProjectId).Message("Web hook queued: project={0} url={1}", ctx.Event.ProjectId, hook.Url).Property("Web Hook Notification", notification).Write();
}
}
public override Task EventProcessingAsync(EventContext context) {
if (!context.Event.IsError())
return Task.CompletedTask;
Error error = context.Event.GetError();
if (error == null)
return Task.CompletedTask;
if (String.IsNullOrWhiteSpace(context.Event.Message))
context.Event.Message = error.Message;
string[] commonUserMethods = { "DataContext.SubmitChanges", "Entities.SaveChanges" };
if (context.HasProperty("CommonMethods"))
commonUserMethods = context.GetProperty<string>("CommonMethods").SplitAndTrim(',');
string[] userNamespaces = null;
if (context.HasProperty("UserNamespaces"))
userNamespaces = context.GetProperty<string>("UserNamespaces").SplitAndTrim(',');
var signature = new ErrorSignature(error, userCommonMethods: commonUserMethods, userNamespaces: userNamespaces);
if (signature.SignatureInfo.Count <= 0)
return Task.CompletedTask;
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
var stackingTarget = error.GetStackingTarget();
if (stackingTarget?.Error?.StackTrace != null && stackingTarget.Error.StackTrace.Count > 0 && !targetInfo.ContainsKey("Message"))
targetInfo["Message"] = stackingTarget.Error.Message;
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
foreach (var key in signature.SignatureInfo.Keys)
context.StackSignatureData.Add(key, signature.SignatureInfo[key]);
return Task.CompletedTask;
}
public override async Task ProcessAsync(EventContext ctx) {
if (ctx.Stack == null || !ctx.Stack.OccurrencesAreCritical)
return;
Log.Trace().Message("Marking error as critical.").Write();
ctx.Event.MarkAsCritical();
}
public async Task AddManualStackSignatureData(string dataKey, string dataValue, bool willAddManualStackSignature) {
var plugin = new ManualStackingPlugin();
var data = new DataDictionary() { { dataKey, dataValue } };
var context = new EventContext(new PersistentEvent { Data = data });
await plugin.EventBatchProcessingAsync(new List<EventContext> { context });
Assert.Equal(willAddManualStackSignature, context.StackSignatureData.Count > 0);
}
public override Task ProcessAsync(EventContext ctx) {
if (ctx.Stack == null || !ctx.Stack.OccurrencesAreCritical)
return Task.CompletedTask;
_logger.Trace("Marking error as critical.");
ctx.Event.MarkAsCritical();
return Task.CompletedTask;
}
public override void Process(EventContext ctx)
{
try {
ctx.Event = _eventRepository.Add(ctx.Event);
} catch (DuplicateDocumentException ex) {
Log.Info().Project(ctx.Event.ProjectId).Message("Ignoring duplicate error submission: {0}", ctx.Event.Id).Write();
ctx.IsCancelled = true;
}
}
public async Task AddManualStackSignatureData(string stackingKey, bool willAddManualStackSignature) {
var ev = new PersistentEvent();
ev.SetManualStackingKey(stackingKey);
var context = new EventContext(ev);
var plugin = new ManualStackingPlugin();
await plugin.EventBatchProcessingAsync(new List<EventContext> { context });
Assert.Equal(willAddManualStackSignature, context.StackSignatureData.Count > 0);
}
public override Task EventProcessingAsync(EventContext context) {
var msi = context.Event.GetManualStackingInfo();
if (msi?.SignatureData != null) {
foreach (var kvp in msi.SignatureData)
context.StackSignatureData.AddItemIfNotEmpty(kvp.Key, kvp.Value);
}
return Task.CompletedTask;
}
public override Task ProcessAsync(EventContext ctx) {
if (!ctx.Organization.HasPremiumFeatures)
return Task.CompletedTask;
// TODO: Do we need a pipeline action to trim keys and remove null values that may be sent by other native clients.
ctx.Event.CopyDataToIndex();
return Task.CompletedTask;
}
public override async Task EventProcessingAsync(EventContext context) {
if (String.IsNullOrEmpty(context.Event.ReferenceId))
return;
// TODO: Look into using a lock on reference id so we can ensure there is no race conditions with setting keys
if (await _cacheClient.AddAsync(GetCacheKey(context), true, TimeSpan.FromMinutes(1)).AnyContext())
return;
context.IsCancelled = true;
}
public override void Process(EventContext ctx)
{
if (ctx.Organization.RetentionDays <= 0)
return;
if (DateTimeOffset.Now.UtcDateTime.Subtract(ctx.Event.Date.UtcDateTime).Days <= ctx.Organization.RetentionDays)
return;
Log.Info().Project(ctx.Event.ProjectId).Message("Discarding event that occurred outside of your retention limit.").Write();
ctx.IsCancelled = true;
}
public override void Process(EventContext ctx)
{
if (ctx.Event.Tags != null)
ctx.Event.Tags.RemoveWhere(t => String.IsNullOrEmpty(t) || t.Length > 255);
if (ctx.Event.Message != null && ctx.Event.Message.Length > 2000)
ctx.Event.Message = ctx.Event.Message.Truncate(2000);
if (ctx.Event.Source != null && ctx.Event.Source.Length > 2000)
ctx.Event.Source = ctx.Event.Source.Truncate(2000);
}
public override async Task EventProcessingAsync(EventContext context) {
if (String.IsNullOrEmpty(context.Event.ReferenceId))
return;
// TODO: Look into using a lock on reference id so we can ensure there is no race conditions with setting keys
if (await _cacheClient.AddAsync(GetCacheKey(context), true, TimeSpan.FromMinutes(1)).AnyContext())
return;
_logger.Warn().Project(context.Event.ProjectId).Message("Discarding event due to duplicate reference id: {0}", context.Event.ReferenceId).Write();
context.IsCancelled = true;
}
public override void Process(EventContext ctx)
{
// TODO: Implement batch incrementing to reduce pipeline cost.
//_organizationCounters.AddOrUpdate(ctx.Event.OrganizationId, 1, (key, value) => value + 1);
//_projectCounters.AddOrUpdate(ctx.Event.ProjectId, 1, (key, value) => value + 1);
_organizationRepository.IncrementEventCounter(ctx.Event.OrganizationId);
_projectRepository.IncrementEventCounter(ctx.Event.ProjectId);
if (!ctx.IsNew)
_stackRepository.IncrementEventCounter(ctx.Event.OrganizationId, ctx.Event.StackId, ctx.Event.Date.UtcDateTime);
}
public override void EventProcessing(EventContext context)
{
var request = context.Event.GetRequestInfo();
if (request == null)
return;
var exclusions = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.DataExclusions)
? DefaultExclusions.Union(context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions)).ToList()
: DefaultExclusions;
context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH));
}
public override void Process(EventContext ctx) {
Task.Factory.StartNewDelayed(1500, () => _publisher.Publish(new EventOccurrence {
Id = ctx.Event.Id,
OrganizationId = ctx.Event.OrganizationId,
ProjectId = ctx.Event.ProjectId,
StackId = ctx.Event.StackId,
Type = ctx.Event.Type,
IsHidden = ctx.Event.IsHidden,
IsFixed = ctx.Event.IsFixed,
IsNotFound = ctx.Event.IsNotFound(),
IsRegression = ctx.IsRegression
}));
}
public override void Process(EventContext ctx) {
if (ctx.StackInfo == null || !ctx.StackInfo.DateFixed.HasValue || ctx.StackInfo.DateFixed.Value >= ctx.Event.Date.UtcDateTime)
return;
Log.Trace().Message("Marking event as an regression.").Write();
_stackRepository.MarkAsRegressed(ctx.StackInfo.Id);
_eventRepository.MarkAsRegressedByStack(ctx.StackInfo.Id);
string signatureHash = ctx.GetProperty<string>("__SignatureHash");
_stackRepository.InvalidateCache(ctx.Event.StackId, signatureHash, ctx.Event.ProjectId);
ctx.Event.IsFixed = false;
ctx.IsRegression = true;
}
public override async Task ProcessAsync(EventContext ctx) {
if (ctx.Event.Tags != null)
ctx.Event.Tags.RemoveWhere(t => String.IsNullOrEmpty(t) || t.Length > 255);
if (ctx.Event.Message != null && ctx.Event.Message.Length > 2000)
ctx.Event.Message = ctx.Event.Message.Truncate(2000);
else if (String.IsNullOrEmpty(ctx.Event.Message))
ctx.Event.Message = null;
if (ctx.Event.Source != null && ctx.Event.Source.Length > 2000)
ctx.Event.Source = ctx.Event.Source.Truncate(2000);
else if (String.IsNullOrEmpty(ctx.Event.Source))
ctx.Event.Source = null;
}
public override void Process(EventContext ctx)
{
_publisher.PublishAsync(new EventOccurrence {
Id = ctx.Event.Id,
OrganizationId = ctx.Event.OrganizationId,
ProjectId = ctx.Event.ProjectId,
StackId = ctx.Event.StackId,
Type = ctx.Event.Type,
IsHidden = ctx.Event.IsHidden,
IsFixed = ctx.Event.IsFixed,
IsNotFound = ctx.Event.IsNotFound(),
IsRegression = ctx.IsRegression
});
}
public override async Task ProcessAsync(EventContext ctx) {
if (ctx.Organization.RetentionDays <= 0)
return;
// If the date is in the future, set it to now using the same offset.
if (DateTimeOffset.Now.UtcDateTime < ctx.Event.Date.UtcDateTime)
ctx.Event.Date = ctx.Event.Date.Subtract(ctx.Event.Date.UtcDateTime - DateTimeOffset.UtcNow);
// Discard events that are being submitted outside of the plan retention limit.
if (DateTimeOffset.Now.UtcDateTime.Subtract(ctx.Event.Date.UtcDateTime).Days <= ctx.Organization.RetentionDays)
return;
Log.Info().Project(ctx.Event.ProjectId).Message("Discarding event that occurred outside of your retention limit.").Write();
ctx.IsCancelled = true;
}
public override void EventProcessing(EventContext context) {
if (!context.Event.IsError())
return;
SimpleError error = context.Event.GetSimpleError();
if (error == null)
return;
if (String.IsNullOrWhiteSpace(context.Event.Message))
context.Event.Message = error.Message;
// TODO: Parse the stack trace and run it through the ErrorSignature.
context.StackSignatureData.Add("ExceptionType", error.Type);
context.StackSignatureData.Add("StackTrace", error.StackTrace.ToSHA1());
}
public override Task ProcessAsync(EventContext ctx) {
ctx.Event.Tags?.RemoveWhere(t => String.IsNullOrEmpty(t) || t.Length > 255);
if (ctx.Event.Message != null && ctx.Event.Message.Length > 2000)
ctx.Event.Message = ctx.Event.Message.Truncate(2000);
else if (String.IsNullOrEmpty(ctx.Event.Message))
ctx.Event.Message = null;
if (ctx.Event.Source != null && ctx.Event.Source.Length > 2000)
ctx.Event.Source = ctx.Event.Source.Truncate(2000);
else if (String.IsNullOrEmpty(ctx.Event.Source))
ctx.Event.Source = null;
return TaskHelper.Completed();
}
public override Task EventProcessingAsync(EventContext context) {
var request = context.Event.GetRequestInfo();
if (request == null)
return TaskHelper.Completed();
var exclusions = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.DataExclusions)
? DefaultExclusions.Union(context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions)).ToList()
: DefaultExclusions;
if (!String.IsNullOrEmpty(request.UserAgent)) {
try {
var info = Parser.GetDefault().Parse(request.UserAgent);
if (!String.Equals(info.UserAgent.Family, "Other")) {
request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family;
if (!String.IsNullOrEmpty(info.UserAgent.Major)) {
request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major;
}
}
if (!String.Equals(info.Device.Family, "Other"))
request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family;
if (!String.Equals(info.OS.Family, "Other")) {
request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family;
if (!String.IsNullOrEmpty(info.OS.Major)) {
request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major;
}
}
var botPatterns = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns)
? context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList()
: new List<string>();
request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns);
} catch (Exception ex) {
Log.Warn().Project(context.Event.ProjectId).Message("Unable to parse user agent {0}. Exception: {1}", request.UserAgent, ex.Message).Write();
}
}
context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH));
return TaskHelper.Completed();
}
public override Task ProcessAsync(EventContext ctx) {
if (!ctx.Organization.HasPremiumFeatures)
return TaskHelper.Completed();
// TODO: Do we need a pipeline action to trim keys and remove null values that may be sent by other native clients.
foreach (string key in ctx.Event.Data.Keys.Where(k => !k.StartsWith("@")).ToArray()) {
string field = key.Trim().ToLower().Replace(' ', '-');
if (field.StartsWith("@") || ctx.Event.Data[key] == null)
continue;
Type dataType = ctx.Event.Data[key].GetType();
if (dataType == typeof(bool)) {
ctx.Event.Idx[field + "-b"] = ctx.Event.Data[key];
} else if (dataType.IsNumeric()) {
ctx.Event.Idx[field + "-n"] = ctx.Event.Data[key];
} else if (dataType == typeof(DateTime) || dataType == typeof(DateTimeOffset)) {
ctx.Event.Idx[field + "-d"] = ctx.Event.Data[key];
} else if (dataType == typeof(string)) {
var input = (string)ctx.Event.Data[key];
if (String.IsNullOrEmpty(input) || input.Length >= 1000)
continue;
if (input.GetJsonType() != JsonType.None)
continue;
if (input[0] == '"')
input = input.TrimStart('"').TrimEnd('"');
bool value;
DateTimeOffset dtoValue;
Decimal decValue;
Double dblValue;
if (Boolean.TryParse(input, out value))
ctx.Event.Idx[field + "-b"] = value;
else if (DateTimeOffset.TryParse(input, out dtoValue))
ctx.Event.Idx[field + "-d"] = dtoValue;
else if (Decimal.TryParse(input, out decValue))
ctx.Event.Idx[field + "-n"] = decValue;
else if (Double.TryParse(input, out dblValue))
ctx.Event.Idx[field + "-n"] = dblValue;
else
ctx.Event.Idx[field + "-s"] = input;
}
}
return TaskHelper.Completed();
}
private string GetCacheKey(EventContext context)
{
return(String.Concat("Project:", context.Project.Id, ":", context.Event.ReferenceId));
}
