public async Task EnsureSingleRegressionAsync() { var utcNow = SystemClock.UtcNow; PersistentEvent ev = EventData.GenerateEvent(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow); var context = new EventContext(ev); await _pipeline.RunAsync(context); Assert.True(context.IsProcessed); Assert.False(context.IsRegression); Assert.False(context.Event.IsFixed); ev = await _eventRepository.GetByIdAsync(ev.Id); Assert.NotNull(ev); var stack = await _stackRepository.GetByIdAsync(ev.StackId); stack.MarkFixed(); await _stackRepository.SaveAsync(stack, true); var contexts = new List <EventContext> { new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(-1))), new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(-1))) }; await _configuration.Client.RefreshAsync(); await _pipeline.RunAsync(contexts); Assert.Equal(0, contexts.Count(c => c.IsRegression)); Assert.Equal(2, contexts.Count(c => !c.IsRegression)); Assert.True(contexts.All(c => c.Event.IsFixed)); contexts = new List <EventContext> { new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1))), new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1))) }; await _configuration.Client.RefreshAsync(); await _pipeline.RunAsync(contexts); Assert.Equal(1, contexts.Count(c => c.IsRegression)); Assert.Equal(1, contexts.Count(c => !c.IsRegression)); Assert.True(contexts.All(c => !c.Event.IsFixed)); contexts = new List <EventContext> { new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1))), new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1))) }; await _configuration.Client.RefreshAsync(); await _pipeline.RunAsync(contexts); Assert.Equal(2, contexts.Count(c => !c.IsRegression)); Assert.True(contexts.All(c => !c.Event.IsFixed)); }
public async Task CanGetByFixedAsync() { var stack = await _repository.AddAsync(StackData.GenerateStack(id: TestConstants.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId)); await RefreshDataAsync(); var results = await _repository.GetByFilterAsync(null, "fixed:true", null, null, DateTime.MinValue, DateTime.MaxValue); Assert.NotNull(results); Assert.Equal(0, results.Total); results = await _repository.GetByFilterAsync(null, "fixed:false", null, null, DateTime.MinValue, DateTime.MaxValue); Assert.NotNull(results); Assert.Equal(1, results.Total); Assert.False(results.Documents.Single().IsRegressed); Assert.Null(results.Documents.Single().DateFixed); stack.MarkFixed(); await _repository.SaveAsync(stack); await RefreshDataAsync(); results = await _repository.GetByFilterAsync(null, "fixed:true", null, null, DateTime.MinValue, DateTime.MaxValue); Assert.NotNull(results); Assert.Equal(1, results.Total); Assert.False(results.Documents.Single().IsRegressed); Assert.NotNull(results.Documents.Single().DateFixed); results = await _repository.GetByFilterAsync(null, "fixed:false", null, null, DateTime.MinValue, DateTime.MaxValue); Assert.NotNull(results); Assert.Equal(0, results.Total); }
public async Task CanGetByFixedAsync() { var stack = await _repository.AddAsync(StackData.GenerateStack(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId), o => o.ImmediateConsistency()); var results = await _repository.FindAsync(q => q.FilterExpression("fixed:true")); Assert.NotNull(results); Assert.Equal(0, results.Total); results = await _repository.FindAsync(q => q.FilterExpression("fixed:false")); Assert.NotNull(results); Assert.Equal(1, results.Total); Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed); Assert.Null(results.Documents.Single().DateFixed); stack.MarkFixed(); await _repository.SaveAsync(stack, o => o.ImmediateConsistency()); results = await _repository.FindAsync(q => q.FilterExpression("fixed:true")); Assert.NotNull(results); Assert.Equal(1, results.Total); Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed); Assert.NotNull(results.Documents.Single().DateFixed); results = await _repository.FindAsync(q => q.FilterExpression("fixed:false")); Assert.NotNull(results); Assert.Equal(0, results.Total); }
public async Task CanCreateUpdateRemoveAsync() { await ResetAsync(); await _repository.RemoveAllAsync(); await _client.RefreshAsync(); Assert.Equal(0, await _repository.CountAsync()); var stack = StackData.GenerateStack(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId); Assert.Null(stack.Id); await _repository.AddAsync(stack); Assert.NotNull(stack.Id); await _client.RefreshAsync(); stack = await _repository.GetByIdAsync(stack.Id); Assert.NotNull(stack); stack.Description = "New Description"; await _repository.SaveAsync(stack); await _repository.RemoveAsync(stack.Id); }
public async Task EnsureSingleRegressionAsync() { await ResetAsync(); var pipeline = IoC.GetInstance <EventPipeline>(); PersistentEvent ev = EventData.GenerateEvent(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow); var context = new EventContext(ev); await pipeline.RunAsync(context); await _client.RefreshAsync(); Assert.True(context.IsProcessed); Assert.False(context.IsRegression); ev = await _eventRepository.GetByIdAsync(ev.Id); Assert.NotNull(ev); var stack = await _stackRepository.GetByIdAsync(ev.StackId); stack.DateFixed = DateTime.UtcNow; stack.IsRegressed = false; await _stackRepository.SaveAsync(stack, true); var contexts = new List <EventContext> { new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1))), new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1))) }; await pipeline.RunAsync(contexts); await _client.RefreshAsync(); Assert.Equal(1, contexts.Count(c => c.IsRegression)); Assert.Equal(1, contexts.Count(c => !c.IsRegression)); contexts = new List <EventContext> { new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1))), new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1))) }; await pipeline.RunAsync(contexts); await _client.RefreshAsync(); Assert.Equal(2, contexts.Count(c => !c.IsRegression)); }
protected override async Task <JobResult> RunInternalAsync(JobContext context) { const int LIMIT = 100; _lastRun = SystemClock.UtcNow; _logger.LogTrace("Start save stack event counts."); // Get list of stacks where snooze has expired var results = await _stackRepository.GetExpiredSnoozedStatuses(SystemClock.UtcNow, o => o.PageLimit(LIMIT)).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var stack in results.Documents) { stack.MarkOpen(); } await _stackRepository.SaveAsync(results.Documents).AnyContext(); // Sleep so we are not hammering the backend. await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext(); if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext()) { break; } if (results.Documents.Count > 0) { await context.RenewLockAsync().AnyContext(); } } _logger.LogTrace("Finished save stack event counts."); return(JobResult.Success); }
public override async Task RunAsync(MigrationContext context) { _logger.LogInformation("Getting duplicate stacks"); var duplicateStackAgg = await _client.SearchAsync <Stack>(q => q .QueryOnQueryString("is_deleted:false") .Size(0) .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); var buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; int total = buckets.Count; int processed = 0; int error = 0; long totalUpdatedEventCount = 0; var lastStatus = SystemClock.Now; int batch = 1; while (buckets.Count > 0) { _logger.LogInformation($"Found {total} duplicate stacks in batch #{batch}."); foreach (var duplicateSignature in buckets) { string projectId = null; string signature = null; try { var parts = duplicateSignature.Key.Split(':'); if (parts.Length != 2) { _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key); continue; } projectId = parts[0]; signature = parts[1]; var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FilterExpression($"signature_hash:{signature}")); if (stacks.Documents.Count < 2) { _logger.LogError("Did not find multiple stacks with signature {SignatureHash} and project {ProjectId}", signature, projectId); continue; } var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id")); var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List <Foundatio.Repositories.Models.KeyedBucket <string> >(); // we only need to update events if more than one stack has events associated to it bool shouldUpdateEvents = eventCountBuckets.Count > 1; // default to using the oldest stack var targetStack = stacks.Documents.OrderBy(s => s.CreatedUtc).First(); var duplicateStacks = stacks.Documents.OrderBy(s => s.CreatedUtc).Skip(1).ToList(); // use the stack that has the most events on it so we can reduce the number of updates if (eventCountBuckets.Count > 0) { var targetStackId = eventCountBuckets.OrderByDescending(b => b.Total).First().Key; targetStack = stacks.Documents.Single(d => d.Id == targetStackId); duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).ToList(); } targetStack.CreatedUtc = stacks.Documents.Min(d => d.CreatedUtc); targetStack.Status = stacks.Documents.FirstOrDefault(d => d.Status != StackStatus.Open)?.Status ?? StackStatus.Open; targetStack.LastOccurrence = stacks.Documents.Max(d => d.LastOccurrence); targetStack.SnoozeUntilUtc = stacks.Documents.Max(d => d.SnoozeUntilUtc); targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed);; targetStack.TotalOccurrences += duplicateStacks.Sum(d => d.TotalOccurrences); targetStack.Tags.AddRange(duplicateStacks.SelectMany(d => d.Tags)); targetStack.References = stacks.Documents.SelectMany(d => d.References).Distinct().ToList(); targetStack.OccurrencesAreCritical = stacks.Documents.Any(d => d.OccurrencesAreCritical); duplicateStacks.ForEach(s => s.IsDeleted = true); await _stackRepository.SaveAsync(duplicateStacks); await _stackRepository.SaveAsync(targetStack); processed++; long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0; _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents); if (shouldUpdateEvents) { var response = await _client.UpdateByQueryAsync <PersistentEvent>(u => u .Query(q => q.Bool(b => b.Must(m => m .Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id))) ))) .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless)) .Conflicts(Elasticsearch.Net.Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(response, LogLevel.Trace); var taskStartedTime = SystemClock.Now; var taskId = response.Task; int attempts = 0; long affectedRecords = 0; do { attempts++; var taskStatus = await _client.Tasks.GetTaskAsync(taskId); var status = taskStatus.Task.Status; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) { _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); } affectedRecords += status.Created + status.Updated + status.Deleted; break; } if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) { _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); } var delay = TimeSpan.FromMilliseconds(50); if (attempts > 20) { delay = TimeSpan.FromSeconds(5); } else if (attempts > 10) { delay = TimeSpan.FromSeconds(1); } else if (attempts > 5) { delay = TimeSpan.FromMilliseconds(250); } await Task.Delay(delay); } while (true); _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); totalUpdatedEventCount += affectedRecords; } if (SystemClock.UtcNow.Subtract(lastStatus) > TimeSpan.FromSeconds(5)) { lastStatus = SystemClock.UtcNow; _logger.LogInformation("Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); await _cache.RemoveByPrefixAsync(nameof(Stack)); } } catch (Exception ex) { error++; _logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}", projectId, signature); } } await _client.Indices.RefreshAsync(_config.Stacks.VersionedName); duplicateStackAgg = await _client.SearchAsync <Stack>(q => q .QueryOnQueryString("is_deleted:false") .Size(0) .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; total += buckets.Count; batch++; _logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); await _cache.RemoveByPrefixAsync(nameof(Stack)); } }
public override async Task ProcessBatchAsync(ICollection <EventContext> contexts) { var stacks = new Dictionary <string, Tuple <bool, Stack> >(); foreach (var ctx in contexts) { if (String.IsNullOrEmpty(ctx.Event.StackId)) { // only add default signature info if no other signature info has been added if (ctx.StackSignatureData.Count == 0) { ctx.StackSignatureData.AddItemIfNotEmpty("Type", ctx.Event.Type); ctx.StackSignatureData.AddItemIfNotEmpty("Source", ctx.Event.Source); } string signatureHash = ctx.StackSignatureData.Values.ToSHA1(); ctx.SignatureHash = signatureHash; Tuple <bool, Stack> value; if (stacks.TryGetValue(signatureHash, out value)) { ctx.Stack = value.Item2; } else { ctx.Stack = await _stackRepository.GetStackBySignatureHashAsync(ctx.Event.ProjectId, signatureHash).AnyContext(); if (ctx.Stack != null) { stacks.Add(signatureHash, Tuple.Create(false, ctx.Stack)); } } if (ctx.Stack == null) { Logger.Trace().Message("Creating new event stack.").Write(); ctx.IsNew = true; string title = _formattingPluginManager.GetStackTitle(ctx.Event); var stack = new Stack { OrganizationId = ctx.Event.OrganizationId, ProjectId = ctx.Event.ProjectId, SignatureInfo = new SettingsDictionary(ctx.StackSignatureData), SignatureHash = signatureHash, Title = title?.Truncate(1000), Tags = ctx.Event.Tags ?? new TagSet(), Type = ctx.Event.Type, TotalOccurrences = 1, FirstOccurrence = ctx.Event.Date.UtcDateTime, LastOccurrence = ctx.Event.Date.UtcDateTime }; ctx.Stack = stack; stacks.Add(signatureHash, Tuple.Create(true, ctx.Stack)); } } else { ctx.Stack = await _stackRepository.GetByIdAsync(ctx.Event.StackId, true).AnyContext(); if (ctx.Stack == null || ctx.Stack.ProjectId != ctx.Event.ProjectId) { ctx.SetError("Invalid StackId."); continue; } ctx.SignatureHash = ctx.Stack.SignatureHash; if (!stacks.ContainsKey(ctx.Stack.SignatureHash)) { stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(false, ctx.Stack)); } else { stacks[ctx.Stack.SignatureHash] = Tuple.Create(false, ctx.Stack); } } if (!ctx.IsNew && ctx.Event.Tags != null && ctx.Event.Tags.Count > 0) { if (ctx.Stack.Tags == null) { ctx.Stack.Tags = new TagSet(); } List <string> newTags = ctx.Event.Tags.Where(t => !ctx.Stack.Tags.Contains(t)).ToList(); if (newTags.Count > 0) { ctx.Stack.Tags.AddRange(newTags); // make sure the stack gets saved if (!stacks.ContainsKey(ctx.Stack.SignatureHash)) { stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(true, ctx.Stack)); } else { stacks[ctx.Stack.SignatureHash] = Tuple.Create(true, stacks[ctx.Stack.SignatureHash].Item2); } } } ctx.Event.IsFirstOccurrence = ctx.IsNew; // sync the fixed and hidden flags to the error occurrence ctx.Event.IsFixed = ctx.Stack.DateFixed.HasValue; ctx.Event.IsHidden = ctx.Stack.IsHidden; } var stacksToAdd = stacks.Where(kvp => kvp.Value.Item1 && String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList(); if (stacksToAdd.Count > 0) { await _stackRepository.AddAsync(stacksToAdd, true, sendNotification : stacksToAdd.Count == 1).AnyContext(); if (stacksToAdd.Count > 1) { await _publisher.PublishAsync(new ExtendedEntityChanged { ChangeType = ChangeType.Added, Type = typeof(Stack).Name, OrganizationId = contexts.First().Organization.Id, ProjectId = contexts.First().Project.Id }).AnyContext(); } } var stacksToSave = stacks.Where(kvp => kvp.Value.Item1 && !String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList(); if (stacksToSave.Count > 0) { await _stackRepository.SaveAsync(stacksToSave, true, sendNotification : false).AnyContext(); // notification will get sent later in the update stats step } // Set stack ids after they have been saved and created contexts.ForEach(ctx => { ctx.Event.StackId = ctx.Stack?.Id; }); }
public async Task CanRunJobWithDiscardedEventUsage() { Log.MinimumLevel = LogLevel.Debug; var organization = await _organizationRepository.GetByIdAsync(TestConstants.OrganizationId); var usage = await _usageService.GetUsageAsync(organization); Assert.Equal(0, usage.MonthlyTotal); usage = await _usageService.GetUsageAsync(organization); Assert.Equal(0, usage.MonthlyTotal); Assert.Equal(0, usage.MonthlyBlocked); var ev = GenerateEvent(type: Event.KnownTypes.Log, source: "test", userIdentity: "test1"); Assert.NotNull(await EnqueueEventPostAsync(ev)); var result = await _job.RunAsync(); Assert.True(result.IsSuccess); await RefreshDataAsync(); var events = await _eventRepository.GetAllAsync(); Assert.Equal(2, events.Total); var logEvent = events.Documents.Single(e => String.Equals(e.Type, Event.KnownTypes.Log)); Assert.NotNull(logEvent); var sessionEvent = events.Documents.Single(e => String.Equals(e.Type, Event.KnownTypes.Session)); Assert.NotNull(sessionEvent); usage = await _usageService.GetUsageAsync(organization); Assert.Equal(1, usage.MonthlyTotal); Assert.Equal(0, usage.MonthlyBlocked); // Mark the stack as discarded var logStack = await _stackRepository.GetByIdAsync(logEvent.StackId); logStack.Status = StackStatus.Discarded; await _stackRepository.SaveAsync(logStack, o => o.ImmediateConsistency()); var sessionStack = await _stackRepository.GetByIdAsync(sessionEvent.StackId); sessionStack.Status = StackStatus.Discarded; await _stackRepository.SaveAsync(sessionStack, o => o.ImmediateConsistency()); // Verify job processed discarded events. Assert.NotNull(await EnqueueEventPostAsync(new List <PersistentEvent> { GenerateEvent(type: Event.KnownTypes.Session, sessionId: "abcdefghi"), GenerateEvent(type: Event.KnownTypes.Log, source: "test", sessionId: "abcdefghi"), GenerateEvent(type: Event.KnownTypes.Log, source: "test", userIdentity: "test3") })); result = await _job.RunAsync(); Assert.True(result.IsSuccess); await RefreshDataAsync(); events = await _eventRepository.GetAllAsync(); Assert.Equal(3, events.Total); usage = await _usageService.GetUsageAsync(organization); Assert.Equal(1, usage.MonthlyTotal); Assert.Equal(0, usage.MonthlyBlocked); }
public override async Task ProcessBatchAsync(ICollection <EventContext> contexts) { var stacks = new Dictionary <string, Tuple <bool, Stack> >(); foreach (var ctx in contexts) { if (String.IsNullOrEmpty(ctx.Event.StackId)) { // only add default signature info if no other signature info has been added if (ctx.StackSignatureData.Count == 0) { ctx.StackSignatureData.AddItemIfNotEmpty("Type", ctx.Event.Type); ctx.StackSignatureData.AddItemIfNotEmpty("Source", ctx.Event.Source); } string signatureHash = ctx.StackSignatureData.Values.ToSHA1(); ctx.SignatureHash = signatureHash; if (stacks.TryGetValue(signatureHash, out var value)) { ctx.Stack = value.Item2; } else { ctx.Stack = await _stackRepository.GetStackBySignatureHashAsync(ctx.Event.ProjectId, signatureHash).AnyContext(); if (ctx.Stack != null) { stacks.Add(signatureHash, Tuple.Create(false, ctx.Stack)); } } if (ctx.Stack == null) { _logger.LogTrace("Creating new event stack."); ctx.IsNew = true; string title = _formattingPluginManager.GetStackTitle(ctx.Event); var stack = new Stack { OrganizationId = ctx.Event.OrganizationId, ProjectId = ctx.Event.ProjectId, SignatureInfo = new SettingsDictionary(ctx.StackSignatureData), SignatureHash = signatureHash, DuplicateSignature = ctx.Event.ProjectId + ":" + signatureHash, Title = title?.Truncate(1000), Tags = ctx.Event.Tags ?? new TagSet(), Type = ctx.Event.Type, TotalOccurrences = 1, FirstOccurrence = ctx.Event.Date.UtcDateTime, LastOccurrence = ctx.Event.Date.UtcDateTime }; if (ctx.Event.Type == Event.KnownTypes.Session) { stack.Status = StackStatus.Ignored; } ctx.Stack = stack; stacks.Add(signatureHash, Tuple.Create(true, ctx.Stack)); } } else { ctx.Stack = await _stackRepository.GetByIdAsync(ctx.Event.StackId, o => o.Cache()).AnyContext(); if (ctx.Stack == null || ctx.Stack.ProjectId != ctx.Event.ProjectId) { ctx.SetError("Invalid StackId."); continue; } ctx.SignatureHash = ctx.Stack.SignatureHash; if (!stacks.ContainsKey(ctx.Stack.SignatureHash)) { stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(false, ctx.Stack)); } else { stacks[ctx.Stack.SignatureHash] = Tuple.Create(false, ctx.Stack); } if (ctx.Stack.Status == StackStatus.Discarded) { ctx.IsDiscarded = true; ctx.IsCancelled = true; } } if (!ctx.IsNew && ctx.Event.Tags != null && ctx.Event.Tags.Count > 0) { if (ctx.Stack.Tags == null) { ctx.Stack.Tags = new TagSet(); } var newTags = ctx.Event.Tags.Where(t => !ctx.Stack.Tags.Contains(t)).ToList(); if (newTags.Count > 0 || ctx.Stack.Tags.Count > 50 || ctx.Stack.Tags.Any(t => t.Length > 100)) { ctx.Stack.Tags.AddRange(newTags); ctx.Stack.Tags.RemoveExcessTags(); // make sure the stack gets saved if (!stacks.ContainsKey(ctx.Stack.SignatureHash)) { stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(true, ctx.Stack)); } else { stacks[ctx.Stack.SignatureHash] = Tuple.Create(true, stacks[ctx.Stack.SignatureHash].Item2); } } } ctx.Event.IsFirstOccurrence = ctx.IsNew; } var stacksToAdd = stacks.Where(kvp => kvp.Value.Item1 && String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList(); if (stacksToAdd.Count > 0) { await _stackRepository.AddAsync(stacksToAdd, o => o.Cache().Notifications(stacksToAdd.Count == 1)).AnyContext(); if (stacksToAdd.Count > 1) { await _publisher.PublishAsync(new EntityChanged { ChangeType = ChangeType.Added, Type = StackTypeName, Data = { { ExtendedEntityChanged.KnownKeys.OrganizationId, contexts.First().Organization.Id }, { ExtendedEntityChanged.KnownKeys.ProjectId, contexts.First().Project.Id } } }).AnyContext(); } } var stacksToSave = stacks.Where(kvp => kvp.Value.Item1 && !String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList(); if (stacksToSave.Count > 0) { await _stackRepository.SaveAsync(stacksToSave, o => o.Cache().Notifications(false)).AnyContext(); // notification will get sent later in the update stats step } // Set stack ids after they have been saved and created contexts.ForEach(ctx => { ctx.Event.StackId = ctx.Stack?.Id; }); }