public async Task EnsureSingleRegressionAsync()
{
var utcNow = SystemClock.UtcNow;
PersistentEvent ev = EventData.GenerateEvent(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow);
var context = new EventContext(ev);
await _pipeline.RunAsync(context);
Assert.True(context.IsProcessed);
Assert.False(context.IsRegression);
Assert.False(context.Event.IsFixed);
ev = await _eventRepository.GetByIdAsync(ev.Id);
Assert.NotNull(ev);
var stack = await _stackRepository.GetByIdAsync(ev.StackId);
stack.MarkFixed();
await _stackRepository.SaveAsync(stack, true);
var contexts = new List <EventContext> {
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(-1))),
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(-1)))
};
await _configuration.Client.RefreshAsync();
await _pipeline.RunAsync(contexts);
Assert.Equal(0, contexts.Count(c => c.IsRegression));
Assert.Equal(2, contexts.Count(c => !c.IsRegression));
Assert.True(contexts.All(c => c.Event.IsFixed));
contexts = new List <EventContext> {
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1))),
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1)))
};
await _configuration.Client.RefreshAsync();
await _pipeline.RunAsync(contexts);
Assert.Equal(1, contexts.Count(c => c.IsRegression));
Assert.Equal(1, contexts.Count(c => !c.IsRegression));
Assert.True(contexts.All(c => !c.Event.IsFixed));
contexts = new List <EventContext> {
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1))),
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: utcNow.AddMinutes(1)))
};
await _configuration.Client.RefreshAsync();
await _pipeline.RunAsync(contexts);
Assert.Equal(2, contexts.Count(c => !c.IsRegression));
Assert.True(contexts.All(c => !c.Event.IsFixed));
}
public async Task CanGetByFixedAsync()
{
var stack = await _repository.AddAsync(StackData.GenerateStack(id: TestConstants.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId));
await RefreshDataAsync();
var results = await _repository.GetByFilterAsync(null, "fixed:true", null, null, DateTime.MinValue, DateTime.MaxValue);
Assert.NotNull(results);
Assert.Equal(0, results.Total);
results = await _repository.GetByFilterAsync(null, "fixed:false", null, null, DateTime.MinValue, DateTime.MaxValue);
Assert.NotNull(results);
Assert.Equal(1, results.Total);
Assert.False(results.Documents.Single().IsRegressed);
Assert.Null(results.Documents.Single().DateFixed);
stack.MarkFixed();
await _repository.SaveAsync(stack);
await RefreshDataAsync();
results = await _repository.GetByFilterAsync(null, "fixed:true", null, null, DateTime.MinValue, DateTime.MaxValue);
Assert.NotNull(results);
Assert.Equal(1, results.Total);
Assert.False(results.Documents.Single().IsRegressed);
Assert.NotNull(results.Documents.Single().DateFixed);
results = await _repository.GetByFilterAsync(null, "fixed:false", null, null, DateTime.MinValue, DateTime.MaxValue);
Assert.NotNull(results);
Assert.Equal(0, results.Total);
}
public async Task CanGetByFixedAsync()
{
var stack = await _repository.AddAsync(StackData.GenerateStack(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId), o => o.ImmediateConsistency());
var results = await _repository.FindAsync(q => q.FilterExpression("fixed:true"));
Assert.NotNull(results);
Assert.Equal(0, results.Total);
results = await _repository.FindAsync(q => q.FilterExpression("fixed:false"));
Assert.NotNull(results);
Assert.Equal(1, results.Total);
Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed);
Assert.Null(results.Documents.Single().DateFixed);
stack.MarkFixed();
await _repository.SaveAsync(stack, o => o.ImmediateConsistency());
results = await _repository.FindAsync(q => q.FilterExpression("fixed:true"));
Assert.NotNull(results);
Assert.Equal(1, results.Total);
Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed);
Assert.NotNull(results.Documents.Single().DateFixed);
results = await _repository.FindAsync(q => q.FilterExpression("fixed:false"));
Assert.NotNull(results);
Assert.Equal(0, results.Total);
}
public async Task CanCreateUpdateRemoveAsync()
{
await ResetAsync();
await _repository.RemoveAllAsync();
await _client.RefreshAsync();
Assert.Equal(0, await _repository.CountAsync());
var stack = StackData.GenerateStack(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId);
Assert.Null(stack.Id);
await _repository.AddAsync(stack);
Assert.NotNull(stack.Id);
await _client.RefreshAsync();
stack = await _repository.GetByIdAsync(stack.Id);
Assert.NotNull(stack);
stack.Description = "New Description";
await _repository.SaveAsync(stack);
await _repository.RemoveAsync(stack.Id);
}
public async Task EnsureSingleRegressionAsync()
{
await ResetAsync();
var pipeline = IoC.GetInstance <EventPipeline>();
PersistentEvent ev = EventData.GenerateEvent(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow);
var context = new EventContext(ev);
await pipeline.RunAsync(context);
await _client.RefreshAsync();
Assert.True(context.IsProcessed);
Assert.False(context.IsRegression);
ev = await _eventRepository.GetByIdAsync(ev.Id);
Assert.NotNull(ev);
var stack = await _stackRepository.GetByIdAsync(ev.StackId);
stack.DateFixed = DateTime.UtcNow;
stack.IsRegressed = false;
await _stackRepository.SaveAsync(stack, true);
var contexts = new List <EventContext> {
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1))),
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1)))
};
await pipeline.RunAsync(contexts);
await _client.RefreshAsync();
Assert.Equal(1, contexts.Count(c => c.IsRegression));
Assert.Equal(1, contexts.Count(c => !c.IsRegression));
contexts = new List <EventContext> {
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1))),
new EventContext(EventData.GenerateEvent(stackId: ev.StackId, projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId, occurrenceDate: DateTime.UtcNow.AddMinutes(1)))
};
await pipeline.RunAsync(contexts);
await _client.RefreshAsync();
Assert.Equal(2, contexts.Count(c => !c.IsRegression));
}
protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
const int LIMIT = 100;
_lastRun = SystemClock.UtcNow;
_logger.LogTrace("Start save stack event counts.");
// Get list of stacks where snooze has expired
var results = await _stackRepository.GetExpiredSnoozedStatuses(SystemClock.UtcNow, o => o.PageLimit(LIMIT)).AnyContext();
while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
{
foreach (var stack in results.Documents)
{
stack.MarkOpen();
}
await _stackRepository.SaveAsync(results.Documents).AnyContext();
// Sleep so we are not hammering the backend.
await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext();
if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext())
{
break;
}
if (results.Documents.Count > 0)
{
await context.RenewLockAsync().AnyContext();
}
}
_logger.LogTrace("Finished save stack event counts.");
return(JobResult.Success);
}
public override async Task RunAsync(MigrationContext context)
{
_logger.LogInformation("Getting duplicate stacks");
var duplicateStackAgg = await _client.SearchAsync <Stack>(q => q
.QueryOnQueryString("is_deleted:false")
.Size(0)
.Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000))));
_logger.LogRequest(duplicateStackAgg, LogLevel.Trace);
var buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets;
int total = buckets.Count;
int processed = 0;
int error = 0;
long totalUpdatedEventCount = 0;
var lastStatus = SystemClock.Now;
int batch = 1;
while (buckets.Count > 0)
{
_logger.LogInformation($"Found {total} duplicate stacks in batch #{batch}.");
foreach (var duplicateSignature in buckets)
{
string projectId = null;
string signature = null;
try {
var parts = duplicateSignature.Key.Split(':');
if (parts.Length != 2)
{
_logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key);
continue;
}
projectId = parts[0];
signature = parts[1];
var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FilterExpression($"signature_hash:{signature}"));
if (stacks.Documents.Count < 2)
{
_logger.LogError("Did not find multiple stacks with signature {SignatureHash} and project {ProjectId}", signature, projectId);
continue;
}
var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id"));
var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List <Foundatio.Repositories.Models.KeyedBucket <string> >();
// we only need to update events if more than one stack has events associated to it
bool shouldUpdateEvents = eventCountBuckets.Count > 1;
// default to using the oldest stack
var targetStack = stacks.Documents.OrderBy(s => s.CreatedUtc).First();
var duplicateStacks = stacks.Documents.OrderBy(s => s.CreatedUtc).Skip(1).ToList();
// use the stack that has the most events on it so we can reduce the number of updates
if (eventCountBuckets.Count > 0)
{
var targetStackId = eventCountBuckets.OrderByDescending(b => b.Total).First().Key;
targetStack = stacks.Documents.Single(d => d.Id == targetStackId);
duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).ToList();
}
targetStack.CreatedUtc = stacks.Documents.Min(d => d.CreatedUtc);
targetStack.Status = stacks.Documents.FirstOrDefault(d => d.Status != StackStatus.Open)?.Status ?? StackStatus.Open;
targetStack.LastOccurrence = stacks.Documents.Max(d => d.LastOccurrence);
targetStack.SnoozeUntilUtc = stacks.Documents.Max(d => d.SnoozeUntilUtc);
targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed);;
targetStack.TotalOccurrences += duplicateStacks.Sum(d => d.TotalOccurrences);
targetStack.Tags.AddRange(duplicateStacks.SelectMany(d => d.Tags));
targetStack.References = stacks.Documents.SelectMany(d => d.References).Distinct().ToList();
targetStack.OccurrencesAreCritical = stacks.Documents.Any(d => d.OccurrencesAreCritical);
duplicateStacks.ForEach(s => s.IsDeleted = true);
await _stackRepository.SaveAsync(duplicateStacks);
await _stackRepository.SaveAsync(targetStack);
processed++;
long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0;
_logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents);
if (shouldUpdateEvents)
{
var response = await _client.UpdateByQueryAsync <PersistentEvent>(u => u
.Query(q => q.Bool(b => b.Must(m => m
.Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id)))
)))
.Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless))
.Conflicts(Elasticsearch.Net.Conflicts.Proceed)
.WaitForCompletion(false));
_logger.LogRequest(response, LogLevel.Trace);
var taskStartedTime = SystemClock.Now;
var taskId = response.Task;
int attempts = 0;
long affectedRecords = 0;
do
{
attempts++;
var taskStatus = await _client.Tasks.GetTaskAsync(taskId);
var status = taskStatus.Task.Status;
if (taskStatus.Completed)
{
// TODO: need to check to see if the task failed or completed successfully. Throw if it failed.
if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30))
{
_logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total);
}
affectedRecords += status.Created + status.Updated + status.Deleted;
break;
}
if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30))
{
_logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total);
}
var delay = TimeSpan.FromMilliseconds(50);
if (attempts > 20)
{
delay = TimeSpan.FromSeconds(5);
}
else if (attempts > 10)
{
delay = TimeSpan.FromSeconds(1);
}
else if (attempts > 5)
{
delay = TimeSpan.FromMilliseconds(250);
}
await Task.Delay(delay);
} while (true);
_logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id));
totalUpdatedEventCount += affectedRecords;
}
if (SystemClock.UtcNow.Subtract(lastStatus) > TimeSpan.FromSeconds(5))
{
lastStatus = SystemClock.UtcNow;
_logger.LogInformation("Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error);
await _cache.RemoveByPrefixAsync(nameof(Stack));
}
}
catch (Exception ex) {
error++;
_logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}", projectId, signature);
}
}
await _client.Indices.RefreshAsync(_config.Stacks.VersionedName);
duplicateStackAgg = await _client.SearchAsync <Stack>(q => q
.QueryOnQueryString("is_deleted:false")
.Size(0)
.Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000))));
_logger.LogRequest(duplicateStackAgg, LogLevel.Trace);
buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets;
total += buckets.Count;
batch++;
_logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error);
await _cache.RemoveByPrefixAsync(nameof(Stack));
}
}
public override async Task ProcessBatchAsync(ICollection <EventContext> contexts)
{
var stacks = new Dictionary <string, Tuple <bool, Stack> >();
foreach (var ctx in contexts)
{
if (String.IsNullOrEmpty(ctx.Event.StackId))
{
// only add default signature info if no other signature info has been added
if (ctx.StackSignatureData.Count == 0)
{
ctx.StackSignatureData.AddItemIfNotEmpty("Type", ctx.Event.Type);
ctx.StackSignatureData.AddItemIfNotEmpty("Source", ctx.Event.Source);
}
string signatureHash = ctx.StackSignatureData.Values.ToSHA1();
ctx.SignatureHash = signatureHash;
Tuple <bool, Stack> value;
if (stacks.TryGetValue(signatureHash, out value))
{
ctx.Stack = value.Item2;
}
else
{
ctx.Stack = await _stackRepository.GetStackBySignatureHashAsync(ctx.Event.ProjectId, signatureHash).AnyContext();
if (ctx.Stack != null)
{
stacks.Add(signatureHash, Tuple.Create(false, ctx.Stack));
}
}
if (ctx.Stack == null)
{
Logger.Trace().Message("Creating new event stack.").Write();
ctx.IsNew = true;
string title = _formattingPluginManager.GetStackTitle(ctx.Event);
var stack = new Stack {
OrganizationId = ctx.Event.OrganizationId,
ProjectId = ctx.Event.ProjectId,
SignatureInfo = new SettingsDictionary(ctx.StackSignatureData),
SignatureHash = signatureHash,
Title = title?.Truncate(1000),
Tags = ctx.Event.Tags ?? new TagSet(),
Type = ctx.Event.Type,
TotalOccurrences = 1,
FirstOccurrence = ctx.Event.Date.UtcDateTime,
LastOccurrence = ctx.Event.Date.UtcDateTime
};
ctx.Stack = stack;
stacks.Add(signatureHash, Tuple.Create(true, ctx.Stack));
}
}
else
{
ctx.Stack = await _stackRepository.GetByIdAsync(ctx.Event.StackId, true).AnyContext();
if (ctx.Stack == null || ctx.Stack.ProjectId != ctx.Event.ProjectId)
{
ctx.SetError("Invalid StackId.");
continue;
}
ctx.SignatureHash = ctx.Stack.SignatureHash;
if (!stacks.ContainsKey(ctx.Stack.SignatureHash))
{
stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(false, ctx.Stack));
}
else
{
stacks[ctx.Stack.SignatureHash] = Tuple.Create(false, ctx.Stack);
}
}
if (!ctx.IsNew && ctx.Event.Tags != null && ctx.Event.Tags.Count > 0)
{
if (ctx.Stack.Tags == null)
{
ctx.Stack.Tags = new TagSet();
}
List <string> newTags = ctx.Event.Tags.Where(t => !ctx.Stack.Tags.Contains(t)).ToList();
if (newTags.Count > 0)
{
ctx.Stack.Tags.AddRange(newTags);
// make sure the stack gets saved
if (!stacks.ContainsKey(ctx.Stack.SignatureHash))
{
stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(true, ctx.Stack));
}
else
{
stacks[ctx.Stack.SignatureHash] = Tuple.Create(true, stacks[ctx.Stack.SignatureHash].Item2);
}
}
}
ctx.Event.IsFirstOccurrence = ctx.IsNew;
// sync the fixed and hidden flags to the error occurrence
ctx.Event.IsFixed = ctx.Stack.DateFixed.HasValue;
ctx.Event.IsHidden = ctx.Stack.IsHidden;
}
var stacksToAdd = stacks.Where(kvp => kvp.Value.Item1 && String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList();
if (stacksToAdd.Count > 0)
{
await _stackRepository.AddAsync(stacksToAdd, true, sendNotification : stacksToAdd.Count == 1).AnyContext();
if (stacksToAdd.Count > 1)
{
await _publisher.PublishAsync(new ExtendedEntityChanged { ChangeType = ChangeType.Added, Type = typeof(Stack).Name, OrganizationId = contexts.First().Organization.Id, ProjectId = contexts.First().Project.Id }).AnyContext();
}
}
var stacksToSave = stacks.Where(kvp => kvp.Value.Item1 && !String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList();
if (stacksToSave.Count > 0)
{
await _stackRepository.SaveAsync(stacksToSave, true, sendNotification : false).AnyContext(); // notification will get sent later in the update stats step
}
// Set stack ids after they have been saved and created
contexts.ForEach(ctx => {
ctx.Event.StackId = ctx.Stack?.Id;
});
}
public async Task CanRunJobWithDiscardedEventUsage()
{
Log.MinimumLevel = LogLevel.Debug;
var organization = await _organizationRepository.GetByIdAsync(TestConstants.OrganizationId);
var usage = await _usageService.GetUsageAsync(organization);
Assert.Equal(0, usage.MonthlyTotal);
usage = await _usageService.GetUsageAsync(organization);
Assert.Equal(0, usage.MonthlyTotal);
Assert.Equal(0, usage.MonthlyBlocked);
var ev = GenerateEvent(type: Event.KnownTypes.Log, source: "test", userIdentity: "test1");
Assert.NotNull(await EnqueueEventPostAsync(ev));
var result = await _job.RunAsync();
Assert.True(result.IsSuccess);
await RefreshDataAsync();
var events = await _eventRepository.GetAllAsync();
Assert.Equal(2, events.Total);
var logEvent = events.Documents.Single(e => String.Equals(e.Type, Event.KnownTypes.Log));
Assert.NotNull(logEvent);
var sessionEvent = events.Documents.Single(e => String.Equals(e.Type, Event.KnownTypes.Session));
Assert.NotNull(sessionEvent);
usage = await _usageService.GetUsageAsync(organization);
Assert.Equal(1, usage.MonthlyTotal);
Assert.Equal(0, usage.MonthlyBlocked);
// Mark the stack as discarded
var logStack = await _stackRepository.GetByIdAsync(logEvent.StackId);
logStack.Status = StackStatus.Discarded;
await _stackRepository.SaveAsync(logStack, o => o.ImmediateConsistency());
var sessionStack = await _stackRepository.GetByIdAsync(sessionEvent.StackId);
sessionStack.Status = StackStatus.Discarded;
await _stackRepository.SaveAsync(sessionStack, o => o.ImmediateConsistency());
// Verify job processed discarded events.
Assert.NotNull(await EnqueueEventPostAsync(new List <PersistentEvent> {
GenerateEvent(type: Event.KnownTypes.Session, sessionId: "abcdefghi"),
GenerateEvent(type: Event.KnownTypes.Log, source: "test", sessionId: "abcdefghi"),
GenerateEvent(type: Event.KnownTypes.Log, source: "test", userIdentity: "test3")
}));
result = await _job.RunAsync();
Assert.True(result.IsSuccess);
await RefreshDataAsync();
events = await _eventRepository.GetAllAsync();
Assert.Equal(3, events.Total);
usage = await _usageService.GetUsageAsync(organization);
Assert.Equal(1, usage.MonthlyTotal);
Assert.Equal(0, usage.MonthlyBlocked);
}
public override async Task ProcessBatchAsync(ICollection <EventContext> contexts)
{
var stacks = new Dictionary <string, Tuple <bool, Stack> >();
foreach (var ctx in contexts)
{
if (String.IsNullOrEmpty(ctx.Event.StackId))
{
// only add default signature info if no other signature info has been added
if (ctx.StackSignatureData.Count == 0)
{
ctx.StackSignatureData.AddItemIfNotEmpty("Type", ctx.Event.Type);
ctx.StackSignatureData.AddItemIfNotEmpty("Source", ctx.Event.Source);
}
string signatureHash = ctx.StackSignatureData.Values.ToSHA1();
ctx.SignatureHash = signatureHash;
if (stacks.TryGetValue(signatureHash, out var value))
{
ctx.Stack = value.Item2;
}
else
{
ctx.Stack = await _stackRepository.GetStackBySignatureHashAsync(ctx.Event.ProjectId, signatureHash).AnyContext();
if (ctx.Stack != null)
{
stacks.Add(signatureHash, Tuple.Create(false, ctx.Stack));
}
}
if (ctx.Stack == null)
{
_logger.LogTrace("Creating new event stack.");
ctx.IsNew = true;
string title = _formattingPluginManager.GetStackTitle(ctx.Event);
var stack = new Stack {
OrganizationId = ctx.Event.OrganizationId,
ProjectId = ctx.Event.ProjectId,
SignatureInfo = new SettingsDictionary(ctx.StackSignatureData),
SignatureHash = signatureHash,
DuplicateSignature = ctx.Event.ProjectId + ":" + signatureHash,
Title = title?.Truncate(1000),
Tags = ctx.Event.Tags ?? new TagSet(),
Type = ctx.Event.Type,
TotalOccurrences = 1,
FirstOccurrence = ctx.Event.Date.UtcDateTime,
LastOccurrence = ctx.Event.Date.UtcDateTime
};
if (ctx.Event.Type == Event.KnownTypes.Session)
{
stack.Status = StackStatus.Ignored;
}
ctx.Stack = stack;
stacks.Add(signatureHash, Tuple.Create(true, ctx.Stack));
}
}
else
{
ctx.Stack = await _stackRepository.GetByIdAsync(ctx.Event.StackId, o => o.Cache()).AnyContext();
if (ctx.Stack == null || ctx.Stack.ProjectId != ctx.Event.ProjectId)
{
ctx.SetError("Invalid StackId.");
continue;
}
ctx.SignatureHash = ctx.Stack.SignatureHash;
if (!stacks.ContainsKey(ctx.Stack.SignatureHash))
{
stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(false, ctx.Stack));
}
else
{
stacks[ctx.Stack.SignatureHash] = Tuple.Create(false, ctx.Stack);
}
if (ctx.Stack.Status == StackStatus.Discarded)
{
ctx.IsDiscarded = true;
ctx.IsCancelled = true;
}
}
if (!ctx.IsNew && ctx.Event.Tags != null && ctx.Event.Tags.Count > 0)
{
if (ctx.Stack.Tags == null)
{
ctx.Stack.Tags = new TagSet();
}
var newTags = ctx.Event.Tags.Where(t => !ctx.Stack.Tags.Contains(t)).ToList();
if (newTags.Count > 0 || ctx.Stack.Tags.Count > 50 || ctx.Stack.Tags.Any(t => t.Length > 100))
{
ctx.Stack.Tags.AddRange(newTags);
ctx.Stack.Tags.RemoveExcessTags();
// make sure the stack gets saved
if (!stacks.ContainsKey(ctx.Stack.SignatureHash))
{
stacks.Add(ctx.Stack.SignatureHash, Tuple.Create(true, ctx.Stack));
}
else
{
stacks[ctx.Stack.SignatureHash] = Tuple.Create(true, stacks[ctx.Stack.SignatureHash].Item2);
}
}
}
ctx.Event.IsFirstOccurrence = ctx.IsNew;
}
var stacksToAdd = stacks.Where(kvp => kvp.Value.Item1 && String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList();
if (stacksToAdd.Count > 0)
{
await _stackRepository.AddAsync(stacksToAdd, o => o.Cache().Notifications(stacksToAdd.Count == 1)).AnyContext();
if (stacksToAdd.Count > 1)
{
await _publisher.PublishAsync(new EntityChanged {
ChangeType = ChangeType.Added,
Type = StackTypeName,
Data =
{
{ ExtendedEntityChanged.KnownKeys.OrganizationId, contexts.First().Organization.Id },
{ ExtendedEntityChanged.KnownKeys.ProjectId, contexts.First().Project.Id }
}
}).AnyContext();
}
}
var stacksToSave = stacks.Where(kvp => kvp.Value.Item1 && !String.IsNullOrEmpty(kvp.Value.Item2.Id)).Select(kvp => kvp.Value.Item2).ToList();
if (stacksToSave.Count > 0)
{
await _stackRepository.SaveAsync(stacksToSave, o => o.Cache().Notifications(false)).AnyContext(); // notification will get sent later in the update stats step
}
// Set stack ids after they have been saved and created
contexts.ForEach(ctx => {
ctx.Event.StackId = ctx.Stack?.Id;
});
}
