public async Task VerifyStackFilter(string filter, int expected, int?expectedInverted = null) { Log.SetLogLevel <StackRepository>(LogLevel.Trace); long totalStacks = await _stackRepository.CountAsync(o => o.IncludeSoftDeletes()); var ctx = new ElasticQueryVisitorContext(); var stackFilter = await new EventStackFilter().GetStackFilterAsync(filter, ctx); _logger.LogInformation("Finding Filter: {Filter}", stackFilter.Filter); var stacks = await _stackRepository.FindAsync(q => q.FilterExpression(stackFilter.Filter), o => o.SoftDeleteMode(SoftDeleteQueryMode.All).PageLimit(1000)); Assert.Equal(expected, stacks.Total); _logger.LogInformation("Finding Inverted Filter: {Filter}", stackFilter.InvertedFilter); var invertedStacks = await _stackRepository.FindAsync(q => q.FilterExpression(stackFilter.InvertedFilter), o => o.SoftDeleteMode(SoftDeleteQueryMode.All).PageLimit(1000)); long expectedInvert = expectedInverted ?? totalStacks - expected; Assert.Equal(expectedInvert, invertedStacks.Total); var stackIds = new HashSet <string>(stacks.Hits.Select(h => h.Id)); var invertedStackIds = new HashSet <string>(invertedStacks.Hits.Select(h => h.Id)); Assert.Empty(stackIds.Intersect(invertedStackIds)); }
public async Task CanGetByFixedAsync() { var stack = await _repository.AddAsync(StackData.GenerateStack(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId), o => o.ImmediateConsistency()); var results = await _repository.FindAsync(q => q.FilterExpression("fixed:true")); Assert.NotNull(results); Assert.Equal(0, results.Total); results = await _repository.FindAsync(q => q.FilterExpression("fixed:false")); Assert.NotNull(results); Assert.Equal(1, results.Total); Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed); Assert.Null(results.Documents.Single().DateFixed); stack.MarkFixed(); await _repository.SaveAsync(stack, o => o.ImmediateConsistency()); results = await _repository.FindAsync(q => q.FilterExpression("fixed:true")); Assert.NotNull(results); Assert.Equal(1, results.Total); Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed); Assert.NotNull(results.Documents.Single().DateFixed); results = await _repository.FindAsync(q => q.FilterExpression("fixed:false")); Assert.NotNull(results); Assert.Equal(0, results.Total); }
public async Task WillSetStackDuplicateSignature() { var stack = StackData.GenerateStack(); stack.DuplicateSignature = null; stack = await _repository.AddAsync(stack, o => o.ImmediateConsistency()); Assert.NotEmpty(stack.ProjectId); Assert.NotEmpty(stack.SignatureHash); Assert.Null(stack.DuplicateSignature); var migration = GetService <SetStackDuplicateSignature>(); var context = new MigrationContext(GetService <ILock>(), _logger, CancellationToken.None); await migration.RunAsync(context); string expectedDuplicateSignature = $"{stack.ProjectId}:{stack.SignatureHash}"; var actualStack = await _repository.GetByIdAsync(stack.Id); Assert.NotEmpty(actualStack.ProjectId); Assert.NotEmpty(actualStack.SignatureHash); Assert.Equal($"{actualStack.ProjectId}:{actualStack.SignatureHash}", actualStack.DuplicateSignature); var results = await _repository.FindAsync(q => q.ElasticFilter(Query <Stack> .Term(s => s.DuplicateSignature, expectedDuplicateSignature))); Assert.Single(results.Documents); }
public async Task WillMergeDuplicatedStacks() { var utcNow = SystemClock.UtcNow; var originalStack = StackData.GenerateStack(); originalStack.Id = ObjectId.GenerateNewId().ToString(); originalStack.TotalOccurrences = 100; var duplicateStack = originalStack.DeepClone(); duplicateStack.Id = ObjectId.GenerateNewId().ToString(); duplicateStack.Status = StackStatus.Fixed; duplicateStack.TotalOccurrences = 10; duplicateStack.LastOccurrence = originalStack.LastOccurrence.AddMinutes(1); duplicateStack.SnoozeUntilUtc = originalStack.SnoozeUntilUtc = null; duplicateStack.DateFixed = duplicateStack.LastOccurrence.AddMinutes(1); duplicateStack.Tags.Add("stack2"); duplicateStack.References.Add("stack2"); duplicateStack.OccurrencesAreCritical = true; originalStack = await _stackRepository.AddAsync(originalStack, o => o.ImmediateConsistency()); duplicateStack = await _stackRepository.AddAsync(duplicateStack, o => o.ImmediateConsistency()); await _eventRepository.AddAsync(EventData.GenerateEvents(count: 100, stackId: originalStack.Id), o => o.ImmediateConsistency()); await _eventRepository.AddAsync(EventData.GenerateEvents(count: 10, stackId: duplicateStack.Id), o => o.ImmediateConsistency()); var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query <Stack> .Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); Assert.Equal(2, results.Total); var migration = GetService <FixDuplicateStacks>(); var context = new MigrationContext(GetService <ILock>(), _logger, CancellationToken.None); await migration.RunAsync(context); await RefreshDataAsync(); results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query <Stack> .Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); Assert.Single(results.Documents); var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); Assert.False(updatedOriginalStack.IsDeleted); var updatedDuplicateStack = await _stackRepository.GetByIdAsync(duplicateStack.Id, o => o.IncludeSoftDeletes()); Assert.True(updatedDuplicateStack.IsDeleted); Assert.Equal(originalStack.CreatedUtc, updatedOriginalStack.CreatedUtc); Assert.Equal(110, updatedOriginalStack.TotalOccurrences); Assert.Equal(StackStatus.Fixed, updatedOriginalStack.Status); Assert.Equal(duplicateStack.LastOccurrence, updatedOriginalStack.LastOccurrence); Assert.Null(updatedOriginalStack.SnoozeUntilUtc); Assert.Equal(duplicateStack.DateFixed, updatedOriginalStack.DateFixed); Assert.Equal(originalStack.Tags.Count + 1, updatedOriginalStack.Tags.Count); Assert.Contains("stack2", updatedOriginalStack.Tags); Assert.Equal(originalStack.References.Count + 1, updatedOriginalStack.References.Count); Assert.Contains("stack2", updatedOriginalStack.References); Assert.True(updatedOriginalStack.OccurrencesAreCritical); }
public override async Task RunAsync(MigrationContext context) { _logger.LogInformation("Getting duplicate stacks"); var duplicateStackAgg = await _client.SearchAsync <Stack>(q => q .QueryOnQueryString("is_deleted:false") .Size(0) .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); var buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; int total = buckets.Count; int processed = 0; int error = 0; long totalUpdatedEventCount = 0; var lastStatus = SystemClock.Now; int batch = 1; while (buckets.Count > 0) { _logger.LogInformation($"Found {total} duplicate stacks in batch #{batch}."); foreach (var duplicateSignature in buckets) { string projectId = null; string signature = null; try { var parts = duplicateSignature.Key.Split(':'); if (parts.Length != 2) { _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key); continue; } projectId = parts[0]; signature = parts[1]; var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FilterExpression($"signature_hash:{signature}")); if (stacks.Documents.Count < 2) { _logger.LogError("Did not find multiple stacks with signature {SignatureHash} and project {ProjectId}", signature, projectId); continue; } var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id")); var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List <Foundatio.Repositories.Models.KeyedBucket <string> >(); // we only need to update events if more than one stack has events associated to it bool shouldUpdateEvents = eventCountBuckets.Count > 1; // default to using the oldest stack var targetStack = stacks.Documents.OrderBy(s => s.CreatedUtc).First(); var duplicateStacks = stacks.Documents.OrderBy(s => s.CreatedUtc).Skip(1).ToList(); // use the stack that has the most events on it so we can reduce the number of updates if (eventCountBuckets.Count > 0) { var targetStackId = eventCountBuckets.OrderByDescending(b => b.Total).First().Key; targetStack = stacks.Documents.Single(d => d.Id == targetStackId); duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).ToList(); } targetStack.CreatedUtc = stacks.Documents.Min(d => d.CreatedUtc); targetStack.Status = stacks.Documents.FirstOrDefault(d => d.Status != StackStatus.Open)?.Status ?? StackStatus.Open; targetStack.LastOccurrence = stacks.Documents.Max(d => d.LastOccurrence); targetStack.SnoozeUntilUtc = stacks.Documents.Max(d => d.SnoozeUntilUtc); targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed);; targetStack.TotalOccurrences += duplicateStacks.Sum(d => d.TotalOccurrences); targetStack.Tags.AddRange(duplicateStacks.SelectMany(d => d.Tags)); targetStack.References = stacks.Documents.SelectMany(d => d.References).Distinct().ToList(); targetStack.OccurrencesAreCritical = stacks.Documents.Any(d => d.OccurrencesAreCritical); duplicateStacks.ForEach(s => s.IsDeleted = true); await _stackRepository.SaveAsync(duplicateStacks); await _stackRepository.SaveAsync(targetStack); processed++; long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0; _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents); if (shouldUpdateEvents) { var response = await _client.UpdateByQueryAsync <PersistentEvent>(u => u .Query(q => q.Bool(b => b.Must(m => m .Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id))) ))) .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless)) .Conflicts(Elasticsearch.Net.Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(response, LogLevel.Trace); var taskStartedTime = SystemClock.Now; var taskId = response.Task; int attempts = 0; long affectedRecords = 0; do { attempts++; var taskStatus = await _client.Tasks.GetTaskAsync(taskId); var status = taskStatus.Task.Status; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) { _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); } affectedRecords += status.Created + status.Updated + status.Deleted; break; } if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) { _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); } var delay = TimeSpan.FromMilliseconds(50); if (attempts > 20) { delay = TimeSpan.FromSeconds(5); } else if (attempts > 10) { delay = TimeSpan.FromSeconds(1); } else if (attempts > 5) { delay = TimeSpan.FromMilliseconds(250); } await Task.Delay(delay); } while (true); _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); totalUpdatedEventCount += affectedRecords; } if (SystemClock.UtcNow.Subtract(lastStatus) > TimeSpan.FromSeconds(5)) { lastStatus = SystemClock.UtcNow; _logger.LogInformation("Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); await _cache.RemoveByPrefixAsync(nameof(Stack)); } } catch (Exception ex) { error++; _logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}", projectId, signature); } } await _client.Indices.RefreshAsync(_config.Stacks.VersionedName); duplicateStackAgg = await _client.SearchAsync <Stack>(q => q .QueryOnQueryString("is_deleted:false") .Size(0) .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; total += buckets.Count; batch++; _logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); await _cache.RemoveByPrefixAsync(nameof(Stack)); } }
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data) { // TODO: Add slack daily summaries var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList(); if (userIds.Count == 0) { _logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name); return(false); } var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext(); var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList(); if (users.Count == 0) { _logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name); return(false); } // TODO: What should we do about suspended organizations. var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { _logger.LogInformation("The organization {organization} for project {ProjectName} may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name); return(false); } _logger.LogInformation("Sending daily summary: users={UserCount} project={project}", users.Count, project.Id); var sf = new AppFilter(project, organization); var systemFilter = new RepositoryQuery <PersistentEvent>().AppFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime); string filter = "type:error (status:open OR status:regressed)"; var result = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression("terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1")).AnyContext(); double total = result.Aggregations.Sum("sum_count")?.Value ?? result.Total; double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0; double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0; bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault(); bool isFreePlan = organization.PlanId == _plans.FreePlan.Id; string fixedFilter = "type:error status:fixed"; var fixedResult = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(fixedFilter).EnforceEventStackFilter().AggregationsExpression("sum:count~1")).AnyContext(); double fixedTotal = fixedResult.Aggregations.Sum("sum_count")?.Value ?? fixedResult.Total; var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime); var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList(); int blockedTotal = usages.Sum(u => u.Blocked); int tooBigTotal = usages.Sum(u => u.TooBig); IReadOnlyCollection <Stack> mostFrequent = null; var stackTerms = result.Aggregations.Terms <string>("terms_stack_id"); if (stackTerms?.Buckets.Count > 0) { mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext(); } IReadOnlyCollection <Stack> newest = null; if (newTotal > 0) { newest = (await _stackRepository.FindAsync(q => q.AppFilter(sf).FilterExpression(filter).SortExpression("-first").DateRange(data.UtcStartTime, data.UtcEndTime, "first"), o => o.PageLimit(3)).AnyContext()).Documents; } foreach (var user in users) { _logger.LogInformation("Queuing {ProjectName} daily summary email ({UtcStartTime}-{UtcEndTime}) for user {EmailAddress}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress); await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext(); } _logger.LogInformation("Done sending daily summary: users={UserCount} project={ProjectName} events={EventCount}", users.Count, project.Name, total); return(true); }
private Task <FindResults <Stack> > GetByFilterAsync(string filter) { return(_repository.FindAsync(q => q.FilterExpression(filter))); }