public async Task VerifyStackFilter(string filter, int expected, int?expectedInverted = null)
{
Log.SetLogLevel <StackRepository>(LogLevel.Trace);
long totalStacks = await _stackRepository.CountAsync(o => o.IncludeSoftDeletes());
var ctx = new ElasticQueryVisitorContext();
var stackFilter = await new EventStackFilter().GetStackFilterAsync(filter, ctx);
_logger.LogInformation("Finding Filter: {Filter}", stackFilter.Filter);
var stacks = await _stackRepository.FindAsync(q => q.FilterExpression(stackFilter.Filter), o => o.SoftDeleteMode(SoftDeleteQueryMode.All).PageLimit(1000));
Assert.Equal(expected, stacks.Total);
_logger.LogInformation("Finding Inverted Filter: {Filter}", stackFilter.InvertedFilter);
var invertedStacks = await _stackRepository.FindAsync(q => q.FilterExpression(stackFilter.InvertedFilter), o => o.SoftDeleteMode(SoftDeleteQueryMode.All).PageLimit(1000));
long expectedInvert = expectedInverted ?? totalStacks - expected;
Assert.Equal(expectedInvert, invertedStacks.Total);
var stackIds = new HashSet <string>(stacks.Hits.Select(h => h.Id));
var invertedStackIds = new HashSet <string>(invertedStacks.Hits.Select(h => h.Id));
Assert.Empty(stackIds.Intersect(invertedStackIds));
}
public async Task CanGetByFixedAsync()
{
var stack = await _repository.AddAsync(StackData.GenerateStack(projectId: TestConstants.ProjectId, organizationId: TestConstants.OrganizationId), o => o.ImmediateConsistency());
var results = await _repository.FindAsync(q => q.FilterExpression("fixed:true"));
Assert.NotNull(results);
Assert.Equal(0, results.Total);
results = await _repository.FindAsync(q => q.FilterExpression("fixed:false"));
Assert.NotNull(results);
Assert.Equal(1, results.Total);
Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed);
Assert.Null(results.Documents.Single().DateFixed);
stack.MarkFixed();
await _repository.SaveAsync(stack, o => o.ImmediateConsistency());
results = await _repository.FindAsync(q => q.FilterExpression("fixed:true"));
Assert.NotNull(results);
Assert.Equal(1, results.Total);
Assert.False(results.Documents.Single().Status == Core.Models.StackStatus.Regressed);
Assert.NotNull(results.Documents.Single().DateFixed);
results = await _repository.FindAsync(q => q.FilterExpression("fixed:false"));
Assert.NotNull(results);
Assert.Equal(0, results.Total);
}
public async Task WillSetStackDuplicateSignature()
{
var stack = StackData.GenerateStack();
stack.DuplicateSignature = null;
stack = await _repository.AddAsync(stack, o => o.ImmediateConsistency());
Assert.NotEmpty(stack.ProjectId);
Assert.NotEmpty(stack.SignatureHash);
Assert.Null(stack.DuplicateSignature);
var migration = GetService <SetStackDuplicateSignature>();
var context = new MigrationContext(GetService <ILock>(), _logger, CancellationToken.None);
await migration.RunAsync(context);
string expectedDuplicateSignature = $"{stack.ProjectId}:{stack.SignatureHash}";
var actualStack = await _repository.GetByIdAsync(stack.Id);
Assert.NotEmpty(actualStack.ProjectId);
Assert.NotEmpty(actualStack.SignatureHash);
Assert.Equal($"{actualStack.ProjectId}:{actualStack.SignatureHash}", actualStack.DuplicateSignature);
var results = await _repository.FindAsync(q => q.ElasticFilter(Query <Stack> .Term(s => s.DuplicateSignature, expectedDuplicateSignature)));
Assert.Single(results.Documents);
}
public async Task WillMergeDuplicatedStacks()
{
var utcNow = SystemClock.UtcNow;
var originalStack = StackData.GenerateStack();
originalStack.Id = ObjectId.GenerateNewId().ToString();
originalStack.TotalOccurrences = 100;
var duplicateStack = originalStack.DeepClone();
duplicateStack.Id = ObjectId.GenerateNewId().ToString();
duplicateStack.Status = StackStatus.Fixed;
duplicateStack.TotalOccurrences = 10;
duplicateStack.LastOccurrence = originalStack.LastOccurrence.AddMinutes(1);
duplicateStack.SnoozeUntilUtc = originalStack.SnoozeUntilUtc = null;
duplicateStack.DateFixed = duplicateStack.LastOccurrence.AddMinutes(1);
duplicateStack.Tags.Add("stack2");
duplicateStack.References.Add("stack2");
duplicateStack.OccurrencesAreCritical = true;
originalStack = await _stackRepository.AddAsync(originalStack, o => o.ImmediateConsistency());
duplicateStack = await _stackRepository.AddAsync(duplicateStack, o => o.ImmediateConsistency());
await _eventRepository.AddAsync(EventData.GenerateEvents(count: 100, stackId: originalStack.Id), o => o.ImmediateConsistency());
await _eventRepository.AddAsync(EventData.GenerateEvents(count: 10, stackId: duplicateStack.Id), o => o.ImmediateConsistency());
var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query <Stack> .Term(s => s.DuplicateSignature, originalStack.DuplicateSignature)));
Assert.Equal(2, results.Total);
var migration = GetService <FixDuplicateStacks>();
var context = new MigrationContext(GetService <ILock>(), _logger, CancellationToken.None);
await migration.RunAsync(context);
await RefreshDataAsync();
results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query <Stack> .Term(s => s.DuplicateSignature, originalStack.DuplicateSignature)));
Assert.Single(results.Documents);
var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes());
Assert.False(updatedOriginalStack.IsDeleted);
var updatedDuplicateStack = await _stackRepository.GetByIdAsync(duplicateStack.Id, o => o.IncludeSoftDeletes());
Assert.True(updatedDuplicateStack.IsDeleted);
Assert.Equal(originalStack.CreatedUtc, updatedOriginalStack.CreatedUtc);
Assert.Equal(110, updatedOriginalStack.TotalOccurrences);
Assert.Equal(StackStatus.Fixed, updatedOriginalStack.Status);
Assert.Equal(duplicateStack.LastOccurrence, updatedOriginalStack.LastOccurrence);
Assert.Null(updatedOriginalStack.SnoozeUntilUtc);
Assert.Equal(duplicateStack.DateFixed, updatedOriginalStack.DateFixed);
Assert.Equal(originalStack.Tags.Count + 1, updatedOriginalStack.Tags.Count);
Assert.Contains("stack2", updatedOriginalStack.Tags);
Assert.Equal(originalStack.References.Count + 1, updatedOriginalStack.References.Count);
Assert.Contains("stack2", updatedOriginalStack.References);
Assert.True(updatedOriginalStack.OccurrencesAreCritical);
}
public override async Task RunAsync(MigrationContext context)
{
_logger.LogInformation("Getting duplicate stacks");
var duplicateStackAgg = await _client.SearchAsync <Stack>(q => q
.QueryOnQueryString("is_deleted:false")
.Size(0)
.Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000))));
_logger.LogRequest(duplicateStackAgg, LogLevel.Trace);
var buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets;
int total = buckets.Count;
int processed = 0;
int error = 0;
long totalUpdatedEventCount = 0;
var lastStatus = SystemClock.Now;
int batch = 1;
while (buckets.Count > 0)
{
_logger.LogInformation($"Found {total} duplicate stacks in batch #{batch}.");
foreach (var duplicateSignature in buckets)
{
string projectId = null;
string signature = null;
try {
var parts = duplicateSignature.Key.Split(':');
if (parts.Length != 2)
{
_logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key);
continue;
}
projectId = parts[0];
signature = parts[1];
var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FilterExpression($"signature_hash:{signature}"));
if (stacks.Documents.Count < 2)
{
_logger.LogError("Did not find multiple stacks with signature {SignatureHash} and project {ProjectId}", signature, projectId);
continue;
}
var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id"));
var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List <Foundatio.Repositories.Models.KeyedBucket <string> >();
// we only need to update events if more than one stack has events associated to it
bool shouldUpdateEvents = eventCountBuckets.Count > 1;
// default to using the oldest stack
var targetStack = stacks.Documents.OrderBy(s => s.CreatedUtc).First();
var duplicateStacks = stacks.Documents.OrderBy(s => s.CreatedUtc).Skip(1).ToList();
// use the stack that has the most events on it so we can reduce the number of updates
if (eventCountBuckets.Count > 0)
{
var targetStackId = eventCountBuckets.OrderByDescending(b => b.Total).First().Key;
targetStack = stacks.Documents.Single(d => d.Id == targetStackId);
duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).ToList();
}
targetStack.CreatedUtc = stacks.Documents.Min(d => d.CreatedUtc);
targetStack.Status = stacks.Documents.FirstOrDefault(d => d.Status != StackStatus.Open)?.Status ?? StackStatus.Open;
targetStack.LastOccurrence = stacks.Documents.Max(d => d.LastOccurrence);
targetStack.SnoozeUntilUtc = stacks.Documents.Max(d => d.SnoozeUntilUtc);
targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed);;
targetStack.TotalOccurrences += duplicateStacks.Sum(d => d.TotalOccurrences);
targetStack.Tags.AddRange(duplicateStacks.SelectMany(d => d.Tags));
targetStack.References = stacks.Documents.SelectMany(d => d.References).Distinct().ToList();
targetStack.OccurrencesAreCritical = stacks.Documents.Any(d => d.OccurrencesAreCritical);
duplicateStacks.ForEach(s => s.IsDeleted = true);
await _stackRepository.SaveAsync(duplicateStacks);
await _stackRepository.SaveAsync(targetStack);
processed++;
long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0;
_logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents);
if (shouldUpdateEvents)
{
var response = await _client.UpdateByQueryAsync <PersistentEvent>(u => u
.Query(q => q.Bool(b => b.Must(m => m
.Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id)))
)))
.Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless))
.Conflicts(Elasticsearch.Net.Conflicts.Proceed)
.WaitForCompletion(false));
_logger.LogRequest(response, LogLevel.Trace);
var taskStartedTime = SystemClock.Now;
var taskId = response.Task;
int attempts = 0;
long affectedRecords = 0;
do
{
attempts++;
var taskStatus = await _client.Tasks.GetTaskAsync(taskId);
var status = taskStatus.Task.Status;
if (taskStatus.Completed)
{
// TODO: need to check to see if the task failed or completed successfully. Throw if it failed.
if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30))
{
_logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total);
}
affectedRecords += status.Created + status.Updated + status.Deleted;
break;
}
if (SystemClock.Now.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30))
{
_logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total);
}
var delay = TimeSpan.FromMilliseconds(50);
if (attempts > 20)
{
delay = TimeSpan.FromSeconds(5);
}
else if (attempts > 10)
{
delay = TimeSpan.FromSeconds(1);
}
else if (attempts > 5)
{
delay = TimeSpan.FromMilliseconds(250);
}
await Task.Delay(delay);
} while (true);
_logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id));
totalUpdatedEventCount += affectedRecords;
}
if (SystemClock.UtcNow.Subtract(lastStatus) > TimeSpan.FromSeconds(5))
{
lastStatus = SystemClock.UtcNow;
_logger.LogInformation("Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error);
await _cache.RemoveByPrefixAsync(nameof(Stack));
}
}
catch (Exception ex) {
error++;
_logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}", projectId, signature);
}
}
await _client.Indices.RefreshAsync(_config.Stacks.VersionedName);
duplicateStackAgg = await _client.SearchAsync <Stack>(q => q
.QueryOnQueryString("is_deleted:false")
.Size(0)
.Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000))));
_logger.LogRequest(duplicateStackAgg, LogLevel.Trace);
buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets;
total += buckets.Count;
batch++;
_logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error);
await _cache.RemoveByPrefixAsync(nameof(Stack));
}
}
private async Task <bool> SendSummaryNotificationAsync(Project project, SummaryNotification data)
{
// TODO: Add slack daily summaries
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary && !String.Equals(n.Key, Project.NotificationIntegrations.Slack)).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
_logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name);
return(false);
}
var results = await _userRepository.GetByIdsAsync(userIds, o => o.Cache()).AnyContext();
var users = results.Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(project.OrganizationId)).ToList();
if (users.Count == 0)
{
_logger.LogInformation("Project {ProjectName} has no users to send summary to.", project.Name);
return(false);
}
// TODO: What should we do about suspended organizations.
var organization = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()).AnyContext();
if (organization == null)
{
_logger.LogInformation("The organization {organization} for project {ProjectName} may have been deleted. No summaries will be sent.", project.OrganizationId, project.Name);
return(false);
}
_logger.LogInformation("Sending daily summary: users={UserCount} project={project}", users.Count, project.Id);
var sf = new AppFilter(project, organization);
var systemFilter = new RepositoryQuery <PersistentEvent>().AppFilter(sf).DateRange(data.UtcStartTime, data.UtcEndTime, (PersistentEvent e) => e.Date).Index(data.UtcStartTime, data.UtcEndTime);
string filter = "type:error (status:open OR status:regressed)";
var result = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression("terms:(first @include:true) terms:(stack_id~3) cardinality:stack_id sum:count~1")).AnyContext();
double total = result.Aggregations.Sum("sum_count")?.Value ?? result.Total;
double newTotal = result.Aggregations.Terms <double>("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0;
double uniqueTotal = result.Aggregations.Cardinality("cardinality_stack_id")?.Value ?? 0;
bool hasSubmittedEvents = total > 0 || project.IsConfigured.GetValueOrDefault();
bool isFreePlan = organization.PlanId == _plans.FreePlan.Id;
string fixedFilter = "type:error status:fixed";
var fixedResult = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(fixedFilter).EnforceEventStackFilter().AggregationsExpression("sum:count~1")).AnyContext();
double fixedTotal = fixedResult.Aggregations.Sum("sum_count")?.Value ?? fixedResult.Total;
var range = new DateTimeRange(data.UtcStartTime, data.UtcEndTime);
var usages = project.OverageHours.Where(u => range.Contains(u.Date)).ToList();
int blockedTotal = usages.Sum(u => u.Blocked);
int tooBigTotal = usages.Sum(u => u.TooBig);
IReadOnlyCollection <Stack> mostFrequent = null;
var stackTerms = result.Aggregations.Terms <string>("terms_stack_id");
if (stackTerms?.Buckets.Count > 0)
{
mostFrequent = await _stackRepository.GetByIdsAsync(stackTerms.Buckets.Select(b => b.Key).ToArray()).AnyContext();
}
IReadOnlyCollection <Stack> newest = null;
if (newTotal > 0)
{
newest = (await _stackRepository.FindAsync(q => q.AppFilter(sf).FilterExpression(filter).SortExpression("-first").DateRange(data.UtcStartTime, data.UtcEndTime, "first"), o => o.PageLimit(3)).AnyContext()).Documents;
}
foreach (var user in users)
{
_logger.LogInformation("Queuing {ProjectName} daily summary email ({UtcStartTime}-{UtcEndTime}) for user {EmailAddress}.", project.Name, data.UtcStartTime, data.UtcEndTime, user.EmailAddress);
await _mailer.SendProjectDailySummaryAsync(user, project, mostFrequent, newest, data.UtcStartTime, hasSubmittedEvents, total, uniqueTotal, newTotal, fixedTotal, blockedTotal, tooBigTotal, isFreePlan).AnyContext();
}
_logger.LogInformation("Done sending daily summary: users={UserCount} project={ProjectName} events={EventCount}", users.Count, project.Name, total);
return(true);
}
private Task <FindResults <Stack> > GetByFilterAsync(string filter)
{
return(_repository.FindAsync(q => q.FilterExpression(filter)));
}
