public async Task CanGetEventTermStatsByTagAsync() { // capture start date before generating data to make sure that our time range for stats includes all items var startDate = DateTime.UtcNow.SubtractDays(60); const int eventCount = 100; await RemoveDataAsync(); await CreateDataAsync(eventCount, false); var fields = FieldAggregationProcessor.Process("term:is_first_occurrence:-F", false); Assert.True(fields.IsValid); var result = await _stats.GetNumbersTermsStatsAsync("tags", fields.Aggregations, startDate, DateTime.UtcNow, null, userFilter : $"project:{TestConstants.ProjectId}"); Assert.Equal(eventCount, result.Total); // each event can be in multiple tag buckets since an event can have up to 3 sample tags Assert.InRange(result.Terms.Sum(t => t.Total), eventCount, eventCount * 3); Assert.InRange(result.Terms.Sum(t => t.Numbers[0]), 1, 25 * TestConstants.EventTags.Count); // new Assert.InRange(result.Terms.Count, 1, TestConstants.EventTags.Count); foreach (var term in result.Terms) { Assert.InRange(term.Numbers[0], 1, 25); // new } }
public async Task CanGetEventTermStatsByTagAsync() { // capture start date before generating data to make sure that our time range for stats includes all items var startDate = SystemClock.UtcNow.SubtractDays(60); const int eventCount = 100; await CreateDataAsync(eventCount, false); Log.MinimumLevel = LogLevel.Trace; var fields = FieldAggregationProcessor.Process("term:is_first_occurrence:-F", false); Assert.True(fields.IsValid); var sf = new ExceptionlessSystemFilterQuery(ProjectData.GenerateSampleProject(), OrganizationData.GenerateSampleOrganization()); var result = await _stats.GetNumbersTermsStatsAsync("tags", fields.Aggregations, startDate, SystemClock.UtcNow, sf, "fixed:false"); Assert.Equal(eventCount, result.Total); // each event can be in multiple tag buckets since an event can have up to 3 sample tags Assert.InRange(result.Terms.Sum(t => t.Total), eventCount, eventCount * 3); Assert.InRange(result.Terms.Sum(t => t.Numbers[0]), 1, 25 * TestConstants.EventTags.Count); // new Assert.InRange(result.Terms.Count, 1, TestConstants.EventTags.Count); foreach (var term in result.Terms) { Assert.InRange(term.Numbers[0], 1, 25); // new } }
private async Task <IHttpActionResult> GetAllByTermsAsync(ICollection <FieldAggregation> fields, string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10) { page = GetPage(page); limit = GetLimit(limit); var skip = GetSkip(page, limit); if (skip > MAXIMUM_SKIP) { return(Ok(new object[0])); } var pr = QueryProcessor.Process(userFilter); if (!pr.IsValid) { return(BadRequest(pr.Message)); } var organizations = await GetAssociatedOrganizationsAsync(_organizationRepository); var ti = GetTimeInfo(time, offset, organizations.GetRetentionUtcCutoff()); systemFilter = String.Join(" ", new [] { systemFilter, BuildSystemFilter(organizations, userFilter, pr.UsesPremiumFeatures) }.Where(f => !String.IsNullOrWhiteSpace(f))); try { var ntsr = await _eventStats.GetNumbersTermsStatsAsync("stack_id", fields, ti.UtcRange.Start, ti.UtcRange.End, systemFilter, userFilter, ti.Offset, GetSkip(page + 1, limit) + 1); if (ntsr.Terms.Count == 0) { return(Ok(new object[0])); } var stackIds = ntsr.Terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray(); var stacks = (await _stackRepository.GetByIdsAsync(stackIds)).Documents.Select(s => s.ApplyOffset(ti.Offset)).ToList(); if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) { var summaries = await GetStackSummariesAsync(stacks, ntsr, systemFilter, ti.UtcRange.Start, ti.UtcRange.End); return(OkWithResourceLinks(summaries.Take(limit).ToList(), summaries.Count > limit, page)); } return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page)); } catch (ApplicationException ex) { _logger.Error().Exception(ex) .Message("An error has occurred. Please check your search filter.") .Property("Search Filter", new { SystemFilter = systemFilter, UserFilter = userFilter, Time = time, Offset = offset, Page = page, Limit = limit }) .Tag("Search") .Identity(ExceptionlessUser.EmailAddress) .Property("User", ExceptionlessUser) .SetActionContext(ActionContext) .Write(); return(BadRequest("An error has occurred. Please check your search filter.")); } }
private async Task <IHttpActionResult> FrequentInternalAsync(string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10) { page = GetPage(page); limit = GetLimit(limit); var skip = GetSkip(page, limit); if (skip > MAXIMUM_SKIP) { return(Ok(new object[0])); } var validationResult = QueryProcessor.Process(userFilter); if (!validationResult.IsValid) { return(BadRequest(validationResult.Message)); } if (String.IsNullOrEmpty(systemFilter)) { systemFilter = await GetAssociatedOrganizationsFilterAsync(_organizationRepository, validationResult.UsesPremiumFeatures, HasOrganizationOrProjectFilter(userFilter)); } var timeInfo = GetTimeInfo(time, offset); try { var terms = (await _eventStats.GetNumbersTermsStatsAsync("stack_id", new List <FieldAggregation>(), timeInfo.UtcRange.Start, timeInfo.UtcRange.End, systemFilter, userFilter, timeInfo.Offset, GetSkip(page + 1, limit) + 1)).Terms; if (terms.Count == 0) { return(Ok(new object[0])); } var stackIds = terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray(); var stacks = (await _stackRepository.GetByIdsAsync(stackIds)).Documents.Select(s => s.ApplyOffset(timeInfo.Offset)).ToList(); if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) { var summaries = GetStackSummaries(stacks, terms); return(OkWithResourceLinks(GetStackSummaries(stacks, terms).Take(limit).ToList(), summaries.Count > limit, page)); } return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page)); } catch (ApplicationException ex) { _logger.Error().Exception(ex) .Property("Search Filter", new { SystemFilter = systemFilter, UserFilter = userFilter, Time = time, Offset = offset, Page = page, Limit = limit }) .Tag("Search") .Identity(ExceptionlessUser.EmailAddress) .Property("User", ExceptionlessUser) .SetActionContext(ActionContext) .Write(); return(BadRequest("An error has occurred. Please check your search filter.")); } }
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> viewOrganizations) { if (viewOrganizations.Count <= 0) { return(viewOrganizations); } var fields = new List <FieldAggregation> { new FieldAggregation { Type = FieldAggregationType.Distinct, Field = "stack_id" } }; var organizations = viewOrganizations.Select(o => new Organization { Id = o.Id, RetentionDays = o.RetentionDays }).ToList(); var sf = new ExceptionlessSystemFilterQuery(organizations); var result = await _stats.GetNumbersTermsStatsAsync("organization_id", fields, organizations.GetRetentionUtcCutoff(), DateTime.MaxValue, sf, max : viewOrganizations.Count); foreach (var organization in viewOrganizations) { var organizationStats = result.Terms.FirstOrDefault(t => t.Term == organization.Id); organization.EventCount = organizationStats?.Total ?? 0; organization.StackCount = (long)(organizationStats?.Numbers[0] ?? 0); organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id); } return(viewOrganizations); }
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> viewProjects) { if (viewProjects.Count <= 0) { return(viewProjects); } var fields = new List <FieldAggregation> { new FieldAggregation { Type = FieldAggregationType.Distinct, Field = "stack_id" } }; var organizations = (await _organizationRepository.GetByIdsAsync(viewProjects.Select(p => p.OrganizationId).ToArray(), true)).Documents; var filter = viewProjects.Select(p => new Project { Id = p.Id, OrganizationId = p.OrganizationId }).ToList().BuildRetentionFilter(organizations); var ntsr = await _stats.GetNumbersTermsStatsAsync("project_id", fields, organizations.GetRetentionUtcCutoff(), DateTime.MaxValue, filter, max : viewProjects.Count); foreach (var project in viewProjects) { var term = ntsr.Terms.FirstOrDefault(t => t.Term == project.Id); project.EventCount = term?.Total ?? 0; project.StackCount = (long)(term?.Numbers[0] ?? 0); } return(viewProjects); }
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> projects) { if (projects.Count <= 0) { return(projects); } var organizations = await _organizationRepository.GetByIdsAsync(projects.Select(p => p.Id).ToArray(), true); StringBuilder builder = new StringBuilder(); for (int index = 0; index < projects.Count; index++) { if (index > 0) { builder.Append(" OR "); } var project = projects[index]; var organization = organizations.Documents.FirstOrDefault(o => o.Id == project.Id); if (organization != null && organization.RetentionDays > 0) { builder.AppendFormat("(project:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", project.Id, organization.RetentionDays); } else { builder.AppendFormat("project:{0}", project.Id); } } var fields = new List <FieldAggregation> { new FieldAggregation { Type = FieldAggregationType.Distinct, Field = "stack_id" } }; var result = await _stats.GetNumbersTermsStatsAsync("project_id", fields, DateTime.MinValue, DateTime.MaxValue, builder.ToString(), max : projects.Count); foreach (var project in projects) { var projectStats = result.Terms.FirstOrDefault(t => t.Term == project.Id); project.EventCount = projectStats?.Total ?? 0; project.StackCount = (long)(projectStats?.Numbers[0] ?? 0); } return(projects); }
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> organizations) { if (organizations.Count <= 0) { return(organizations); } StringBuilder builder = new StringBuilder(); for (int index = 0; index < organizations.Count; index++) { if (index > 0) { builder.Append(" OR "); } var organization = organizations[index]; if (organization.RetentionDays > 0) { builder.AppendFormat("(organization:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", organization.Id, organization.RetentionDays); } else { builder.AppendFormat("organization:{0}", organization.Id); } } var fields = new List <FieldAggregation> { new FieldAggregation { Type = FieldAggregationType.Distinct, Field = "stack_id" } }; var result = await _stats.GetNumbersTermsStatsAsync("organization_id", fields, DateTime.MinValue, DateTime.MaxValue, builder.ToString(), max : organizations.Count); foreach (var organization in organizations) { var organizationStats = result.Terms.FirstOrDefault(t => t.Term == organization.Id); organization.EventCount = organizationStats?.Total ?? 0; organization.StackCount = (long)(organizationStats?.Numbers[0] ?? 0); organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id); } return(organizations); }