public async Task CanGetEventTermStatsByTagAsync()
{
// capture start date before generating data to make sure that our time range for stats includes all items
var startDate = DateTime.UtcNow.SubtractDays(60);
const int eventCount = 100;
await RemoveDataAsync();
await CreateDataAsync(eventCount, false);
var fields = FieldAggregationProcessor.Process("term:is_first_occurrence:-F", false);
Assert.True(fields.IsValid);
var result = await _stats.GetNumbersTermsStatsAsync("tags", fields.Aggregations, startDate, DateTime.UtcNow, null, userFilter : $"project:{TestConstants.ProjectId}");
Assert.Equal(eventCount, result.Total);
// each event can be in multiple tag buckets since an event can have up to 3 sample tags
Assert.InRange(result.Terms.Sum(t => t.Total), eventCount, eventCount * 3);
Assert.InRange(result.Terms.Sum(t => t.Numbers[0]), 1, 25 * TestConstants.EventTags.Count); // new
Assert.InRange(result.Terms.Count, 1, TestConstants.EventTags.Count);
foreach (var term in result.Terms)
{
Assert.InRange(term.Numbers[0], 1, 25); // new
}
}
public async Task CanGetEventTermStatsByTagAsync()
{
// capture start date before generating data to make sure that our time range for stats includes all items
var startDate = SystemClock.UtcNow.SubtractDays(60);
const int eventCount = 100;
await CreateDataAsync(eventCount, false);
Log.MinimumLevel = LogLevel.Trace;
var fields = FieldAggregationProcessor.Process("term:is_first_occurrence:-F", false);
Assert.True(fields.IsValid);
var sf = new ExceptionlessSystemFilterQuery(ProjectData.GenerateSampleProject(), OrganizationData.GenerateSampleOrganization());
var result = await _stats.GetNumbersTermsStatsAsync("tags", fields.Aggregations, startDate, SystemClock.UtcNow, sf, "fixed:false");
Assert.Equal(eventCount, result.Total);
// each event can be in multiple tag buckets since an event can have up to 3 sample tags
Assert.InRange(result.Terms.Sum(t => t.Total), eventCount, eventCount * 3);
Assert.InRange(result.Terms.Sum(t => t.Numbers[0]), 1, 25 * TestConstants.EventTags.Count); // new
Assert.InRange(result.Terms.Count, 1, TestConstants.EventTags.Count);
foreach (var term in result.Terms)
{
Assert.InRange(term.Numbers[0], 1, 25); // new
}
}
private async Task <IHttpActionResult> GetAllByTermsAsync(ICollection <FieldAggregation> fields, string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10)
{
page = GetPage(page);
limit = GetLimit(limit);
var skip = GetSkip(page, limit);
if (skip > MAXIMUM_SKIP)
{
return(Ok(new object[0]));
}
var pr = QueryProcessor.Process(userFilter);
if (!pr.IsValid)
{
return(BadRequest(pr.Message));
}
var organizations = await GetAssociatedOrganizationsAsync(_organizationRepository);
var ti = GetTimeInfo(time, offset, organizations.GetRetentionUtcCutoff());
systemFilter = String.Join(" ", new [] { systemFilter, BuildSystemFilter(organizations, userFilter, pr.UsesPremiumFeatures) }.Where(f => !String.IsNullOrWhiteSpace(f)));
try {
var ntsr = await _eventStats.GetNumbersTermsStatsAsync("stack_id", fields, ti.UtcRange.Start, ti.UtcRange.End, systemFilter, userFilter, ti.Offset, GetSkip(page + 1, limit) + 1);
if (ntsr.Terms.Count == 0)
{
return(Ok(new object[0]));
}
var stackIds = ntsr.Terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray();
var stacks = (await _stackRepository.GetByIdsAsync(stackIds)).Documents.Select(s => s.ApplyOffset(ti.Offset)).ToList();
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase))
{
var summaries = await GetStackSummariesAsync(stacks, ntsr, systemFilter, ti.UtcRange.Start, ti.UtcRange.End);
return(OkWithResourceLinks(summaries.Take(limit).ToList(), summaries.Count > limit, page));
}
return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page));
} catch (ApplicationException ex) {
_logger.Error().Exception(ex)
.Message("An error has occurred. Please check your search filter.")
.Property("Search Filter", new { SystemFilter = systemFilter, UserFilter = userFilter, Time = time, Offset = offset, Page = page, Limit = limit })
.Tag("Search")
.Identity(ExceptionlessUser.EmailAddress)
.Property("User", ExceptionlessUser)
.SetActionContext(ActionContext)
.Write();
return(BadRequest("An error has occurred. Please check your search filter."));
}
}
private async Task <IHttpActionResult> FrequentInternalAsync(string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10)
{
page = GetPage(page);
limit = GetLimit(limit);
var skip = GetSkip(page, limit);
if (skip > MAXIMUM_SKIP)
{
return(Ok(new object[0]));
}
var validationResult = QueryProcessor.Process(userFilter);
if (!validationResult.IsValid)
{
return(BadRequest(validationResult.Message));
}
if (String.IsNullOrEmpty(systemFilter))
{
systemFilter = await GetAssociatedOrganizationsFilterAsync(_organizationRepository, validationResult.UsesPremiumFeatures, HasOrganizationOrProjectFilter(userFilter));
}
var timeInfo = GetTimeInfo(time, offset);
try {
var terms = (await _eventStats.GetNumbersTermsStatsAsync("stack_id", new List <FieldAggregation>(), timeInfo.UtcRange.Start, timeInfo.UtcRange.End, systemFilter, userFilter, timeInfo.Offset, GetSkip(page + 1, limit) + 1)).Terms;
if (terms.Count == 0)
{
return(Ok(new object[0]));
}
var stackIds = terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray();
var stacks = (await _stackRepository.GetByIdsAsync(stackIds)).Documents.Select(s => s.ApplyOffset(timeInfo.Offset)).ToList();
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase))
{
var summaries = GetStackSummaries(stacks, terms);
return(OkWithResourceLinks(GetStackSummaries(stacks, terms).Take(limit).ToList(), summaries.Count > limit, page));
}
return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page));
} catch (ApplicationException ex) {
_logger.Error().Exception(ex)
.Property("Search Filter", new { SystemFilter = systemFilter, UserFilter = userFilter, Time = time, Offset = offset, Page = page, Limit = limit })
.Tag("Search")
.Identity(ExceptionlessUser.EmailAddress)
.Property("User", ExceptionlessUser)
.SetActionContext(ActionContext)
.Write();
return(BadRequest("An error has occurred. Please check your search filter."));
}
}
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> viewOrganizations)
{
if (viewOrganizations.Count <= 0)
{
return(viewOrganizations);
}
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
}
};
var organizations = viewOrganizations.Select(o => new Organization {
Id = o.Id, RetentionDays = o.RetentionDays
}).ToList();
var sf = new ExceptionlessSystemFilterQuery(organizations);
var result = await _stats.GetNumbersTermsStatsAsync("organization_id", fields, organizations.GetRetentionUtcCutoff(), DateTime.MaxValue, sf, max : viewOrganizations.Count);
foreach (var organization in viewOrganizations)
{
var organizationStats = result.Terms.FirstOrDefault(t => t.Term == organization.Id);
organization.EventCount = organizationStats?.Total ?? 0;
organization.StackCount = (long)(organizationStats?.Numbers[0] ?? 0);
organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id);
}
return(viewOrganizations);
}
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> viewProjects)
{
if (viewProjects.Count <= 0)
{
return(viewProjects);
}
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
}
};
var organizations = (await _organizationRepository.GetByIdsAsync(viewProjects.Select(p => p.OrganizationId).ToArray(), true)).Documents;
var filter = viewProjects.Select(p => new Project {
Id = p.Id, OrganizationId = p.OrganizationId
}).ToList().BuildRetentionFilter(organizations);
var ntsr = await _stats.GetNumbersTermsStatsAsync("project_id", fields, organizations.GetRetentionUtcCutoff(), DateTime.MaxValue, filter, max : viewProjects.Count);
foreach (var project in viewProjects)
{
var term = ntsr.Terms.FirstOrDefault(t => t.Term == project.Id);
project.EventCount = term?.Total ?? 0;
project.StackCount = (long)(term?.Numbers[0] ?? 0);
}
return(viewProjects);
}
private async Task <List <ViewProject> > PopulateProjectStatsAsync(List <ViewProject> projects)
{
if (projects.Count <= 0)
{
return(projects);
}
var organizations = await _organizationRepository.GetByIdsAsync(projects.Select(p => p.Id).ToArray(), true);
StringBuilder builder = new StringBuilder();
for (int index = 0; index < projects.Count; index++)
{
if (index > 0)
{
builder.Append(" OR ");
}
var project = projects[index];
var organization = organizations.Documents.FirstOrDefault(o => o.Id == project.Id);
if (organization != null && organization.RetentionDays > 0)
{
builder.AppendFormat("(project:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", project.Id, organization.RetentionDays);
}
else
{
builder.AppendFormat("project:{0}", project.Id);
}
}
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
}
};
var result = await _stats.GetNumbersTermsStatsAsync("project_id", fields, DateTime.MinValue, DateTime.MaxValue, builder.ToString(), max : projects.Count);
foreach (var project in projects)
{
var projectStats = result.Terms.FirstOrDefault(t => t.Term == project.Id);
project.EventCount = projectStats?.Total ?? 0;
project.StackCount = (long)(projectStats?.Numbers[0] ?? 0);
}
return(projects);
}
private async Task <List <ViewOrganization> > PopulateOrganizationStatsAsync(List <ViewOrganization> organizations)
{
if (organizations.Count <= 0)
{
return(organizations);
}
StringBuilder builder = new StringBuilder();
for (int index = 0; index < organizations.Count; index++)
{
if (index > 0)
{
builder.Append(" OR ");
}
var organization = organizations[index];
if (organization.RetentionDays > 0)
{
builder.AppendFormat("(organization:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", organization.Id, organization.RetentionDays);
}
else
{
builder.AppendFormat("organization:{0}", organization.Id);
}
}
var fields = new List <FieldAggregation> {
new FieldAggregation {
Type = FieldAggregationType.Distinct, Field = "stack_id"
}
};
var result = await _stats.GetNumbersTermsStatsAsync("organization_id", fields, DateTime.MinValue, DateTime.MaxValue, builder.ToString(), max : organizations.Count);
foreach (var organization in organizations)
{
var organizationStats = result.Terms.FirstOrDefault(t => t.Term == organization.Id);
organization.EventCount = organizationStats?.Total ?? 0;
organization.StackCount = (long)(organizationStats?.Numbers[0] ?? 0);
organization.ProjectCount = await _projectRepository.GetCountByOrganizationIdAsync(organization.Id);
}
return(organizations);
}
