public void CanGetEventTermStatsByTag()
{
// capture start date before generating data to make sure that our time range for stats includes all items
var startDate = DateTime.UtcNow.SubtractDays(60);
const int eventCount = 100;
RemoveData();
CreateData(eventCount, false);
_client.Refresh(d => d.Force());
_statsClient.DisplayStats();
var result = _stats.GetTermsStats(startDate, DateTime.UtcNow, "tags", null, userFilter: "project:" + TestConstants.ProjectId);
Assert.Equal(eventCount, result.Total);
// each event can be in multiple tag buckets since an event can have up to 3 sample tags
Assert.InRange(result.Terms.Sum(t => t.Total), eventCount, eventCount * 3);
Assert.InRange(result.Terms.Sum(t => t.New), 1, 25 * TestConstants.EventTags.Count);
Assert.InRange(result.Terms.Count, 1, TestConstants.EventTags.Count);
foreach (var term in result.Terms)
{
Assert.InRange(term.New, 1, 25);
//Assert.InRange(term.Unique, 1, 25);
Assert.Equal(term.Total, term.Timeline.Sum(t => t.Total));
}
}
private IHttpActionResult FrequentInternal(string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10)
{
page = GetPage(page);
limit = GetLimit(limit);
var skip = GetSkip(page, limit);
if (skip > MAXIMUM_SKIP)
{
return(Ok(new object[0]));
}
var validationResult = QueryProcessor.Process(userFilter);
if (!validationResult.IsValid)
{
return(BadRequest(validationResult.Message));
}
if (String.IsNullOrEmpty(systemFilter))
{
systemFilter = GetAssociatedOrganizationsFilter(_organizationRepository, validationResult.UsesPremiumFeatures, HasOrganizationOrProjectFilter(userFilter));
}
var timeInfo = GetTimeInfo(time, offset);
ICollection <TermStatsItem> terms;
try {
terms = _eventStats.GetTermsStats(timeInfo.UtcRange.Start, timeInfo.UtcRange.End, "stack_id", systemFilter, userFilter, timeInfo.Offset, GetSkip(page + 1, limit) + 1).Terms;
} catch (ApplicationException ex) {
Log.Error().Exception(ex)
.Property("Search Filter", new { SystemFilter = systemFilter, UserFilter = userFilter, Time = time, Offset = offset, Page = page, Limit = limit })
.Tag("Search")
.Property("User", ExceptionlessUser)
.ContextProperty("HttpActionContext", ActionContext)
.Write();
return(BadRequest("An error has occurred. Please check your search filter."));
}
if (terms.Count == 0)
{
return(Ok(new object[0]));
}
var stackIds = terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray();
var stacks = _stackRepository.GetByIds(stackIds).Select(s => s.ApplyOffset(timeInfo.Offset)).ToList();
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.InvariantCultureIgnoreCase))
{
var summaries = GetStackSummaries(stacks, terms);
return(OkWithResourceLinks(GetStackSummaries(stacks, terms).Take(limit).ToList(), summaries.Count > limit, page));
}
return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page));
}
public IHttpActionResult FrequentInternal(string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10)
{
page = GetPage(page);
limit = GetLimit(limit);
var skip = GetSkip(page, limit);
if (skip > MAXIMUM_SKIP)
{
return(Ok(new object[0]));
}
var validationResult = QueryValidator.Validate(userFilter);
if (!validationResult.IsValid)
{
return(BadRequest(validationResult.Message));
}
if (String.IsNullOrEmpty(systemFilter))
{
systemFilter = GetAssociatedOrganizationsFilter(_organizationRepository, validationResult.UsesPremiumFeatures, HasOrganizationOrProjectFilter(userFilter));
}
var timeInfo = GetTimeInfo(time, offset);
var terms = _eventStats.GetTermsStats(timeInfo.UtcRange.Start, timeInfo.UtcRange.End, "stack_id", systemFilter, userFilter, timeInfo.Offset, GetSkip(page + 1, limit) + 1).Terms;
if (terms.Count == 0)
{
return(Ok(new object[0]));
}
var stackIds = terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray();
var stacks = _stackRepository.GetByIds(stackIds).Select(s => s.ApplyOffset(timeInfo.Offset)).ToList();
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.InvariantCultureIgnoreCase))
{
var summaries = GetStackSummaries(stacks, terms);
return(OkWithResourceLinks(GetStackSummaries(stacks, terms).Take(limit).ToList(), summaries.Count > limit, page));
}
return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page));
}
public void CanGetEventTermStats()
{
// capture start date before generating data to make sure that our time range for stats includes all items
var startDate = DateTime.UtcNow.SubtractDays(60);
const int eventCount = 10;
RemoveData();
CreateData(eventCount);
_client.Refresh();
var result = _stats.GetTermsStats(startDate, DateTime.UtcNow, "tags");
Assert.Equal(eventCount, result.Total);
}
public IHttpActionResult FrequentInternal(string systemFilter = null, string userFilter = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10)
{
page = GetPage(page);
limit = GetLimit(limit);
var skip = GetSkip(page, limit);
if (skip > MAXIMUM_SKIP)
{
return(Ok(new object[0]));
}
if (String.IsNullOrEmpty(systemFilter))
{
systemFilter = GetAssociatedOrganizationsFilter();
}
var timeInfo = GetTimeInfo(time, offset);
var terms = _eventStats.GetTermsStats(timeInfo.UtcRange.Start, timeInfo.UtcRange.End, "stack_id", systemFilter, userFilter, timeInfo.Offset, GetSkip(page + 1, limit) + 1).Terms;
if (terms.Count == 0)
{
return(Ok(new object[0]));
}
// TODO: Apply retention cutoff
var stackIds = terms.Skip(skip).Take(limit + 1).Select(t => t.Term).ToArray();
var stacks = _stackRepository.GetByIds(stackIds).Select(s => s.ApplyOffset(timeInfo.Offset)).ToList();
// TODO: Add header that contains the number of stacks outside of the retention period.
if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.InvariantCultureIgnoreCase))
{
var summaries = GetStackSummaries(stacks, terms);
return(OkWithResourceLinks(GetStackSummaries(stacks, terms).Take(limit).ToList(), summaries.Count > limit, page));
}
return(OkWithResourceLinks(stacks.Take(limit).ToList(), stacks.Count > limit, page));
}
private List <ViewProject> PopulateProjectStats(List <ViewProject> projects)
{
if (projects.Count <= 0)
{
return(projects);
}
var organizations = _organizationRepository.GetByIds(projects.Select(p => p.Id).ToArray(), useCache: true);
StringBuilder builder = new StringBuilder();
for (int index = 0; index < projects.Count; index++)
{
if (index > 0)
{
builder.Append(" OR ");
}
var project = projects[index];
var organization = organizations.FirstOrDefault(o => o.Id == project.Id);
if (organization != null && organization.RetentionDays > 0)
{
builder.AppendFormat("(project:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", project.Id, organization.RetentionDays);
}
else
{
builder.AppendFormat("project:{0}", project.Id);
}
}
var result = _stats.GetTermsStats(DateTime.MinValue, DateTime.MaxValue, "project_id", builder.ToString());
foreach (var project in projects)
{
var projectStats = result.Terms.FirstOrDefault(t => t.Term == project.Id);
project.EventCount = projectStats != null ? projectStats.Total : 0;
project.StackCount = projectStats != null ? projectStats.Unique : 0;
}
return(projects);
}
private List <ViewOrganization> PopulateOrganizationStats(List <ViewOrganization> organizations)
{
if (organizations.Count <= 0)
{
return(organizations);
}
StringBuilder builder = new StringBuilder();
for (int index = 0; index < organizations.Count; index++)
{
if (index > 0)
{
builder.Append(" OR ");
}
var organization = organizations[index];
if (organization.RetentionDays > 0)
{
builder.AppendFormat("(organization:{0} AND (date:[now/d-{1}d TO now/d+1d}} OR last:[now/d-{1}d TO now/d+1d}}))", organization.Id, organization.RetentionDays);
}
else
{
builder.AppendFormat("organization:{0}", organization.Id);
}
}
var result = _stats.GetTermsStats(DateTime.MinValue, DateTime.MaxValue, "organization_id", builder.ToString());
foreach (var organization in organizations)
{
var organizationStats = result.Terms.FirstOrDefault(t => t.Term == organization.Id);
organization.EventCount = organizationStats != null ? organizationStats.Total : 0;
organization.StackCount = organizationStats != null ? organizationStats.Unique : 0;
organization.ProjectCount = _projectRepository.GetByOrganizationId(organization.Id, useCache: true).Count;
}
return(organizations);
}
private void ProcessSummaryNotification(SummaryNotification data)
{
var project = _projectRepository.GetById(data.Id, true);
var organization = _organizationRepository.GetById(project.OrganizationId, true);
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
return;
}
var users = _userRepository.GetByIds(userIds).Where(u => u.IsEmailAddressVerified).ToList();
if (users.Count == 0)
{
return;
}
long count;
var paging = new PagingOptions {
Limit = 5
};
List <Stack> newest = _stackRepository.GetNew(project.Id, data.UtcStartTime, data.UtcEndTime, paging).ToList();
var result = _stats.GetTermsStats(data.UtcStartTime, data.UtcEndTime, "stack_id", "project:" + data.Id, max: 5);
var mostFrequent = result.Terms.Take(5).ToList();
var stacks = _stackRepository.GetByIds(mostFrequent.Select(s => s.Term).ToList());
foreach (var frequent in mostFrequent)
{
var stack = stacks.SingleOrDefault(s => s.Id == frequent.Term);
if (stack == null)
{
mostFrequent.RemoveAll(r => r.Term == frequent.Term);
continue;
}
// Stat's Id and Total properties are already calculated in the Results.
//frequent.Type = stack.SignatureInfo.ContainsKey("ExceptionType") ? stack.SignatureInfo["ExceptionType"] : null;
//frequent.Method = stack.SignatureInfo.ContainsKey("Method") ? stack.SignatureInfo["Method"] : null;
//frequent.Path = stack.SignatureInfo.ContainsKey("Path") ? stack.SignatureInfo["Path"] : null;
//frequent.Is404 = stack.SignatureInfo.ContainsKey("Path");
//frequent.Title = stack.Title;
//frequent.First = stack.FirstOccurrence;
//frequent.Last = stack.LastOccurrence;
}
var notification = new SummaryNotificationModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
//Total = result.Total,
//PerHourAverage = result.PerHourAverage,
//NewTotal = result.NewTotal,
//New = newest,
//UniqueTotal = result.UniqueTotal,
//MostFrequent = mostFrequent,
//HasSubmittedErrors = project.TotalErrorCount > 0,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users.Where(u => u.EmailNotificationsEnabled))
{
_mailer.SendSummaryNotification(user.EmailAddress, notification);
}
}
private void ProcessSummaryNotification(SummaryNotification data)
{
var project = _projectRepository.GetById(data.Id, true);
var organization = _organizationRepository.GetById(project.OrganizationId, true);
var userIds = project.NotificationSettings.Where(n => n.Value.SendDailySummary).Select(n => n.Key).ToList();
if (userIds.Count == 0)
{
Log.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
var users = _userRepository.GetByIds(userIds).Where(u => u.IsEmailAddressVerified && u.EmailNotificationsEnabled && u.OrganizationIds.Contains(organization.Id)).ToList();
if (users.Count == 0)
{
Log.Info().Message("Project \"{0}\" has no users to send summary to.", project.Id).Write();
return;
}
Log.Info().Message("Sending daily summary: users={0} project={1}", users.Count, project.Id).Write();
var paging = new PagingOptions {
Limit = 5
};
List <Stack> newest = _stackRepository.GetNew(project.Id, data.UtcStartTime, data.UtcEndTime, paging).ToList();
var result = _stats.GetTermsStats(data.UtcStartTime, data.UtcEndTime, "stack_id", "type:error project:" + data.Id, max: 5);
//var termStatsList = result.Terms.Take(5).ToList();
//var stacks = _stackRepository.GetByIds(termStatsList.Select(s => s.Term).ToList());
bool hasSubmittedErrors = result.Total > 0;
if (!hasSubmittedErrors)
{
hasSubmittedErrors = _eventRepository.GetCountByProjectId(project.Id) > 0;
}
var mostFrequent = new List <EventStackResult>();
//foreach (var termStats in termStatsList) {
// var stack = stacks.SingleOrDefault(s => s.Id == termStats.Term);
// if (stack == null)
// continue;
// mostFrequent.Add(new EventStackResult {
// First = termStats.FirstOccurrence,
// Last = termStats.LastOccurrence,
// Id = stack.Id,
// Title = stack.Title,
// Total = termStats.Total,
// Type = stack.SignatureInfo.ContainsKey("ExceptionType") ? stack.SignatureInfo["ExceptionType"] : null,
// Method = stack.SignatureInfo.ContainsKey("Method") ? stack.SignatureInfo["Method"] : null,
// Path = stack.SignatureInfo.ContainsKey("Source") ? stack.SignatureInfo["Source"] : null,
// Is404 = stack.SignatureInfo.ContainsKey("Type") && stack.SignatureInfo["Type"] == "404"
// });
//}
var notification = new DailySummaryModel {
ProjectId = project.Id,
ProjectName = project.Name,
StartDate = data.UtcStartTime,
EndDate = data.UtcEndTime,
Total = result.Total,
PerHourAverage = result.Total / data.UtcEndTime.Subtract(data.UtcStartTime).TotalHours,
NewTotal = result.New,
New = newest,
UniqueTotal = result.Unique,
MostFrequent = mostFrequent,
HasSubmittedEvents = hasSubmittedErrors,
IsFreePlan = organization.PlanId == BillingManager.FreePlan.Id
};
foreach (var user in users)
{
_mailer.SendDailySummary(user.EmailAddress, notification);
}
Log.Info().Message("Done sending daily summary: users={0} project={1} events={2}", users.Count, project.Id, notification.Total).Write();
}