protected bool ShouldApplySystemFilter(IExceptionlessSystemFilterQuery sf, string filter)
{
// Apply filter to non admin user.
if (!Request.IsGlobalAdmin())
{
return(true);
}
// Apply filter as it's scoped via a controller action.
if (!sf.IsUserOrganizationsFilter)
{
return(true);
}
// Empty user filter
if (String.IsNullOrEmpty(filter))
{
return(true);
}
// Used for impersonating a user. Only skip the filter if it contains an org, project or stack.
bool hasOrganizationOrProjectOrStackFilter = filter.Contains("organization:") || filter.Contains("project:") || filter.Contains("stack:");
return(!hasOrganizationOrProjectOrStackFilter);
}
public Task <FindResults <PersistentEvent> > GetByFilterAsync(IExceptionlessSystemFilterQuery systemFilter, string userFilter, string sort, string field, DateTime utcStart, DateTime utcEnd, PagingOptions paging)
{
var search = new ExceptionlessQuery()
.WithDateRange(utcStart, utcEnd, field ?? ElasticType.GetFieldName(e => e.Date))
.WithIndexes(utcStart, utcEnd)
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithPaging(paging);
search = !String.IsNullOrEmpty(sort) ? search.WithSort(sort) : search.WithSortDescending((PersistentEvent e) => e.Date);
return(FindAsync(search));
}
public Task <FindResults <Stack> > GetByFilterAsync(IExceptionlessSystemFilterQuery systemFilter, string userFilter, string sort, string field, DateTime utcStart, DateTime utcEnd, PagingOptions paging)
{
var search = new ExceptionlessQuery()
.WithDateRange(utcStart, utcEnd, field ?? ElasticType.GetFieldName(s => s.LastOccurrence))
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithPaging(paging)
.WithSort(sort);
search = !String.IsNullOrEmpty(sort) ? search.WithSort(sort) : search.WithSortDescending((Stack s) => s.LastOccurrence);
return(FindAsync(search));
}
public Task <FindResults <Stack> > GetByFilterAsync(IExceptionlessSystemFilterQuery systemFilter, string userFilter, SortingOptions sorting, string field, DateTime utcStart, DateTime utcEnd, PagingOptions paging)
{
if (sorting.Fields.Count == 0)
{
sorting.Fields.Add(new FieldSort {
Field = StackIndexType.Fields.LastOccurrence, Order = SortOrder.Descending
});
}
var search = new ExceptionlessQuery()
.WithDateRange(utcStart, utcEnd, field ?? StackIndexType.Fields.LastOccurrence)
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithPaging(paging)
.WithSort(sorting);
return(FindAsync(search));
}
protected virtual async Task <IHttpActionResult> GetCountAsync(IExceptionlessSystemFilterQuery sf, TimeInfo ti, string filter = null, string aggregations = null)
{
var pr = await _validator.ValidateQueryAsync(filter);
if (!pr.IsValid)
{
return(BadRequest(pr.Message));
}
var far = await _validator.ValidateAggregationsAsync(aggregations);
if (!far.IsValid)
{
return(BadRequest(far.Message));
}
sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || far.UsesPremiumFeatures;
IRepositoryQuery query = new ExceptionlessQuery()
.WithSystemFilter(ShouldApplySystemFilter(sf, filter) ? sf : null)
.WithDateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field)
.WithIndexes(ti.Range.UtcStart, ti.Range.UtcEnd);
CountResult result;
try {
result = await _repository.CountBySearchAsync(query, filter, aggregations);
} catch (Exception ex) {
_logger.Error().Exception(ex)
.Message("An error has occurred. Please check your filter or aggregations.")
.Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Aggregations = aggregations })
.Tag("Search")
.Identity(CurrentUser.EmailAddress)
.Property("User", CurrentUser)
.SetActionContext(ActionContext)
.Write();
return(BadRequest("An error has occurred. Please check your search filter."));
}
return(Ok(result));
}
private async Task <string> GetNextEventIdAsync(PersistentEvent ev, IExceptionlessSystemFilterQuery systemFilter = null, string userFilter = null, DateTime?utcStart = null, DateTime?utcEnd = null)
{
if (ev == null)
{
return(null);
}
if (!utcStart.HasValue || utcStart.Value.IsBefore(ev.Date.UtcDateTime))
{
utcStart = ev.Date.UtcDateTime;
}
if (!utcEnd.HasValue || utcEnd.Value.IsAfter(SystemClock.UtcNow))
{
utcEnd = SystemClock.UtcNow;
}
var utcEventDate = ev.Date.UtcDateTime;
// utcEnd is before the current event date.
if (utcStart > utcEventDate || utcEnd < utcEventDate)
{
return(null);
}
if (String.IsNullOrEmpty(userFilter))
{
userFilter = String.Concat(EventIndexType.Alias.StackId, ":", ev.StackId);
}
var results = await FindAsync(new ExceptionlessQuery()
.WithDateRange(utcEventDate, utcEnd, (PersistentEvent e) => e.Date)
.WithIndexes(utcStart, utcEventDate)
.WithSortAscending((PersistentEvent e) => e.Date)
.WithLimit(10)
.IncludeFields((PersistentEvent e) => e.Id, e => e.Date)
.WithSystemFilter(systemFilter)
.WithElasticFilter(!Query <PersistentEvent> .Ids(ids => ids.Values(ev.Id)))
.WithFilter(userFilter)).AnyContext();
if (results.Total == 0)
{
return(null);
}
// make sure we don't have records with the exact same occurrence date
if (results.Documents.All(t => t.Date != ev.Date))
{
return(results.Documents.OrderBy(t => t.Date).ThenBy(t => t.Id).First().Id);
}
// we have records with the exact same occurrence date, we need to figure out the order of those
// put our target error into the mix, sort it and return the result after the target
var unionResults = results.Documents.Union(new[] { ev })
.OrderBy(t => t.Date.Ticks).ThenBy(t => t.Id)
.ToList();
var index = unionResults.FindIndex(t => t.Id == ev.Id);
return(index == unionResults.Count - 1 ? null : unionResults[index + 1].Id);
}
public async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(PersistentEvent ev, IExceptionlessSystemFilterQuery systemFilter, string userFilter, DateTime?utcStart, DateTime?utcEnd)
{
var previous = await GetPreviousEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext();
var next = await GetNextEventIdAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext();
return(new PreviousAndNextEventIdResult {
Previous = previous,
Next = next
});
}
public async Task <NumbersTermStatsResult> GetNumbersTermsStatsAsync(string term, IEnumerable <FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, IExceptionlessSystemFilterQuery systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25)
{
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
{
throw new ArgumentException("Must be a valid term.", nameof(term));
}
if (!displayTimeOffset.HasValue)
{
displayTimeOffset = TimeSpan.Zero;
}
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndexType.Fields.Date)
.WithIndexes(utcStart, utcEnd);
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
{
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
}
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var descriptor = new SearchDescriptor <PersistentEvent>()
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Indices(_configuration.Events.Event.GetIndexesByQuery(filter))
.Type(_configuration.Events.Event.Name)
.Aggregations(agg => BuildAggregations(agg
.Terms("terms", t => BuildTermSort(t
.Field(term)
.Size(max)
.Aggregations(agg2 => BuildAggregations(agg2
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date)), fields)
), fields)
), fields)
);
_configuration.Events.Event.QueryBuilder.ConfigureSearch(filter, GetQueryOptions(), descriptor);
var response = await _configuration.Client.SearchAsync <PersistentEvent>(descriptor).AnyContext();
_logger.Trace(() => response.GetRequest());
if (!response.IsValid)
{
string message = $"Retrieving stats failed: {response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message).Property("request", response.GetRequest()).Write();
throw new ApplicationException(message, response.ConnectionStatus.OriginalException);
}
var stats = new NumbersTermStatsResult {
Total = response.Total,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value),
Numbers = GetNumbers(response.Aggs, fields)
};
var terms = response.Aggs.Terms("terms");
if (terms != null)
{
stats.Terms.AddRange(terms.Items.Select(i => {
var item = new NumbersTermStatsItem {
Total = i.DocCount,
Term = i.Key,
Numbers = GetNumbers(i, fields)
};
var termFirstOccurrence = i.Min("first_occurrence");
if (termFirstOccurrence?.Value != null)
{
item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var termLastOccurrence = i.Max("last_occurrence");
if (termLastOccurrence?.Value != null)
{
item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
return(item);
}));
}
return(stats);
}
public async Task <NumbersStatsResult> GetNumbersStatsAsync(IEnumerable <FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, IExceptionlessSystemFilterQuery systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null)
{
if (!displayTimeOffset.HasValue)
{
displayTimeOffset = TimeSpan.Zero;
}
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndexType.Fields.Date)
.WithIndexes(utcStart, utcEnd);
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
{
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
}
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var descriptor = new SearchDescriptor <PersistentEvent>()
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Indices(_configuration.Events.Event.GetIndexesByQuery(filter))
.Type(_configuration.Events.Event.Name)
.Aggregations(agg => BuildAggregations(agg, fields));
_configuration.Events.Event.QueryBuilder.ConfigureSearch(filter, GetQueryOptions(), descriptor);
var response = await _configuration.Client.SearchAsync <PersistentEvent>(descriptor).AnyContext();
_logger.Trace(() => response.GetRequest());
if (!response.IsValid)
{
string message = $"Retrieving stats failed: {response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message).Property("request", response.GetRequest()).Write();
throw new ApplicationException(message, response.ConnectionStatus.OriginalException);
}
return(new NumbersStatsResult {
Total = response.Total,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value),
Numbers = GetNumbers(response.Aggs, fields)
});
}
public async Task <NumbersTimelineStatsResult> GetNumbersTimelineStatsAsync(IEnumerable <FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, IExceptionlessSystemFilterQuery systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int desiredDataPoints = 100)
{
if (!displayTimeOffset.HasValue)
{
displayTimeOffset = TimeSpan.Zero;
}
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndexType.Fields.Date)
.WithIndexes(utcStart, utcEnd);
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
{
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
}
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var descriptor = new SearchDescriptor <PersistentEvent>()
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Indices(_configuration.Events.Event.GetIndexesByQuery(filter))
.Type(_configuration.Events.Event.Name)
.Aggregations(agg => BuildAggregations(agg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
.Aggregations(agg2 => BuildAggregations(agg2, fields))
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date)), fields)
);
_configuration.Events.Event.QueryBuilder.ConfigureSearch(filter, GetQueryOptions(), descriptor);
var response = await _configuration.Client.SearchAsync <PersistentEvent>(descriptor).AnyContext();
_logger.Trace(() => response.GetRequest());
if (!response.IsValid)
{
string message = $"Retrieving stats failed: {response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message).Property("request", response.GetRequest()).Write();
throw new ApplicationException(message, response.ConnectionStatus.OriginalException);
}
var stats = new NumbersTimelineStatsResult {
Total = response.Total, Numbers = GetNumbers(response.Aggs, fields)
};
var timeline = response.Aggs.DateHistogram("timelime");
if (timeline != null)
{
stats.Timeline.AddRange(timeline.Items.Select(i => new NumbersTimelineItem {
Date = i.Date,
Total = i.DocCount,
Numbers = GetNumbers(i, fields)
}));
}
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
var totalHours = stats.End.Subtract(stats.Start).TotalHours;
if (totalHours > 0.0)
{
stats.AvgPerHour = stats.Total / totalHours;
}
if (stats.Timeline.Count <= 0)
{
return(stats);
}
var firstOccurrence = response.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
{
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var lastOccurrence = response.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
{
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
return(stats);
}
public static async Task <PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(this IEventRepository repository, string id, IExceptionlessSystemFilterQuery systemFilter = null, string userFilter = null, DateTime?utcStart = null, DateTime?utcEnd = null)
{
var ev = await repository.GetByIdAsync(id, true).AnyContext();
return(await repository.GetPreviousAndNextEventIdsAsync(ev, systemFilter, userFilter, utcStart, utcEnd).AnyContext());
}
private async Task <string> GetNextEventIdAsync(PersistentEvent ev, IExceptionlessSystemFilterQuery systemFilter = null, string userFilter = null, DateTime?utcStart = null, DateTime?utcEnd = null)
{
if (ev == null)
{
return(null);
}
if (!utcStart.HasValue)
{
utcStart = DateTime.MinValue;
}
if (!utcEnd.HasValue)
{
utcEnd = DateTime.MaxValue;
}
var utcEventDate = ev.Date.ToUniversalTime().DateTime;
// utcEnd is before the current event date.
if (utcStart > utcEventDate || utcEnd < utcEventDate)
{
return(null);
}
if (String.IsNullOrEmpty(userFilter))
{
userFilter = "stack:" + ev.StackId;
}
var results = await FindAsync(new ExceptionlessQuery()
.WithDateRange(utcEventDate, utcEnd, EventIndexType.Fields.Date)
.WithIndexes(utcStart, utcEventDate)
.WithSort(EventIndexType.Fields.Date, SortOrder.Ascending)
.WithLimit(10)
.WithSelectedFields("id", "date")
.WithSystemFilter(systemFilter)
.WithElasticFilter(!Filter <PersistentEvent> .Ids(new[] { ev.Id }))
.WithFilter(userFilter)).AnyContext();
if (results.Total == 0)
{
return(null);
}
// make sure we don't have records with the exact same occurrence date
if (results.Documents.All(t => t.Date != ev.Date))
{
return(results.Documents.OrderBy(t => t.Date).ThenBy(t => t.Id).First().Id);
}
// we have records with the exact same occurrence date, we need to figure out the order of those
// put our target error into the mix, sort it and return the result after the target
var unionResults = results.Documents.Union(new[] { ev })
.OrderBy(t => t.Date.Ticks).ThenBy(t => t.Id)
.ToList();
var index = unionResults.FindIndex(t => t.Id == ev.Id);
return(index == unionResults.Count - 1 ? null : unionResults[index + 1].Id);
}
