public static TimeInfo ApplyMinimumUtcStartDate(this TimeInfo timeInfo, DateTime minimumUtcStartDate) { if (timeInfo.UtcRange.UtcStart < minimumUtcStartDate) timeInfo.UtcRange = new DateTimeRange(minimumUtcStartDate.SafeAdd(timeInfo.Offset), timeInfo.UtcRange.End); return timeInfo; }
protected DateTimeRange FromRelationAmountTime(string relation, int amount, string size, DateTime now) { relation = relation.ToLower(); size = size.ToLower(); if (amount < 1) throw new ArgumentException("Time amount can't be 0."); TimeSpan intervalSpan = Helper.GetTimeSpanFromName(size); if (intervalSpan != TimeSpan.Zero) { var totalSpan = TimeSpan.FromTicks(intervalSpan.Ticks * amount); switch (relation) { case "last": case "past": case "previous": return new DateTimeRange(now.Floor(intervalSpan).SafeSubtract(totalSpan), now); case "this": case "next": return new DateTimeRange(now, now.SafeAdd(totalSpan).Ceiling(intervalSpan).SubtractMilliseconds(1)); } } else if (size == "week" || size == "weeks") { switch (relation) { case "last": case "past": case "previous": return new DateTimeRange(now.SubtractWeeks(amount).StartOfDay(), now); case "this": case "next": return new DateTimeRange(now, now.AddWeeks(amount).EndOfDay()); } } else if (size == "month" || size == "months") { switch (relation) { case "last": case "past": case "previous": return new DateTimeRange(now.SubtractMonths(amount).StartOfDay(), now); case "this": case "next": return new DateTimeRange(now, now.AddMonths(amount).EndOfDay()); } } else if (size == "year" || size == "years") { switch (relation) { case "last": case "past": case "previous": return new DateTimeRange(now.SubtractYears(amount).StartOfDay(), now); case "this": case "next": return new DateTimeRange(now, now.AddYears(amount).EndOfDay()); } } return null; }
protected DateTime? FromRelationAmountTime(string relation, int amount, string size, DateTime now, bool isUpperLimit) { relation = relation.ToLower(); size = size.ToLower(); if (amount < 1) throw new ArgumentException("Time amount can't be 0."); TimeSpan intervalSpan = Helper.GetTimeSpanFromName(size); if (intervalSpan != TimeSpan.Zero) { var totalSpan = TimeSpan.FromTicks(intervalSpan.Ticks * amount); switch (relation) { case "ago": return isUpperLimit ? now.SafeSubtract(totalSpan).Ceiling(intervalSpan).SubtractMilliseconds(1) : now.SafeSubtract(totalSpan).Floor(intervalSpan); case "from now": return isUpperLimit ? now.SafeAdd(totalSpan).Ceiling(intervalSpan).SubtractMilliseconds(1) : now.SafeAdd(totalSpan).Floor(intervalSpan); } } else if (size == "week" || size == "weeks") { switch (relation) { case "ago": return isUpperLimit ? now.SubtractWeeks(amount).EndOfDay() : now.SubtractWeeks(amount).StartOfDay(); case "from now": return isUpperLimit ? now.AddWeeks(amount).EndOfDay() : now.AddWeeks(amount).StartOfDay(); } } else if (size == "month" || size == "months") { switch (relation) { case "ago": return isUpperLimit ? now.SubtractMonths(amount).EndOfDay() : now.SubtractMonths(amount).StartOfDay(); case "from now": return isUpperLimit ? now.AddMonths(amount).EndOfDay() : now.AddMonths(amount).StartOfDay(); } } else if (size == "year" || size == "years") { switch (relation) { case "ago": return isUpperLimit ? now.SubtractYears(amount).EndOfDay() : now.SubtractYears(amount).StartOfDay(); case "from now": return isUpperLimit ? now.AddYears(amount).EndOfDay() : now.AddYears(amount).StartOfDay(); } } return null; }
public async Task<NumbersStatsResult> GetNumbersStatsAsync(IEnumerable<FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext(); utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var response = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery<PersistentEvent>(filter)) .Aggregations(agg => BuildAggregations(agg, fields)) ).AnyContext(); if (!response.IsValid) { _logger.Error("Retrieving stats failed: {0}", response.ServerError.Error); throw new ApplicationException("Retrieving stats failed."); } return new NumbersStatsResult { Total = response.Total, Start = utcStart.SafeAdd(displayTimeOffset.Value), End = utcEnd.SafeAdd(displayTimeOffset.Value), Numbers = GetNumbers(response.Aggs, fields) }; }
public async Task<EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) throw new ArgumentException("Must be a valid term.", nameof(term)); var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(d => _queryBuilder.BuildQuery<PersistentEvent>(filter)) .SortAscending(ev => ev.Date) .Take(1)).AnyContext(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.DateRanges.Clear(); filter.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date); filter.Indices.Clear(); filter.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); } } utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery<PersistentEvent>(filter)) .Aggregations(agg => agg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ).AnyContext(); if (!res.IsValid) { Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventTermStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0, Start = utcStart.SafeAdd(displayTimeOffset.Value), End = utcEnd.SafeAdd(displayTimeOffset.Value) }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) stats.Unique = (long)unique.Value; var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var terms = res.Aggs.Terms("terms"); if (terms == null) return stats; stats.Terms.AddRange(terms.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique"); if (timelineUnique?.Value != null) count = (long)timelineUnique.Value; var termNew = i.Terms("new"); var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0 }; var termFirstOccurrence = i.Min("first_occurrence"); if (termFirstOccurrence?.Value != null) item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var termLastOccurrence = i.Max("last_occurrence"); if (termLastOccurrence?.Value != null) item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var timeLine = i.DateHistogram("timelime"); if (timeLine != null) { item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); } return item; })); return stats; }
public async Task<EventStatsResult> GetOccurrenceStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticSearchOptions<PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s .IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Filter(d => filter.GetElasticSearchFilter()) .SortAscending(ev => ev.Date) .Take(1)).AnyContext(); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Query(filter.GetElasticSearchQuery()) .Aggregations(agg => agg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ).AnyContext(); _elasticClient.DisableTrace(); if (!res.IsValid) { Logger.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) stats.Unique = (long)unique.Value; var timeline = res.Aggs.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique?.Value != null) count = (long)timelineUnique.Value; var timelineNew = i.Terms("tl_new"); return new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }; })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Total / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }
public async Task<EventStatsResult> GetOccurrenceStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext(); utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery<PersistentEvent>(filter)) .Aggregations(agg => agg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ).AnyContext(); if (!res.IsValid) { Logger.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) stats.Unique = (long)unique.Value; var timeline = res.Aggs.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique?.Value != null) count = (long)timelineUnique.Value; var timelineNew = i.Terms("tl_new"); return new EventTimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }; })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Total / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }
public async Task<SessionStatsResult> GetSessionStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithFieldEquals(EventIndex.Fields.PersistentEvent.Type, Event.KnownTypes.Session) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext(); utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery<PersistentEvent>(filter)) .Aggregations(agg => agg .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) .Aggregations(agg2 => agg2 .Average("tl_avg_duration", u => u.Field(ev => ev.Value)) .Cardinality("tl_users", u => u .Field(EventIndex.Fields.PersistentEvent.UserRaw) .PrecisionThreshold(100) ) ) ) .Average("avg_duration", u => u.Field(ev => ev.Value)) .Cardinality("users", u => u .Field(EventIndex.Fields.PersistentEvent.UserRaw) .PrecisionThreshold(100) ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ).AnyContext(); if (!res.IsValid) { Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } var stats = new SessionStatsResult { Sessions = res.Total }; var averageDuration = res.Aggs.Average("avg_duration"); if (averageDuration?.Value != null) stats.AvgDuration = (decimal)averageDuration.Value; var users = res.Aggs.Cardinality("users"); if (users?.Value != null) stats.Users = (long)users.Value; var timeline = res.Aggs.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { var item = new SessionTimelineItem { Date = i.Date, Sessions = i.DocCount }; var timelineAverageDuration = i.Average("tl_avg_duration"); if (timelineAverageDuration?.Value != null) item.AvgDuration = (decimal)timelineAverageDuration.Value; var timelineUsers = i.Cardinality("tl_users"); if (timelineUsers?.Value != null) item.Users = (long)timelineUsers.Value; return item; })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Sessions / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) throw new ArgumentException("Must be a valid term.", "term"); var filter = new ElasticSearchOptions<PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(); _client.EnableTrace(); // if no start date then figure out first event date if (!filter.UseStartDate) { var result = _client.Search<PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1)); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } _client.DisableTrace(); var filtered = res.Aggs.Filter("filtered"); if (filtered == null) return new EventTermStatsResult(); var stats = new EventTermStatsResult { Total = filtered.DocCount }; stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique").Value; if (timelineUnique.HasValue) count = (long)timelineUnique.Value; var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0 }; var firstOccurrence = i.Min("first_occurrence").Value; var lastOccurrence = i.Max("last_occurrence").Value; if (firstOccurrence.HasValue) item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); if (lastOccurrence.HasValue) item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); return item; })); stats.Start = utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); return stats; }
public async Task<NumbersTermStatsResult> GetNumbersTermsStatsAsync(string term, IEnumerable<FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25) { var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) throw new ArgumentException("Must be a valid term.", nameof(term)); if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext(); utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var response = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery<PersistentEvent>(filter)) .Aggregations(agg => BuildAggregations(agg .Terms("terms", t => BuildTermSort(t .Field(term) .Size(max) .Aggregations(agg2 => BuildAggregations(agg2 .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)), fields) ), fields) ), fields) ) ).AnyContext(); if (!response.IsValid) { _logger.Error("Retrieving stats failed: {0}", response.ServerError.Error); throw new ApplicationException("Retrieving stats failed."); } var stats = new NumbersTermStatsResult { Total = response.Total, Start = utcStart.SafeAdd(displayTimeOffset.Value), End = utcEnd.SafeAdd(displayTimeOffset.Value), Numbers = GetNumbers(response.Aggs, fields) }; var terms = response.Aggs.Terms("terms"); if (terms != null) { stats.Terms.AddRange(terms.Items.Select(i => { var item = new NumbersTermStatsItem { Total = i.DocCount, Term = i.Key, Numbers = GetNumbers(i, fields) }; var termFirstOccurrence = i.Min("first_occurrence"); if (termFirstOccurrence?.Value != null) item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var termLastOccurrence = i.Max("last_occurrence"); if (termLastOccurrence?.Value != null) item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return item; })); } return stats; }
public async Task<NumbersTimelineStatsResult> GetNumbersTimelineStatsAsync(IEnumerable<FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext(); utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var response = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery<PersistentEvent>(filter)) .Aggregations(agg => BuildAggregations(agg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) .Aggregations(agg2 => BuildAggregations(agg2, fields)) ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)), fields) ) ).AnyContext(); if (!response.IsValid) { _logger.Error("Retrieving stats failed: {0}", response.ServerError.Error); throw new ApplicationException("Retrieving stats failed."); } var stats = new NumbersTimelineStatsResult { Total = response.Total, Numbers = GetNumbers(response.Aggs, fields) }; var timeline = response.Aggs.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => new NumbersTimelineItem { Date = i.Date, Total = i.DocCount, Numbers = GetNumbers(i, fields) })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Total / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = response.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = response.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }