public async Task<EventStatsResult> GetOccurrenceStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s .IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(d => _queryBuilder.BuildQuery<PersistentEvent>(filter)) .SortAscending(ev => ev.Date) .Take(1)).AnyContext(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.DateRanges.Clear(); filter.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date); filter.Indices.Clear(); filter.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); } } utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery<PersistentEvent>(filter)) .Aggregations(agg => agg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ).AnyContext(); if (!res.IsValid) { Logger.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) stats.Unique = (long)unique.Value; var timeline = res.Aggs.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique?.Value != null) count = (long)timelineUnique.Value; var timelineNew = i.Terms("tl_new"); return new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }; })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Total / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticSearchOptions<PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(String.Format("'{0}-'yyyyMM", _eventIndex.VersionedName)); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = _elasticClient.Search<PersistentEvent>(s => s .IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Filter(d => filter.GetElasticSearchFilter()) .SortAscending(ev => ev.Date) .Take(1)); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(String.Format("'{0}-'yyyyMM", _eventIndex.VersionedName)); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = _elasticClient.Search<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ) ) ); _elasticClient.DisableTrace(); if (!res.IsValid) { Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var filtered = res.Aggs.Filter("filtered"); if (filtered == null) return new EventStatsResult(); var newTerms = filtered.Terms("new"); var stats = new EventStatsResult { Total = filtered.DocCount, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = filtered.Cardinality("unique"); if (unique != null && unique.Value.HasValue) stats.Unique = (long)unique.Value; var timeline = filtered.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique != null && timelineUnique.Value.HasValue) count = (long)timelineUnique.Value; var timelineNew = i.Terms("tl_new"); return new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }; })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Total / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = filtered.Min("first_occurrence"); if (firstOccurrence != null && firstOccurrence.Value.HasValue) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = filtered.Max("last_occurrence"); if (lastOccurrence != null && lastOccurrence.Value.HasValue) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }