public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string query = null, TimeSpan?utcOffset = null, int desiredDataPoints = 100) { if (!utcOffset.HasValue) { utcOffset = TimeSpan.Zero; } var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange(); _client.EnableTrace(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(String.Join(",", options.Indices)) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => options.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(utcOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } _client.DisableTrace(); var stats = new EventStatsResult { Total = res.Aggs.Filter("filtered").DocCount, New = res.Aggs.Filter("filtered").Terms("new").Items.Count > 0 ? res.Aggs.Filter("filtered").Terms("new").Items[0].DocCount : 0 }; var unique = res.Aggs.Filter("filtered").Cardinality("unique").Value; if (unique.HasValue) { stats.Unique = (long)unique.Value; } stats.Timeline.AddRange(res.Aggs.Filter("filtered").DateHistogram("timelime").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique").Value; if (timelineUnique.HasValue) { count = (long)timelineUnique.Value; } return(new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0 }); })); stats.Start = utcStart.Add(utcOffset.Value); stats.End = utcEnd.Add(utcOffset.Value); stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours; if (stats.Timeline.Count <= 0) { return(stats); } var firstOccurrence = res.Aggs.Filter("filtered").Min("first_occurrence").Value; var lastOccurrence = res.Aggs.Filter("filtered").Max("last_occurrence").Value; if (firstOccurrence.HasValue) { stats.FirstOccurrence = firstOccurrence.Value.ToDateTime(); } if (lastOccurrence.HasValue) { stats.LastOccurrence = lastOccurrence.Value.ToDateTime(); } return(stats); }
public async Task <EventStatsResult> GetOccurrenceStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) { displayTimeOffset = TimeSpan.Zero; } var filter = new ElasticQuery() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date) .WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.DateRanges.First().UseStartDate) { await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext(); } utcStart = filter.DateRanges.First().GetStartDate(); utcEnd = filter.DateRanges.First().GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = await _elasticClient.SearchAsync <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName) .Query(_queryBuilder.BuildQuery <PersistentEvent>(filter)) .Aggregations(agg => agg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ).AnyContext(); if (!res.IsValid) { Logger.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) { stats.Unique = (long)unique.Value; } var timeline = res.Aggs.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique?.Value != null) { count = (long)timelineUnique.Value; } var timelineNew = i.Terms("tl_new"); return(new EventTimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }); })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) { stats.AvgPerHour = stats.Total / totalHours; } if (stats.Timeline.Count <= 0) { return(stats); } var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) { stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) { stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } return(stats); }
public async Task <EventStatsResult> GetOccurrenceStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) { displayTimeOffset = TimeSpan.Zero; } var filter = new ElasticSearchOptions <PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = await _elasticClient.SearchAsync <PersistentEvent>(s => s .IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Filter(d => filter.GetElasticSearchFilter()) .SortAscending(ev => ev.Date) .Take(1)).AnyContext(); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = await _elasticClient.SearchAsync <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Query(filter.GetElasticSearchQuery()) .Aggregations(agg => agg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ).AnyContext(); _elasticClient.DisableTrace(); if (!res.IsValid) { Logger.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) { stats.Unique = (long)unique.Value; } var timeline = res.Aggs.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique?.Value != null) { count = (long)timelineUnique.Value; } var timelineNew = i.Terms("tl_new"); return(new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }); })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) { stats.AvgPerHour = stats.Total / totalHours; } if (stats.Timeline.Count <= 0) { return(stats); } var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) { stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) { stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } return(stats); }
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) { displayTimeOffset = TimeSpan.Zero; } var filter = new ElasticSearchOptions <PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(); _client.EnableTrace(); // if no start date then figure out first event date if (!filter.UseStartDate) { var result = _client.Search <PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1)); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } _client.DisableTrace(); var filtered = res.Aggs.Filter("filtered"); if (filtered == null) { return(new EventStatsResult()); } var stats = new EventStatsResult { Total = filtered.DocCount, New = filtered.Terms("new").Items.Count > 0 ? filtered.Terms("new").Items[0].DocCount : 0 }; var unique = filtered.Cardinality("unique").Value; if (unique.HasValue) { stats.Unique = (long)unique.Value; } stats.Timeline.AddRange(filtered.DateHistogram("timelime").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique").Value; if (timelineUnique.HasValue) { count = (long)timelineUnique.Value; } return(new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0 }); })); stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours; if (stats.Timeline.Count <= 0) { return(stats); } var firstOccurrence = filtered.Min("first_occurrence").Value; var lastOccurrence = filtered.Max("last_occurrence").Value; if (firstOccurrence.HasValue) { stats.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } if (lastOccurrence.HasValue) { stats.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } return(stats); }