public async Task<EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
throw new ArgumentException("Must be a valid term.", nameof(term));
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date)
.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate) {
// TODO: Cache this to save an extra search request when a date range isn't filtered.
var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(d => _queryBuilder.BuildQuery<PersistentEvent>(filter))
.SortAscending(ev => ev.Date)
.Take(1)).AnyContext();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.DateRanges.Clear();
filter.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date);
filter.Indices.Clear();
filter.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
}
}
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(_queryBuilder.BuildQuery<PersistentEvent>(filter))
.Aggregations(agg => agg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
).AnyContext();
if (!res.IsValid) {
Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
var newTerms = res.Aggs.Terms("new");
var stats = new EventTermStatsResult {
Total = res.Total,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value)
};
var unique = res.Aggs.Cardinality("unique");
if (unique?.Value != null)
stats.Unique = (long)unique.Value;
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var terms = res.Aggs.Terms("terms");
if (terms == null)
return stats;
stats.Terms.AddRange(terms.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique");
if (timelineUnique?.Value != null)
count = (long)timelineUnique.Value;
var termNew = i.Terms("new");
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0
};
var termFirstOccurrence = i.Min("first_occurrence");
if (termFirstOccurrence?.Value != null)
item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var termLastOccurrence = i.Max("last_occurrence");
if (termLastOccurrence?.Value != null)
item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var timeLine = i.DateHistogram("timelime");
if (timeLine != null) {
item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
}
return item;
}));
return stats;
}
private async Task UpdateFilterStartDateRangesAsync(ElasticQuery filter, DateTime utcEnd) {
// TODO: Cache this to save an extra search request when a date range isn't filtered.
var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(d => _queryBuilder.BuildQuery<PersistentEvent>(filter))
.SortAscending(ev => ev.Date)
.Take(1)).AnyContext();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
filter.DateRanges.Clear();
filter.WithDateRange(firstEvent.Source.Date.UtcDateTime, utcEnd, EventIndex.Fields.PersistentEvent.Date);
filter.Indices.Clear();
filter.WithIndices(firstEvent.Source.Date.UtcDateTime, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
}
}