public static TimeInfo ApplyMinimumUtcStartDate(this TimeInfo timeInfo, DateTime minimumUtcStartDate)
{
if (timeInfo.UtcRange.UtcStart < minimumUtcStartDate)
timeInfo.UtcRange = new DateTimeRange(minimumUtcStartDate.SafeAdd(timeInfo.Offset), timeInfo.UtcRange.End);
return timeInfo;
}
protected DateTimeRange FromRelationAmountTime(string relation, int amount, string size, DateTime now) {
relation = relation.ToLower();
size = size.ToLower();
if (amount < 1)
throw new ArgumentException("Time amount can't be 0.");
TimeSpan intervalSpan = Helper.GetTimeSpanFromName(size);
if (intervalSpan != TimeSpan.Zero) {
var totalSpan = TimeSpan.FromTicks(intervalSpan.Ticks * amount);
switch (relation) {
case "last":
case "past":
case "previous":
return new DateTimeRange(now.Floor(intervalSpan).SafeSubtract(totalSpan), now);
case "this":
case "next":
return new DateTimeRange(now, now.SafeAdd(totalSpan).Ceiling(intervalSpan).SubtractMilliseconds(1));
}
} else if (size == "week" || size == "weeks") {
switch (relation) {
case "last":
case "past":
case "previous":
return new DateTimeRange(now.SubtractWeeks(amount).StartOfDay(), now);
case "this":
case "next":
return new DateTimeRange(now, now.AddWeeks(amount).EndOfDay());
}
} else if (size == "month" || size == "months") {
switch (relation) {
case "last":
case "past":
case "previous":
return new DateTimeRange(now.SubtractMonths(amount).StartOfDay(), now);
case "this":
case "next":
return new DateTimeRange(now, now.AddMonths(amount).EndOfDay());
}
} else if (size == "year" || size == "years") {
switch (relation) {
case "last":
case "past":
case "previous":
return new DateTimeRange(now.SubtractYears(amount).StartOfDay(), now);
case "this":
case "next":
return new DateTimeRange(now, now.AddYears(amount).EndOfDay());
}
}
return null;
}
protected DateTime? FromRelationAmountTime(string relation, int amount, string size, DateTime now, bool isUpperLimit) {
relation = relation.ToLower();
size = size.ToLower();
if (amount < 1)
throw new ArgumentException("Time amount can't be 0.");
TimeSpan intervalSpan = Helper.GetTimeSpanFromName(size);
if (intervalSpan != TimeSpan.Zero) {
var totalSpan = TimeSpan.FromTicks(intervalSpan.Ticks * amount);
switch (relation) {
case "ago":
return isUpperLimit ? now.SafeSubtract(totalSpan).Ceiling(intervalSpan).SubtractMilliseconds(1) : now.SafeSubtract(totalSpan).Floor(intervalSpan);
case "from now":
return isUpperLimit ? now.SafeAdd(totalSpan).Ceiling(intervalSpan).SubtractMilliseconds(1) : now.SafeAdd(totalSpan).Floor(intervalSpan);
}
} else if (size == "week" || size == "weeks") {
switch (relation) {
case "ago":
return isUpperLimit ? now.SubtractWeeks(amount).EndOfDay() : now.SubtractWeeks(amount).StartOfDay();
case "from now":
return isUpperLimit ? now.AddWeeks(amount).EndOfDay() : now.AddWeeks(amount).StartOfDay();
}
} else if (size == "month" || size == "months") {
switch (relation) {
case "ago":
return isUpperLimit ? now.SubtractMonths(amount).EndOfDay() : now.SubtractMonths(amount).StartOfDay();
case "from now":
return isUpperLimit ? now.AddMonths(amount).EndOfDay() : now.AddMonths(amount).StartOfDay();
}
} else if (size == "year" || size == "years") {
switch (relation) {
case "ago":
return isUpperLimit ? now.SubtractYears(amount).EndOfDay() : now.SubtractYears(amount).StartOfDay();
case "from now":
return isUpperLimit ? now.AddYears(amount).EndOfDay() : now.AddYears(amount).StartOfDay();
}
}
return null;
}
public async Task<NumbersStatsResult> GetNumbersStatsAsync(IEnumerable<FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date)
.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var response = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(_queryBuilder.BuildQuery<PersistentEvent>(filter))
.Aggregations(agg => BuildAggregations(agg, fields))
).AnyContext();
if (!response.IsValid) {
_logger.Error("Retrieving stats failed: {0}", response.ServerError.Error);
throw new ApplicationException("Retrieving stats failed.");
}
return new NumbersStatsResult {
Total = response.Total,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value),
Numbers = GetNumbers(response.Aggs, fields)
};
}
public async Task<EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
throw new ArgumentException("Must be a valid term.", nameof(term));
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date)
.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate) {
// TODO: Cache this to save an extra search request when a date range isn't filtered.
var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(d => _queryBuilder.BuildQuery<PersistentEvent>(filter))
.SortAscending(ev => ev.Date)
.Take(1)).AnyContext();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.DateRanges.Clear();
filter.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date);
filter.Indices.Clear();
filter.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
}
}
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(_queryBuilder.BuildQuery<PersistentEvent>(filter))
.Aggregations(agg => agg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
).AnyContext();
if (!res.IsValid) {
Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
var newTerms = res.Aggs.Terms("new");
var stats = new EventTermStatsResult {
Total = res.Total,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value)
};
var unique = res.Aggs.Cardinality("unique");
if (unique?.Value != null)
stats.Unique = (long)unique.Value;
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var terms = res.Aggs.Terms("terms");
if (terms == null)
return stats;
stats.Terms.AddRange(terms.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique");
if (timelineUnique?.Value != null)
count = (long)timelineUnique.Value;
var termNew = i.Terms("new");
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0
};
var termFirstOccurrence = i.Min("first_occurrence");
if (termFirstOccurrence?.Value != null)
item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var termLastOccurrence = i.Max("last_occurrence");
if (termLastOccurrence?.Value != null)
item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var timeLine = i.DateHistogram("timelime");
if (timeLine != null) {
item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
}
return item;
}));
return stats;
}
public async Task<EventStatsResult> GetOccurrenceStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticSearchOptions<PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.UseStartDate) {
// TODO: Cache this to save an extra search request when a date range isn't filtered.
_elasticClient.EnableTrace();
var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Filter(d => filter.GetElasticSearchFilter())
.SortAscending(ev => ev.Date)
.Take(1)).AnyContext();
_elasticClient.DisableTrace();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
_elasticClient.EnableTrace();
var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Query(filter.GetElasticSearchQuery())
.Aggregations(agg => agg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
).AnyContext();
_elasticClient.DisableTrace();
if (!res.IsValid) {
Logger.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
var newTerms = res.Aggs.Terms("new");
var stats = new EventStatsResult {
Total = res.Total,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0
};
var unique = res.Aggs.Cardinality("unique");
if (unique?.Value != null)
stats.Unique = (long)unique.Value;
var timeline = res.Aggs.DateHistogram("timelime");
if (timeline != null) {
stats.Timeline.AddRange(timeline.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique");
if (timelineUnique?.Value != null)
count = (long)timelineUnique.Value;
var timelineNew = i.Terms("tl_new");
return new TimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0
};
}));
}
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
var totalHours = stats.End.Subtract(stats.Start).TotalHours;
if (totalHours > 0.0)
stats.AvgPerHour = stats.Total / totalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return stats;
}
public async Task<EventStatsResult> GetOccurrenceStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date)
.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(_queryBuilder.BuildQuery<PersistentEvent>(filter))
.Aggregations(agg => agg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
).AnyContext();
if (!res.IsValid) {
Logger.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
var newTerms = res.Aggs.Terms("new");
var stats = new EventStatsResult {
Total = res.Total,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0
};
var unique = res.Aggs.Cardinality("unique");
if (unique?.Value != null)
stats.Unique = (long)unique.Value;
var timeline = res.Aggs.DateHistogram("timelime");
if (timeline != null) {
stats.Timeline.AddRange(timeline.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique");
if (timelineUnique?.Value != null)
count = (long)timelineUnique.Value;
var timelineNew = i.Terms("tl_new");
return new EventTimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0
};
}));
}
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
var totalHours = stats.End.Subtract(stats.Start).TotalHours;
if (totalHours > 0.0)
stats.AvgPerHour = stats.Total / totalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return stats;
}
public async Task<SessionStatsResult> GetSessionStatsAsync(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithFieldEquals(EventIndex.Fields.PersistentEvent.Type, Event.KnownTypes.Session)
.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date)
.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(_queryBuilder.BuildQuery<PersistentEvent>(filter))
.Aggregations(agg => agg
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
.Aggregations(agg2 => agg2
.Average("tl_avg_duration", u => u.Field(ev => ev.Value))
.Cardinality("tl_users", u => u
.Field(EventIndex.Fields.PersistentEvent.UserRaw)
.PrecisionThreshold(100)
)
)
)
.Average("avg_duration", u => u.Field(ev => ev.Value))
.Cardinality("users", u => u
.Field(EventIndex.Fields.PersistentEvent.UserRaw)
.PrecisionThreshold(100)
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
).AnyContext();
if (!res.IsValid) {
Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
var stats = new SessionStatsResult { Sessions = res.Total };
var averageDuration = res.Aggs.Average("avg_duration");
if (averageDuration?.Value != null)
stats.AvgDuration = (decimal)averageDuration.Value;
var users = res.Aggs.Cardinality("users");
if (users?.Value != null)
stats.Users = (long)users.Value;
var timeline = res.Aggs.DateHistogram("timelime");
if (timeline != null) {
stats.Timeline.AddRange(timeline.Items.Select(i => {
var item = new SessionTimelineItem {
Date = i.Date,
Sessions = i.DocCount
};
var timelineAverageDuration = i.Average("tl_avg_duration");
if (timelineAverageDuration?.Value != null)
item.AvgDuration = (decimal)timelineAverageDuration.Value;
var timelineUsers = i.Cardinality("tl_users");
if (timelineUsers?.Value != null)
item.Users = (long)timelineUsers.Value;
return item;
}));
}
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
var totalHours = stats.End.Subtract(stats.Start).TotalHours;
if (totalHours > 0.0)
stats.AvgPerHour = stats.Sessions / totalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return stats;
}
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
throw new ArgumentException("Must be a valid term.", "term");
var filter = new ElasticSearchOptions<PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange();
_client.EnableTrace();
// if no start date then figure out first event date
if (!filter.UseStartDate) {
var result = _client.Search<PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1));
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange();
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*"))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => filter.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
)
)
)
);
if (!res.IsValid) {
Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
_client.DisableTrace();
var filtered = res.Aggs.Filter("filtered");
if (filtered == null)
return new EventTermStatsResult();
var stats = new EventTermStatsResult {
Total = filtered.DocCount
};
stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique").Value;
if (timelineUnique.HasValue)
count = (long)timelineUnique.Value;
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
};
var firstOccurrence = i.Min("first_occurrence").Value;
var lastOccurrence = i.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
if (lastOccurrence.HasValue)
item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
return item;
}));
stats.Start = utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
return stats;
}
public async Task<NumbersTermStatsResult> GetNumbersTermsStatsAsync(string term, IEnumerable<FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25) {
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
throw new ArgumentException("Must be a valid term.", nameof(term));
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date)
.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var response = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(_queryBuilder.BuildQuery<PersistentEvent>(filter))
.Aggregations(agg => BuildAggregations(agg
.Terms("terms", t => BuildTermSort(t
.Field(term)
.Size(max)
.Aggregations(agg2 => BuildAggregations(agg2
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date)), fields)
), fields)
), fields)
)
).AnyContext();
if (!response.IsValid) {
_logger.Error("Retrieving stats failed: {0}", response.ServerError.Error);
throw new ApplicationException("Retrieving stats failed.");
}
var stats = new NumbersTermStatsResult {
Total = response.Total,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value),
Numbers = GetNumbers(response.Aggs, fields)
};
var terms = response.Aggs.Terms("terms");
if (terms != null) {
stats.Terms.AddRange(terms.Items.Select(i => {
var item = new NumbersTermStatsItem {
Total = i.DocCount,
Term = i.Key,
Numbers = GetNumbers(i, fields)
};
var termFirstOccurrence = i.Min("first_occurrence");
if (termFirstOccurrence?.Value != null)
item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var termLastOccurrence = i.Max("last_occurrence");
if (termLastOccurrence?.Value != null)
item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return item;
}));
}
return stats;
}
public async Task<NumbersTimelineStatsResult> GetNumbersTimelineStatsAsync(IEnumerable<FieldAggregation> fields, DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticQuery()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithDateRange(utcStart, utcEnd, EventIndex.Fields.PersistentEvent.Date)
.WithIndices(utcStart, utcEnd, $"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.DateRanges.First().UseStartDate)
await UpdateFilterStartDateRangesAsync(filter, utcEnd).AnyContext();
utcStart = filter.DateRanges.First().GetStartDate();
utcEnd = filter.DateRanges.First().GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var response = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.AliasName)
.Query(_queryBuilder.BuildQuery<PersistentEvent>(filter))
.Aggregations(agg => BuildAggregations(agg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
.Aggregations(agg2 => BuildAggregations(agg2, fields))
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date)), fields)
)
).AnyContext();
if (!response.IsValid) {
_logger.Error("Retrieving stats failed: {0}", response.ServerError.Error);
throw new ApplicationException("Retrieving stats failed.");
}
var stats = new NumbersTimelineStatsResult { Total = response.Total, Numbers = GetNumbers(response.Aggs, fields) };
var timeline = response.Aggs.DateHistogram("timelime");
if (timeline != null) {
stats.Timeline.AddRange(timeline.Items.Select(i => new NumbersTimelineItem {
Date = i.Date,
Total = i.DocCount,
Numbers = GetNumbers(i, fields)
}));
}
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
var totalHours = stats.End.Subtract(stats.Start).TotalHours;
if (totalHours > 0.0)
stats.AvgPerHour = stats.Total / totalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = response.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = response.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return stats;
}
