public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticSearchOptions<PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange();
_client.EnableTrace();
// if no start date then figure out first event date
if (!filter.UseStartDate) {
var result = _client.Search<PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1));
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange();
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*"))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => filter.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
)
)
);
if (!res.IsValid) {
Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
_client.DisableTrace();
var filtered = res.Aggs.Filter("filtered");
if (filtered == null)
return new EventStatsResult();
var stats = new EventStatsResult {
Total = filtered.DocCount,
New = filtered.Terms("new").Items.Count > 0 ? filtered.Terms("new").Items[0].DocCount : 0
};
var unique = filtered.Cardinality("unique").Value;
if (unique.HasValue)
stats.Unique = (long)unique.Value;
stats.Timeline.AddRange(filtered.DateHistogram("timelime").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique").Value;
if (timelineUnique.HasValue)
count = (long)timelineUnique.Value;
return new TimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0
};
}));
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = filtered.Min("first_occurrence").Value;
var lastOccurrence = filtered.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
stats.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
if (lastOccurrence.HasValue)
stats.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return stats;
}
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string query = null, TimeSpan? utcOffset = null, int desiredDataPoints = 100)
{
if (!utcOffset.HasValue)
utcOffset = TimeSpan.Zero;
var options = new ElasticSearchOptions<PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange();
_client.EnableTrace();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(String.Join(",", options.Indices))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => options.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
.TimeZone(HoursAndMinutes(utcOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
)
)
);
if (!res.IsValid) {
Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
_client.DisableTrace();
var stats = new EventStatsResult {
Total = res.Aggs.Filter("filtered").DocCount,
New = res.Aggs.Filter("filtered").Terms("new").Items.Count > 0 ? res.Aggs.Filter("filtered").Terms("new").Items[0].DocCount : 0
};
var unique = res.Aggs.Filter("filtered").Cardinality("unique").Value;
if (unique.HasValue)
stats.Unique = (long)unique.Value;
stats.Timeline.AddRange(res.Aggs.Filter("filtered").DateHistogram("timelime").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique").Value;
if (timelineUnique.HasValue)
count = (long)timelineUnique.Value;
return new TimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0
};
}));
stats.Start = utcStart.Add(utcOffset.Value);
stats.End = utcEnd.Add(utcOffset.Value);
stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = res.Aggs.Filter("filtered").Min("first_occurrence").Value;
var lastOccurrence = res.Aggs.Filter("filtered").Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
stats.FirstOccurrence = firstOccurrence.Value.ToDateTime();
if (lastOccurrence.HasValue)
stats.LastOccurrence = lastOccurrence.Value.ToDateTime();
return stats;
}
