public HangingRequestsHandler(
ElasticSearchClient <IndexRequestDbo> elasticClient,
ElasticSearchOptions elasticOptions)
{
_elasticClient = elasticClient;
_elasticOptions = elasticOptions;
}
public static IWebHostBuilder UseCkoSerilog <T>(this IWebHostBuilder builder)
{
//A note about logging:
//ILogger<Startup> lives in Microsoft.Extensions.Logging
//Whereas Log.Logger lives in Serilog.ILogger
//Hence also the reason for the AddSingleton(Log.Logger) in ConfigureServices.
//However, both log to the same Serilog sinks as configured in Program.cs
var elasticSearchOptions = new ElasticSearchOptions();
//This is the preferred way of configuring Serilog. Try not to do in in Startup, this has
//some disadvantages: https://github.com/serilog/serilog-aspnetcore
//UseSerilog configures DI and sets Log.Logger
builder.UseSerilog((hostingContext, loggerConfiguration) => {
hostingContext.Configuration.GetSection("ElasticSearch").Bind(elasticSearchOptions);
loggerConfiguration.ConfigureCkoLogger <T>(hostingContext.Configuration, hostingContext.HostingEnvironment.EnvironmentName, elasticSearchOptions);
});
//Despite .UseSerilog() we still need to inject Serilog.ILogger
//Some of our classes depend on Serilog.ILogger
//Others on Microsoft.Extensions.Logging.ILogger
builder.ConfigureServices(collection =>
collection.AddSingleton <ILogger>(Log.Logger)
);
return(builder);
}
public QueueForIndex(ElasticSearchClient <IndexRequestDbo> client, ElasticSearchOptions options)
{
_client = client;
_options = options;
EnsureIndexInElasticCreated();
}
public List <string> ExistsByStackIds(string[] stackIds)
{
var options = new ElasticSearchOptions <PersistentEvent>().WithStackIds(stackIds);
var descriptor = new SearchDescriptor <PersistentEvent>().Filter(options.GetElasticSearchFilter()).Source(s => s.Include("stack_id")).Size(stackIds.Length);
var results = _elasticClient.Search <PersistentEvent>(descriptor);
return(results.Documents.Select(e => e.StackId).ToList());
}
public Reindexer(
ElasticSearchClient <IndexRequestDbo> requestsClient,
ElasticSearchOptions options,
QueueForIndex queueForIndex)
{
_requestsClient = requestsClient;
_options = options;
_queueForIndex = queueForIndex;
}
public static void AddElasticSearch(this IServiceCollection services, Action <ConnectionSettings> configureElasticSearch)
{
services.AddSingleton(cp => {
ElasticSearchOptions options = cp.GetRequiredService <IOptionsSnapshot <ElasticSearchOptions> >().Value;
ConnectionSettings connectionSettings =
new ConnectionSettings(new StaticConnectionPool(options.Uris));
configureElasticSearch(connectionSettings);
return(connectionSettings);
});
services.AddScoped(c => new ElasticClient(c.GetService <ConnectionSettings>()));
}
private void AddElastic(IServiceCollection services)
{
services.AddScoped <IElasticSearchService, ElasticSearchService>();
services.AddScoped <ElasticClient>(provider =>
{
var elasticOptions = new ElasticSearchOptions();
Configuration.Bind("ElasticSearchOptions", elasticOptions);
var uris = elasticOptions.HostUrls.Split(",").Select(u => new Uri(u)).ToArray();
var connectionPool = new SniffingConnectionPool(uris);
var settings = new ConnectionSettings(connectionPool);
var client = new ElasticClient(settings);
return(client);
});
}
public Indexer(
ElasticSearchClient <Document> client,
ElasticSearchOptions options,
QueueForIndex indexRequestsQueue,
IHttpClientFactory httpClientFactory,
SiteMapGetter siteMapGetter,
PagesPerSiteLimiter pagesPerSiteLimiter)
{
_client = client;
_options = options;
this.indexRequestsQueue = indexRequestsQueue;
this.siteMapGetter = siteMapGetter;
this.pagesPerSiteLimiter = pagesPerSiteLimiter;
httpClient = httpClientFactory.CreateClient("Page downloader");
}
public static void AddElasticsearch(this IServiceCollection services, Action <ElasticSearchOptions> action = null)
{
ElasticSearchOptions options = new ElasticSearchOptions();
action?.Invoke(options);
services.AddSingleton(options);
services.AddSingleton <IEsClientProvider, EsClientProvider>();
services.AddSingleton <IElasticSearchTransPortService, ElasticSearchTransPortService>();
TransPortServiceDependencyInjection.AddFunc((serviceProvider, name) =>
{
if (name.Equals(ElasticsearchConstant.ELASTICSEARCHNAME, StringComparison.OrdinalIgnoreCase))
{
return(serviceProvider.GetService <IElasticSearchTransPortService>());
}
return(null);
});
}
public static string Format(BooleanExpression booleanExpression, ElasticSearchOptions elasticSearchOptions)
{
elasticSearchOptions ??= new ElasticSearchOptions();
var visitor = new ElasticSearchVisitor(elasticSearchOptions);
visitor.Visit(booleanExpression);
var filter = visitor.GetBool();
while (BoolOptimizer.Optimize(filter) || RangeOptimizer.OptimizeRange(filter))
{
//Iterate until no more optimizations
}
var serializedFilter = JsonSerializer.Serialize(filter, new JsonSerializerOptions()
{
Converters =
{
new BoolSerializer()
}
});
return(serializedFilter);
}
public ElasticSearchVisitor(ElasticSearchOptions elasticSearchOptions)
{
_elasticSearchOptions = elasticSearchOptions;
}
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string query = null, TimeSpan? utcOffset = null, int max = 25, int desiredDataPoints = 10)
{
if (!utcOffset.HasValue)
utcOffset = TimeSpan.Zero;
var allowedTerms = new[] { "tags", "stack_id", "organization_id" };
if (!allowedTerms.Contains(term))
throw new ArgumentException("Must be a valid term.", "term");
var options = new ElasticSearchOptions<PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange();
_client.EnableTrace();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(String.Join(",", options.Indices))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => options.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(utcOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
)
)
)
);
if (!res.IsValid) {
Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
_client.DisableTrace();
var stats = new EventTermStatsResult {
Total = res.Aggs.Filter("filtered").DocCount,
};
stats.Terms.AddRange(res.Aggs.Filter("filtered").DateHistogram("terms").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique").Value;
if (timelineUnique.HasValue)
count = (long)timelineUnique.Value;
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
};
var firstOccurrence = i.Min("first_occurrence").Value;
var lastOccurrence = i.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
item.FirstOccurrence = firstOccurrence.Value.ToDateTime();
if (lastOccurrence.HasValue)
item.LastOccurrence = lastOccurrence.Value.ToDateTime();
item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
return item;
}));
stats.Start = utcStart.Add(utcOffset.Value);
stats.End = utcEnd.Add(utcOffset.Value);
return stats;
}
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string query = null, TimeSpan? utcOffset = null, int desiredDataPoints = 100)
{
if (!utcOffset.HasValue)
utcOffset = TimeSpan.Zero;
var options = new ElasticSearchOptions<PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange();
_client.EnableTrace();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(String.Join(",", options.Indices))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => options.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
.TimeZone(HoursAndMinutes(utcOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
)
)
);
if (!res.IsValid) {
Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
_client.DisableTrace();
var stats = new EventStatsResult {
Total = res.Aggs.Filter("filtered").DocCount,
New = res.Aggs.Filter("filtered").Terms("new").Items.Count > 0 ? res.Aggs.Filter("filtered").Terms("new").Items[0].DocCount : 0
};
var unique = res.Aggs.Filter("filtered").Cardinality("unique").Value;
if (unique.HasValue)
stats.Unique = (long)unique.Value;
stats.Timeline.AddRange(res.Aggs.Filter("filtered").DateHistogram("timelime").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique").Value;
if (timelineUnique.HasValue)
count = (long)timelineUnique.Value;
return new TimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0
};
}));
stats.Start = utcStart.Add(utcOffset.Value);
stats.End = utcEnd.Add(utcOffset.Value);
stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = res.Aggs.Filter("filtered").Min("first_occurrence").Value;
var lastOccurrence = res.Aggs.Filter("filtered").Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
stats.FirstOccurrence = firstOccurrence.Value.ToDateTime();
if (lastOccurrence.HasValue)
stats.LastOccurrence = lastOccurrence.Value.ToDateTime();
return stats;
}
public List<string> ExistsByStackIds(string[] stackIds) {
var options = new ElasticSearchOptions<PersistentEvent>().WithStackIds(stackIds);
var descriptor = new SearchDescriptor<PersistentEvent>().Filter(options.GetElasticSearchFilter()).Source(s => s.Include("stack_id")).Size(stackIds.Length);
var results = _elasticClient.Search<PersistentEvent>(descriptor);
return results.Documents.Select(e => e.StackId).ToList();
}
public static IServiceCollection AddElasticSearch(this IServiceCollection services, Action <ElasticSearchOptions> options)
{
var opt = new ElasticSearchOptions();
options.Invoke(opt);
if (opt.Active)
{
var conn_settings = new ConnectionSettings(new Uri(opt.Url))
.DefaultIndex(opt.DefaultIndex)
.DefaultMappingFor <Person>(p_desc => p_desc
.IndexName("person")
.Ignore(p => p.Artists)
)
.DefaultMappingFor <Artist>(a_desc => a_desc
.IndexName("artist")
.Ignore(a => a.CurrentMembers)
.Ignore(a => a.PastMembers)
.Ignore(a => a.Songs)
)
.DefaultMappingFor <Song>(s_desc => s_desc
.IndexName("song")
.Ignore(s => s.Artists)
.Ignore(s => s.Lyrics)
);
var client = new ElasticClient(conn_settings);
var response_people = client.Indices.Create("person", index => index
.Map <Person>(map => map
.Properties(props => props
.Text(desc => desc.Name(person => person.FirstName))
.Text(desc => desc.Name(person => person.LastName))
.Date(desc => desc.Name(person => person.Born))
.Date(desc => desc.Name(person => person.Died))
.Completion(desc => desc.Name(p => p.NameSuggest))
)
)
);
var response_artists = client.Indices.Create("artist", index => index
.Map <Artist>(map => map
.Properties(props => props
.Text(desc => desc.Name(artist => artist.Name))
.Text(a_comp => a_comp.Name(a => a.Name))
.Number(desc => desc.Name(artist => artist.YearStarted))
.Number(desc => desc.Name(artist => artist.YearQuit))
.Completion(desc => desc.Name(a => a.NameSuggest))
)
)
);
var response_songs = client.Indices.Create("song", index => index
.Map <Song>(map => map
.Properties(props => props.Text(desc => desc.Name(song => song.Title))
.Completion(s_comp => s_comp.Name(s => s.TitleSuggest))
)
)
);
return(services.AddSingleton <IElasticClient>(client));
}
else
{
return(services.AddSingleton <IElasticClient>(new ElasticClient()));
}
}
public void OnGet()
{
ElsOptions = _elsOptions.Value;
}
public async Task<EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
throw new ArgumentException("Must be a valid term.", nameof(term));
var filter = new ElasticSearchOptions<PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.UseStartDate) {
// TODO: Cache this to save an extra search request when a date range isn't filtered.
_elasticClient.EnableTrace();
var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Filter(d => filter.GetElasticSearchFilter())
.SortAscending(ev => ev.Date)
.Take(1)).AnyContext();
_elasticClient.DisableTrace();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
_elasticClient.EnableTrace();
var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Query(filter.GetElasticSearchQuery())
.Aggregations(agg => agg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
).AnyContext();
_elasticClient.DisableTrace();
if (!res.IsValid) {
Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
var newTerms = res.Aggs.Terms("new");
var stats = new EventTermStatsResult {
Total = res.Total,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value)
};
var unique = res.Aggs.Cardinality("unique");
if (unique?.Value != null)
stats.Unique = (long)unique.Value;
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var terms = res.Aggs.Terms("terms");
if (terms == null)
return stats;
stats.Terms.AddRange(terms.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique");
if (timelineUnique?.Value != null)
count = (long)timelineUnique.Value;
var termNew = i.Terms("new");
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0
};
var termFirstOccurrence = i.Min("first_occurrence");
if (termFirstOccurrence?.Value != null)
item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var termLastOccurrence = i.Max("last_occurrence");
if (termLastOccurrence?.Value != null)
item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var timeLine = i.DateHistogram("timelime");
if (timeLine != null) {
item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
}
return item;
}));
return stats;
}
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10)
{
if (!displayTimeOffset.HasValue)
{
displayTimeOffset = TimeSpan.Zero;
}
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
{
throw new ArgumentException("Must be a valid term.", "term");
}
var filter = new ElasticSearchOptions <PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange();
_client.EnableTrace();
// if no start date then figure out first event date
if (!filter.UseStartDate)
{
var result = _client.Search <PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1));
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null)
{
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange();
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search <PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*"))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => filter.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
)
)
)
);
if (!res.IsValid)
{
Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
_client.DisableTrace();
var filtered = res.Aggs.Filter("filtered");
if (filtered == null)
{
return(new EventTermStatsResult());
}
var stats = new EventTermStatsResult {
Total = filtered.DocCount
};
stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique").Value;
if (timelineUnique.HasValue)
{
count = (long)timelineUnique.Value;
}
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
};
var firstOccurrence = i.Min("first_occurrence").Value;
var lastOccurrence = i.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
{
item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
if (lastOccurrence.HasValue)
{
item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
return(item);
}));
stats.Start = utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
return(stats);
}
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string query = null, TimeSpan?utcOffset = null, int desiredDataPoints = 100)
{
if (!utcOffset.HasValue)
{
utcOffset = TimeSpan.Zero;
}
var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange();
_client.EnableTrace();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search <PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(String.Join(",", options.Indices))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => options.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
.TimeZone(HoursAndMinutes(utcOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
)
)
);
if (!res.IsValid)
{
Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
_client.DisableTrace();
var stats = new EventStatsResult {
Total = res.Aggs.Filter("filtered").DocCount,
New = res.Aggs.Filter("filtered").Terms("new").Items.Count > 0 ? res.Aggs.Filter("filtered").Terms("new").Items[0].DocCount : 0
};
var unique = res.Aggs.Filter("filtered").Cardinality("unique").Value;
if (unique.HasValue)
{
stats.Unique = (long)unique.Value;
}
stats.Timeline.AddRange(res.Aggs.Filter("filtered").DateHistogram("timelime").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique").Value;
if (timelineUnique.HasValue)
{
count = (long)timelineUnique.Value;
}
return(new TimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0
});
}));
stats.Start = utcStart.Add(utcOffset.Value);
stats.End = utcEnd.Add(utcOffset.Value);
stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours;
if (stats.Timeline.Count <= 0)
{
return(stats);
}
var firstOccurrence = res.Aggs.Filter("filtered").Min("first_occurrence").Value;
var lastOccurrence = res.Aggs.Filter("filtered").Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
{
stats.FirstOccurrence = firstOccurrence.Value.ToDateTime();
}
if (lastOccurrence.HasValue)
{
stats.LastOccurrence = lastOccurrence.Value.ToDateTime();
}
return(stats);
}
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
throw new ArgumentException("Must be a valid term.", "term");
var filter = new ElasticSearchOptions<PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange();
_client.EnableTrace();
// if no start date then figure out first event date
if (!filter.UseStartDate) {
var result = _client.Search<PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1));
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange();
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*"))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => filter.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
)
)
)
);
if (!res.IsValid) {
Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
_client.DisableTrace();
var filtered = res.Aggs.Filter("filtered");
if (filtered == null)
return new EventTermStatsResult();
var stats = new EventTermStatsResult {
Total = filtered.DocCount
};
stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique").Value;
if (timelineUnique.HasValue)
count = (long)timelineUnique.Value;
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
};
var firstOccurrence = i.Min("first_occurrence").Value;
var lastOccurrence = i.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
if (lastOccurrence.HasValue)
item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
return item;
}));
stats.Start = utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
return stats;
}
public static LoggerConfiguration ConfigureCkoLogger <T>(this LoggerConfiguration loggerConfiguration, IConfiguration configuration, string envName, ElasticSearchOptions elasticSearchOptions)
{
loggerConfiguration
.ReadFrom.Configuration(configuration)
.Enrich.WithProperty("HostName", GetHostName())
.Enrich.WithProperty("Version", ReflectionUtils.GetAssemblyVersion <T>())
.Enrich.WithProperty("Environment", envName)
.WriteTo.Elasticsearch(
new ElasticsearchSinkOptions(elasticSearchOptions.Node)
{
AutoRegisterTemplate = elasticSearchOptions.AutoRegisterTemplate,
IndexFormat = elasticSearchOptions.IndexFormat,
MinimumLogEventLevel = Serilog.Events.LogEventLevel.Information
});
return(loggerConfiguration);
}
private static IServiceCollection AddElasticSearch(this IServiceCollection services, ElasticSearchOptions options)
{
var uri = new Uri(options.BootstrapServers);
var connectionSettings = new ConnectionSettings(uri);
connectionSettings.EnableDebugMode();
if (!string.IsNullOrWhiteSpace(options.DefaultIndex))
{
connectionSettings.DefaultIndex(options.DefaultIndex);
}
services.AddSingleton <IElasticClient>(new ElasticClient(connectionSettings));
return(services);
}
public async Task <EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10)
{
if (!displayTimeOffset.HasValue)
{
displayTimeOffset = TimeSpan.Zero;
}
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
{
throw new ArgumentException("Must be a valid term.", nameof(term));
}
var filter = new ElasticSearchOptions <PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.UseStartDate)
{
// TODO: Cache this to save an extra search request when a date range isn't filtered.
_elasticClient.EnableTrace();
var result = await _elasticClient.SearchAsync <PersistentEvent>(s => s
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Filter(d => filter.GetElasticSearchFilter())
.SortAscending(ev => ev.Date)
.Take(1)).AnyContext();
_elasticClient.DisableTrace();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null)
{
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
_elasticClient.EnableTrace();
var res = await _elasticClient.SearchAsync <PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Query(filter.GetElasticSearchQuery())
.Aggregations(agg => agg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
).AnyContext();
_elasticClient.DisableTrace();
if (!res.IsValid)
{
Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
var newTerms = res.Aggs.Terms("new");
var stats = new EventTermStatsResult {
Total = res.Total,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value)
};
var unique = res.Aggs.Cardinality("unique");
if (unique?.Value != null)
{
stats.Unique = (long)unique.Value;
}
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
{
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
{
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var terms = res.Aggs.Terms("terms");
if (terms == null)
{
return(stats);
}
stats.Terms.AddRange(terms.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique");
if (timelineUnique?.Value != null)
{
count = (long)timelineUnique.Value;
}
var termNew = i.Terms("new");
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0
};
var termFirstOccurrence = i.Min("first_occurrence");
if (termFirstOccurrence?.Value != null)
{
item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var termLastOccurrence = i.Max("last_occurrence");
if (termLastOccurrence?.Value != null)
{
item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var timeLine = i.DateHistogram("timelime");
if (timeLine != null)
{
item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
}
return(item);
}));
return(stats);
}
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticSearchOptions<PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange();
// if no start date then figure out first event date
if (!filter.UseStartDate) {
// TODO: Cache this to save an extra search request when a date range isn't filtered.
_elasticClient.EnableTrace();
var result = _elasticClient.Search<PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1));
_elasticClient.DisableTrace();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange();
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
_elasticClient.EnableTrace();
var res = _elasticClient.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*"))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => filter.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
)
)
);
_elasticClient.DisableTrace();
if (!res.IsValid) {
Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
var filtered = res.Aggs.Filter("filtered");
if (filtered == null)
return new EventStatsResult();
var newTerms = filtered.Terms("new");
var stats = new EventStatsResult {
Total = filtered.DocCount,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0
};
var unique = filtered.Cardinality("unique");
if (unique != null && unique.Value.HasValue)
stats.Unique = (long)unique.Value;
var timeline = filtered.DateHistogram("timelime");
if (timeline != null) {
stats.Timeline.AddRange(timeline.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique");
if (timelineUnique != null && timelineUnique.Value.HasValue)
count = (long)timelineUnique.Value;
var timelineNew = i.Terms("tl_new");
return new TimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0
};
}));
}
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
var totalHours = stats.End.Subtract(stats.Start).TotalHours;
if (totalHours > 0.0)
stats.AvgPerHour = stats.Total / totalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = filtered.Min("first_occurrence");
if (firstOccurrence != null && firstOccurrence.Value.HasValue)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = filtered.Max("last_occurrence");
if (lastOccurrence != null && lastOccurrence.Value.HasValue)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return stats;
}
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string query = null, TimeSpan?utcOffset = null, int max = 25, int desiredDataPoints = 10)
{
if (!utcOffset.HasValue)
{
utcOffset = TimeSpan.Zero;
}
var allowedTerms = new[] { "tags", "stack_id", "organization_id" };
if (!allowedTerms.Contains(term))
{
throw new ArgumentException("Must be a valid term.", "term");
}
var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange();
_client.EnableTrace();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search <PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(String.Join(",", options.Indices))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => options.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(utcOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
)
)
)
);
if (!res.IsValid)
{
Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
_client.DisableTrace();
var stats = new EventTermStatsResult {
Total = res.Aggs.Filter("filtered").DocCount,
};
stats.Terms.AddRange(res.Aggs.Filter("filtered").DateHistogram("terms").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique").Value;
if (timelineUnique.HasValue)
{
count = (long)timelineUnique.Value;
}
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
};
var firstOccurrence = i.Min("first_occurrence").Value;
var lastOccurrence = i.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
{
item.FirstOccurrence = firstOccurrence.Value.ToDateTime();
}
if (lastOccurrence.HasValue)
{
item.LastOccurrence = lastOccurrence.Value.ToDateTime();
}
item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
return(item);
}));
stats.Start = utcStart.Add(utcOffset.Value);
stats.End = utcEnd.Add(utcOffset.Value);
return(stats);
}
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) {
if (!displayTimeOffset.HasValue)
displayTimeOffset = TimeSpan.Zero;
var filter = new ElasticSearchOptions<PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange(String.Format("'{0}-'yyyyMM", _eventIndex.VersionedName));
// if no start date then figure out first event date
if (!filter.UseStartDate) {
// TODO: Cache this to save an extra search request when a date range isn't filtered.
_elasticClient.EnableTrace();
var result = _elasticClient.Search<PersistentEvent>(s => s
.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Filter(d => filter.GetElasticSearchFilter())
.SortAscending(ev => ev.Date)
.Take(1));
_elasticClient.DisableTrace();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null) {
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange(String.Format("'{0}-'yyyyMM", _eventIndex.VersionedName));
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
_elasticClient.EnableTrace();
var res = _elasticClient.Search<PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => filter.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.DateHistogram("timelime", t => t
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.Aggregations(agg2 => agg2
.Cardinality("tl_unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("tl_new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", t => t.Field(ev => ev.Date))
.Max("last_occurrence", t => t.Field(ev => ev.Date))
)
)
)
);
_elasticClient.DisableTrace();
if (!res.IsValid) {
Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving stats failed.");
}
var filtered = res.Aggs.Filter("filtered");
if (filtered == null)
return new EventStatsResult();
var newTerms = filtered.Terms("new");
var stats = new EventStatsResult {
Total = filtered.DocCount,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0
};
var unique = filtered.Cardinality("unique");
if (unique != null && unique.Value.HasValue)
stats.Unique = (long)unique.Value;
var timeline = filtered.DateHistogram("timelime");
if (timeline != null) {
stats.Timeline.AddRange(timeline.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("tl_unique");
if (timelineUnique != null && timelineUnique.Value.HasValue)
count = (long)timelineUnique.Value;
var timelineNew = i.Terms("tl_new");
return new TimelineItem {
Date = i.Date,
Total = i.DocCount,
Unique = count,
New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0
};
}));
}
stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
var totalHours = stats.End.Subtract(stats.Start).TotalHours;
if (totalHours > 0.0)
stats.AvgPerHour = stats.Total / totalHours;
if (stats.Timeline.Count <= 0)
return stats;
var firstOccurrence = filtered.Min("first_occurrence");
if (firstOccurrence != null && firstOccurrence.Value.HasValue)
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
var lastOccurrence = filtered.Max("last_occurrence");
if (lastOccurrence != null && lastOccurrence.Value.HasValue)
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
return stats;
}
