public HangingRequestsHandler( ElasticSearchClient <IndexRequestDbo> elasticClient, ElasticSearchOptions elasticOptions) { _elasticClient = elasticClient; _elasticOptions = elasticOptions; }
public static IWebHostBuilder UseCkoSerilog <T>(this IWebHostBuilder builder) { //A note about logging: //ILogger<Startup> lives in Microsoft.Extensions.Logging //Whereas Log.Logger lives in Serilog.ILogger //Hence also the reason for the AddSingleton(Log.Logger) in ConfigureServices. //However, both log to the same Serilog sinks as configured in Program.cs var elasticSearchOptions = new ElasticSearchOptions(); //This is the preferred way of configuring Serilog. Try not to do in in Startup, this has //some disadvantages: https://github.com/serilog/serilog-aspnetcore //UseSerilog configures DI and sets Log.Logger builder.UseSerilog((hostingContext, loggerConfiguration) => { hostingContext.Configuration.GetSection("ElasticSearch").Bind(elasticSearchOptions); loggerConfiguration.ConfigureCkoLogger <T>(hostingContext.Configuration, hostingContext.HostingEnvironment.EnvironmentName, elasticSearchOptions); }); //Despite .UseSerilog() we still need to inject Serilog.ILogger //Some of our classes depend on Serilog.ILogger //Others on Microsoft.Extensions.Logging.ILogger builder.ConfigureServices(collection => collection.AddSingleton <ILogger>(Log.Logger) ); return(builder); }
public QueueForIndex(ElasticSearchClient <IndexRequestDbo> client, ElasticSearchOptions options) { _client = client; _options = options; EnsureIndexInElasticCreated(); }
public List <string> ExistsByStackIds(string[] stackIds) { var options = new ElasticSearchOptions <PersistentEvent>().WithStackIds(stackIds); var descriptor = new SearchDescriptor <PersistentEvent>().Filter(options.GetElasticSearchFilter()).Source(s => s.Include("stack_id")).Size(stackIds.Length); var results = _elasticClient.Search <PersistentEvent>(descriptor); return(results.Documents.Select(e => e.StackId).ToList()); }
public Reindexer( ElasticSearchClient <IndexRequestDbo> requestsClient, ElasticSearchOptions options, QueueForIndex queueForIndex) { _requestsClient = requestsClient; _options = options; _queueForIndex = queueForIndex; }
public static void AddElasticSearch(this IServiceCollection services, Action <ConnectionSettings> configureElasticSearch) { services.AddSingleton(cp => { ElasticSearchOptions options = cp.GetRequiredService <IOptionsSnapshot <ElasticSearchOptions> >().Value; ConnectionSettings connectionSettings = new ConnectionSettings(new StaticConnectionPool(options.Uris)); configureElasticSearch(connectionSettings); return(connectionSettings); }); services.AddScoped(c => new ElasticClient(c.GetService <ConnectionSettings>())); }
private void AddElastic(IServiceCollection services) { services.AddScoped <IElasticSearchService, ElasticSearchService>(); services.AddScoped <ElasticClient>(provider => { var elasticOptions = new ElasticSearchOptions(); Configuration.Bind("ElasticSearchOptions", elasticOptions); var uris = elasticOptions.HostUrls.Split(",").Select(u => new Uri(u)).ToArray(); var connectionPool = new SniffingConnectionPool(uris); var settings = new ConnectionSettings(connectionPool); var client = new ElasticClient(settings); return(client); }); }
public Indexer( ElasticSearchClient <Document> client, ElasticSearchOptions options, QueueForIndex indexRequestsQueue, IHttpClientFactory httpClientFactory, SiteMapGetter siteMapGetter, PagesPerSiteLimiter pagesPerSiteLimiter) { _client = client; _options = options; this.indexRequestsQueue = indexRequestsQueue; this.siteMapGetter = siteMapGetter; this.pagesPerSiteLimiter = pagesPerSiteLimiter; httpClient = httpClientFactory.CreateClient("Page downloader"); }
public static void AddElasticsearch(this IServiceCollection services, Action <ElasticSearchOptions> action = null) { ElasticSearchOptions options = new ElasticSearchOptions(); action?.Invoke(options); services.AddSingleton(options); services.AddSingleton <IEsClientProvider, EsClientProvider>(); services.AddSingleton <IElasticSearchTransPortService, ElasticSearchTransPortService>(); TransPortServiceDependencyInjection.AddFunc((serviceProvider, name) => { if (name.Equals(ElasticsearchConstant.ELASTICSEARCHNAME, StringComparison.OrdinalIgnoreCase)) { return(serviceProvider.GetService <IElasticSearchTransPortService>()); } return(null); }); }
public static string Format(BooleanExpression booleanExpression, ElasticSearchOptions elasticSearchOptions) { elasticSearchOptions ??= new ElasticSearchOptions(); var visitor = new ElasticSearchVisitor(elasticSearchOptions); visitor.Visit(booleanExpression); var filter = visitor.GetBool(); while (BoolOptimizer.Optimize(filter) || RangeOptimizer.OptimizeRange(filter)) { //Iterate until no more optimizations } var serializedFilter = JsonSerializer.Serialize(filter, new JsonSerializerOptions() { Converters = { new BoolSerializer() } }); return(serializedFilter); }
public ElasticSearchVisitor(ElasticSearchOptions elasticSearchOptions) { _elasticSearchOptions = elasticSearchOptions; }
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string query = null, TimeSpan? utcOffset = null, int max = 25, int desiredDataPoints = 10) { if (!utcOffset.HasValue) utcOffset = TimeSpan.Zero; var allowedTerms = new[] { "tags", "stack_id", "organization_id" }; if (!allowedTerms.Contains(term)) throw new ArgumentException("Must be a valid term.", "term"); var options = new ElasticSearchOptions<PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange(); _client.EnableTrace(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(String.Join(",", options.Indices)) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => options.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(utcOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } _client.DisableTrace(); var stats = new EventTermStatsResult { Total = res.Aggs.Filter("filtered").DocCount, }; stats.Terms.AddRange(res.Aggs.Filter("filtered").DateHistogram("terms").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique").Value; if (timelineUnique.HasValue) count = (long)timelineUnique.Value; var item = new TermStatsItem { Total = i.DocCount, Unique = count, New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0 }; var firstOccurrence = i.Min("first_occurrence").Value; var lastOccurrence = i.Max("last_occurrence").Value; if (firstOccurrence.HasValue) item.FirstOccurrence = firstOccurrence.Value.ToDateTime(); if (lastOccurrence.HasValue) item.LastOccurrence = lastOccurrence.Value.ToDateTime(); item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); return item; })); stats.Start = utcStart.Add(utcOffset.Value); stats.End = utcEnd.Add(utcOffset.Value); return stats; }
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string query = null, TimeSpan? utcOffset = null, int desiredDataPoints = 100) { if (!utcOffset.HasValue) utcOffset = TimeSpan.Zero; var options = new ElasticSearchOptions<PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange(); _client.EnableTrace(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(String.Join(",", options.Indices)) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => options.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(utcOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } _client.DisableTrace(); var stats = new EventStatsResult { Total = res.Aggs.Filter("filtered").DocCount, New = res.Aggs.Filter("filtered").Terms("new").Items.Count > 0 ? res.Aggs.Filter("filtered").Terms("new").Items[0].DocCount : 0 }; var unique = res.Aggs.Filter("filtered").Cardinality("unique").Value; if (unique.HasValue) stats.Unique = (long)unique.Value; stats.Timeline.AddRange(res.Aggs.Filter("filtered").DateHistogram("timelime").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique").Value; if (timelineUnique.HasValue) count = (long)timelineUnique.Value; return new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0 }; })); stats.Start = utcStart.Add(utcOffset.Value); stats.End = utcEnd.Add(utcOffset.Value); stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = res.Aggs.Filter("filtered").Min("first_occurrence").Value; var lastOccurrence = res.Aggs.Filter("filtered").Max("last_occurrence").Value; if (firstOccurrence.HasValue) stats.FirstOccurrence = firstOccurrence.Value.ToDateTime(); if (lastOccurrence.HasValue) stats.LastOccurrence = lastOccurrence.Value.ToDateTime(); return stats; }
public List<string> ExistsByStackIds(string[] stackIds) { var options = new ElasticSearchOptions<PersistentEvent>().WithStackIds(stackIds); var descriptor = new SearchDescriptor<PersistentEvent>().Filter(options.GetElasticSearchFilter()).Source(s => s.Include("stack_id")).Size(stackIds.Length); var results = _elasticClient.Search<PersistentEvent>(descriptor); return results.Documents.Select(e => e.StackId).ToList(); }
public static IServiceCollection AddElasticSearch(this IServiceCollection services, Action <ElasticSearchOptions> options) { var opt = new ElasticSearchOptions(); options.Invoke(opt); if (opt.Active) { var conn_settings = new ConnectionSettings(new Uri(opt.Url)) .DefaultIndex(opt.DefaultIndex) .DefaultMappingFor <Person>(p_desc => p_desc .IndexName("person") .Ignore(p => p.Artists) ) .DefaultMappingFor <Artist>(a_desc => a_desc .IndexName("artist") .Ignore(a => a.CurrentMembers) .Ignore(a => a.PastMembers) .Ignore(a => a.Songs) ) .DefaultMappingFor <Song>(s_desc => s_desc .IndexName("song") .Ignore(s => s.Artists) .Ignore(s => s.Lyrics) ); var client = new ElasticClient(conn_settings); var response_people = client.Indices.Create("person", index => index .Map <Person>(map => map .Properties(props => props .Text(desc => desc.Name(person => person.FirstName)) .Text(desc => desc.Name(person => person.LastName)) .Date(desc => desc.Name(person => person.Born)) .Date(desc => desc.Name(person => person.Died)) .Completion(desc => desc.Name(p => p.NameSuggest)) ) ) ); var response_artists = client.Indices.Create("artist", index => index .Map <Artist>(map => map .Properties(props => props .Text(desc => desc.Name(artist => artist.Name)) .Text(a_comp => a_comp.Name(a => a.Name)) .Number(desc => desc.Name(artist => artist.YearStarted)) .Number(desc => desc.Name(artist => artist.YearQuit)) .Completion(desc => desc.Name(a => a.NameSuggest)) ) ) ); var response_songs = client.Indices.Create("song", index => index .Map <Song>(map => map .Properties(props => props.Text(desc => desc.Name(song => song.Title)) .Completion(s_comp => s_comp.Name(s => s.TitleSuggest)) ) ) ); return(services.AddSingleton <IElasticClient>(client)); } else { return(services.AddSingleton <IElasticClient>(new ElasticClient())); } }
public void OnGet() { ElsOptions = _elsOptions.Value; }
public async Task<EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) throw new ArgumentException("Must be a valid term.", nameof(term)); var filter = new ElasticSearchOptions<PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = await _elasticClient.SearchAsync<PersistentEvent>(s => s .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Filter(d => filter.GetElasticSearchFilter()) .SortAscending(ev => ev.Date) .Take(1)).AnyContext(); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = await _elasticClient.SearchAsync<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Query(filter.GetElasticSearchQuery()) .Aggregations(agg => agg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ).AnyContext(); _elasticClient.DisableTrace(); if (!res.IsValid) { Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventTermStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0, Start = utcStart.SafeAdd(displayTimeOffset.Value), End = utcEnd.SafeAdd(displayTimeOffset.Value) }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) stats.Unique = (long)unique.Value; var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var terms = res.Aggs.Terms("terms"); if (terms == null) return stats; stats.Terms.AddRange(terms.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique"); if (timelineUnique?.Value != null) count = (long)timelineUnique.Value; var termNew = i.Terms("new"); var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0 }; var termFirstOccurrence = i.Min("first_occurrence"); if (termFirstOccurrence?.Value != null) item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var termLastOccurrence = i.Max("last_occurrence"); if (termLastOccurrence?.Value != null) item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var timeLine = i.DateHistogram("timelime"); if (timeLine != null) { item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); } return item; })); return stats; }
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) { displayTimeOffset = TimeSpan.Zero; } var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) { throw new ArgumentException("Must be a valid term.", "term"); } var filter = new ElasticSearchOptions <PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(); _client.EnableTrace(); // if no start date then figure out first event date if (!filter.UseStartDate) { var result = _client.Search <PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1)); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } _client.DisableTrace(); var filtered = res.Aggs.Filter("filtered"); if (filtered == null) { return(new EventTermStatsResult()); } var stats = new EventTermStatsResult { Total = filtered.DocCount }; stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique").Value; if (timelineUnique.HasValue) { count = (long)timelineUnique.Value; } var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0 }; var firstOccurrence = i.Min("first_occurrence").Value; var lastOccurrence = i.Max("last_occurrence").Value; if (firstOccurrence.HasValue) { item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } if (lastOccurrence.HasValue) { item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); return(item); })); stats.Start = utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); return(stats); }
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string query = null, TimeSpan?utcOffset = null, int desiredDataPoints = 100) { if (!utcOffset.HasValue) { utcOffset = TimeSpan.Zero; } var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange(); _client.EnableTrace(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(String.Join(",", options.Indices)) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => options.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(utcOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } _client.DisableTrace(); var stats = new EventStatsResult { Total = res.Aggs.Filter("filtered").DocCount, New = res.Aggs.Filter("filtered").Terms("new").Items.Count > 0 ? res.Aggs.Filter("filtered").Terms("new").Items[0].DocCount : 0 }; var unique = res.Aggs.Filter("filtered").Cardinality("unique").Value; if (unique.HasValue) { stats.Unique = (long)unique.Value; } stats.Timeline.AddRange(res.Aggs.Filter("filtered").DateHistogram("timelime").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique").Value; if (timelineUnique.HasValue) { count = (long)timelineUnique.Value; } return(new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = i.Terms("tl_new").Items.Count > 0 ? i.Terms("tl_new").Items[0].DocCount : 0 }); })); stats.Start = utcStart.Add(utcOffset.Value); stats.End = utcEnd.Add(utcOffset.Value); stats.AvgPerHour = stats.Total / stats.End.Subtract(stats.Start).TotalHours; if (stats.Timeline.Count <= 0) { return(stats); } var firstOccurrence = res.Aggs.Filter("filtered").Min("first_occurrence").Value; var lastOccurrence = res.Aggs.Filter("filtered").Max("last_occurrence").Value; if (firstOccurrence.HasValue) { stats.FirstOccurrence = firstOccurrence.Value.ToDateTime(); } if (lastOccurrence.HasValue) { stats.LastOccurrence = lastOccurrence.Value.ToDateTime(); } return(stats); }
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) throw new ArgumentException("Must be a valid term.", "term"); var filter = new ElasticSearchOptions<PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(); _client.EnableTrace(); // if no start date then figure out first event date if (!filter.UseStartDate) { var result = _client.Search<PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1)); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } _client.DisableTrace(); var filtered = res.Aggs.Filter("filtered"); if (filtered == null) return new EventTermStatsResult(); var stats = new EventTermStatsResult { Total = filtered.DocCount }; stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique").Value; if (timelineUnique.HasValue) count = (long)timelineUnique.Value; var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0 }; var firstOccurrence = i.Min("first_occurrence").Value; var lastOccurrence = i.Max("last_occurrence").Value; if (firstOccurrence.HasValue) item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); if (lastOccurrence.HasValue) item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); return item; })); stats.Start = utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); return stats; }
public static LoggerConfiguration ConfigureCkoLogger <T>(this LoggerConfiguration loggerConfiguration, IConfiguration configuration, string envName, ElasticSearchOptions elasticSearchOptions) { loggerConfiguration .ReadFrom.Configuration(configuration) .Enrich.WithProperty("HostName", GetHostName()) .Enrich.WithProperty("Version", ReflectionUtils.GetAssemblyVersion <T>()) .Enrich.WithProperty("Environment", envName) .WriteTo.Elasticsearch( new ElasticsearchSinkOptions(elasticSearchOptions.Node) { AutoRegisterTemplate = elasticSearchOptions.AutoRegisterTemplate, IndexFormat = elasticSearchOptions.IndexFormat, MinimumLogEventLevel = Serilog.Events.LogEventLevel.Information }); return(loggerConfiguration); }
private static IServiceCollection AddElasticSearch(this IServiceCollection services, ElasticSearchOptions options) { var uri = new Uri(options.BootstrapServers); var connectionSettings = new ConnectionSettings(uri); connectionSettings.EnableDebugMode(); if (!string.IsNullOrWhiteSpace(options.DefaultIndex)) { connectionSettings.DefaultIndex(options.DefaultIndex); } services.AddSingleton <IElasticClient>(new ElasticClient(connectionSettings)); return(services); }
public async Task <EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) { displayTimeOffset = TimeSpan.Zero; } var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) { throw new ArgumentException("Must be a valid term.", nameof(term)); } var filter = new ElasticSearchOptions <PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = await _elasticClient.SearchAsync <PersistentEvent>(s => s .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Filter(d => filter.GetElasticSearchFilter()) .SortAscending(ev => ev.Date) .Take(1)).AnyContext(); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = await _elasticClient.SearchAsync <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Query(filter.GetElasticSearchQuery()) .Aggregations(agg => agg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ).AnyContext(); _elasticClient.DisableTrace(); if (!res.IsValid) { Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventTermStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0, Start = utcStart.SafeAdd(displayTimeOffset.Value), End = utcEnd.SafeAdd(displayTimeOffset.Value) }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) { stats.Unique = (long)unique.Value; } var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) { stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) { stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var terms = res.Aggs.Terms("terms"); if (terms == null) { return(stats); } stats.Terms.AddRange(terms.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique"); if (timelineUnique?.Value != null) { count = (long)timelineUnique.Value; } var termNew = i.Terms("new"); var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0 }; var termFirstOccurrence = i.Min("first_occurrence"); if (termFirstOccurrence?.Value != null) { item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var termLastOccurrence = i.Max("last_occurrence"); if (termLastOccurrence?.Value != null) { item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var timeLine = i.DateHistogram("timelime"); if (timeLine != null) { item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); } return(item); })); return(stats); }
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticSearchOptions<PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = _elasticClient.Search<PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1)); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = _elasticClient.Search<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository<PersistentEvent>.EventsIndexName, "-*")) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ) ) ); _elasticClient.DisableTrace(); if (!res.IsValid) { Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var filtered = res.Aggs.Filter("filtered"); if (filtered == null) return new EventStatsResult(); var newTerms = filtered.Terms("new"); var stats = new EventStatsResult { Total = filtered.DocCount, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = filtered.Cardinality("unique"); if (unique != null && unique.Value.HasValue) stats.Unique = (long)unique.Value; var timeline = filtered.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique != null && timelineUnique.Value.HasValue) count = (long)timelineUnique.Value; var timelineNew = i.Terms("tl_new"); return new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }; })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Total / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = filtered.Min("first_occurrence"); if (firstOccurrence != null && firstOccurrence.Value.HasValue) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = filtered.Max("last_occurrence"); if (lastOccurrence != null && lastOccurrence.Value.HasValue) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string query = null, TimeSpan?utcOffset = null, int max = 25, int desiredDataPoints = 10) { if (!utcOffset.HasValue) { utcOffset = TimeSpan.Zero; } var allowedTerms = new[] { "tags", "stack_id", "organization_id" }; if (!allowedTerms.Contains(term)) { throw new ArgumentException("Must be a valid term.", "term"); } var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange(); _client.EnableTrace(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(String.Join(",", options.Indices)) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => options.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(utcOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } _client.DisableTrace(); var stats = new EventTermStatsResult { Total = res.Aggs.Filter("filtered").DocCount, }; stats.Terms.AddRange(res.Aggs.Filter("filtered").DateHistogram("terms").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique").Value; if (timelineUnique.HasValue) { count = (long)timelineUnique.Value; } var item = new TermStatsItem { Total = i.DocCount, Unique = count, New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0 }; var firstOccurrence = i.Min("first_occurrence").Value; var lastOccurrence = i.Max("last_occurrence").Value; if (firstOccurrence.HasValue) { item.FirstOccurrence = firstOccurrence.Value.ToDateTime(); } if (lastOccurrence.HasValue) { item.LastOccurrence = lastOccurrence.Value.ToDateTime(); } item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); return(item); })); stats.Start = utcStart.Add(utcOffset.Value); stats.End = utcEnd.Add(utcOffset.Value); return(stats); }
public EventStatsResult GetOccurrenceStats(DateTime utcStart, DateTime utcEnd, string systemFilter, string userFilter = null, TimeSpan? displayTimeOffset = null, int desiredDataPoints = 100) { if (!displayTimeOffset.HasValue) displayTimeOffset = TimeSpan.Zero; var filter = new ElasticSearchOptions<PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter<PersistentEvent>.Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(String.Format("'{0}-'yyyyMM", _eventIndex.VersionedName)); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = _elasticClient.Search<PersistentEvent>(s => s .IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Filter(d => filter.GetElasticSearchFilter()) .SortAscending(ev => ev.Date) .Take(1)); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(String.Format("'{0}-'yyyyMM", _eventIndex.VersionedName)); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = _elasticClient.Search<PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .DateHistogram("timelime", t => t .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .Aggregations(agg2 => agg2 .Cardinality("tl_unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("tl_new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) ) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", t => t.Field(ev => ev.Date)) .Max("last_occurrence", t => t.Field(ev => ev.Date)) ) ) ) ); _elasticClient.DisableTrace(); if (!res.IsValid) { Log.Error().Message("Retrieving stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving stats failed."); } var filtered = res.Aggs.Filter("filtered"); if (filtered == null) return new EventStatsResult(); var newTerms = filtered.Terms("new"); var stats = new EventStatsResult { Total = filtered.DocCount, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0 }; var unique = filtered.Cardinality("unique"); if (unique != null && unique.Value.HasValue) stats.Unique = (long)unique.Value; var timeline = filtered.DateHistogram("timelime"); if (timeline != null) { stats.Timeline.AddRange(timeline.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("tl_unique"); if (timelineUnique != null && timelineUnique.Value.HasValue) count = (long)timelineUnique.Value; var timelineNew = i.Terms("tl_new"); return new TimelineItem { Date = i.Date, Total = i.DocCount, Unique = count, New = timelineNew != null && timelineNew.Items.Count > 0 ? timelineNew.Items[0].DocCount : 0 }; })); } stats.Start = stats.Timeline.Count > 0 ? stats.Timeline.Min(tl => tl.Date).SafeAdd(displayTimeOffset.Value) : utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); var totalHours = stats.End.Subtract(stats.Start).TotalHours; if (totalHours > 0.0) stats.AvgPerHour = stats.Total / totalHours; if (stats.Timeline.Count <= 0) return stats; var firstOccurrence = filtered.Min("first_occurrence"); if (firstOccurrence != null && firstOccurrence.Value.HasValue) stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); var lastOccurrence = filtered.Max("last_occurrence"); if (lastOccurrence != null && lastOccurrence.Value.HasValue) stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); return stats; }